chore(fff-core): clippy 1.98 (#808)

clippy 1.98 added chunks_exact_to_as_chunks, which fails CI under
-D warnings. SEEN_WORDS is a multiple of 8, so the remainder is
always empty and the fixed-size chunk keeps codegen identical or
better.

Co-authored-by: gustav-fff <286169375+gustav-fff@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,58 @@
+name: Bug report
+description: Report a crash, hang, or incorrect behavior in fff (any frontend — nvim plugin, Node/Bun SDK, MCP server, C SDK).
+title: "[Bug]: "
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: frontend
+    attributes:
+      label: Which fff frontend?
+      options:
+        - Neovim plugin (fff.nvim)
+        - MCP server (fff-mcp)
+        - Node SDK (@ff-labs/fff-node)
+        - Bun SDK
+        - C SDK (libfff)
+        - Other / multiple
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: has logs
+      description: |
+        Attach your fff log file — the single most useful thing for debugging.
+
+        fff writes a fresh log file on every process startup, named `fff+<unix-ts>+<pid>.log`, and keeps the last 20. Find the file matching your crashed/buggy run and drag-and-drop it here (or paste its contents).
+
+        Where the log files live:
+
+        | Frontend | Linux / macOS | Windows |
+        |---|---|---|
+        | Neovim plugin | `~/.local/state/nvim/log/fff+*.log` | `%LOCALAPPDATA%\nvim-data\log\fff+*.log` |
+        | MCP server (`fff-mcp`) | `~/.cache/fff_mcp+*.log` (override with `--log-file`) | `%LOCALAPPDATA%\fff_mcp+*.log` |
+        | Node / Bun SDK | path you passed as `logFilePath` to `FileFinder.create({...})` | same |
+        | C SDK | path you passed as `log_file_path` in `FffCreateOptions` | same |
+
+        Neovim users: run `:FFFOpenLog` to open the current session's log directly.
+
+        Also some useful commands:
+
+        ```sh
+        # Neovim plugin
+        ls -t ~/.local/state/nvim/log/fff+*.log | head -1
+
+        # MCP server
+        ls -t ~/.cache/fff_mcp+*.log | head -1
+        ```
+    validations:
+      required: false
+
+  - type: textarea
+    id: body
+    attributes:
+      label: Description
+      description: Please provide as much helpful information as you can
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Discussion / question
+    url: https://github.com/dmtrKovalenko/fff/discussions
+    about: For usage questions, design discussion, or anything that's not a bug or feature request.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,34 @@
+name: Feature request
+description: Suggest something new for fff
+title: "[Suggestion]: "
+labels: ["enhancement"]
+body:
+  - type: dropdown
+    id: frontend
+    attributes:
+      label: Which fff frontend(s)?
+      multiple: true
+      options:
+        - Neovim plugin (fff.nvim)
+        - MCP server (fff-mcp)
+        - Node SDK (@ff-labs/fff-node)
+        - Bun SDK (@ff-labs/fff-bun)
+        - C lib (libfff)
+        - Core or Rust crate
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem are you trying to solve?
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: If you have an idea of the shape of the API, describe it here.
+    validations:
+      required: false

.github/workflows/external-tests.yml

@@ -3,8 +3,14 @@ name: e2e Tests
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 env:
   CARGO_TERM_COLOR: always
@@ -12,6 +18,14 @@ env:
   # Force Node 24 for all JS-based actions to avoid the libuv
   # process_title assertion crash on Windows (known Node 20 bug).
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  # e2e only needs a working binary, so skip fat LTO (same settings as the `ci`
+  # profile releases ship). Overriding release keeps artifacts in target/release.
+  CARGO_PROFILE_RELEASE_LTO: thin
+  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   lua-tests:
@@ -19,6 +33,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     # e2e tests could be flaky on CI so we do not block release creation if they failed
     continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }}
+    # alpine-musl installs rust via apk and has no sccache, so keep this scoped.
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      # fff-search alone exceeds the 600s default on windows, and the server
+      # sees no new requests while it compiles, so it would idle out mid-unit.
+      SCCACHE_IDLE_TIMEOUT: "0"
     strategy:
       fail-fast: false
       matrix:
@@ -26,7 +47,6 @@ jobs:
           - os: ubuntu-latest
           - os: macos-latest
           - os: windows-latest
-            target: x86_64-pc-windows-msvc
     steps:
       - uses: actions/checkout@v5
       - uses: oven-sh/setup-bun@v2
@@ -43,20 +63,23 @@ jobs:
           cache-on-failure: false
           cache-key: "v2-lua-e2e"
           rustflags: ""
-          target: ${{ matrix.target || '' }}
 
-      - name: Build Rust binary (Windows)
-        if: matrix.target
-        run: cargo build --release --target ${{ matrix.target }} -p fff-nvim --features zlob
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
 
-      - name: Copy binary to target/release (Windows)
-        if: matrix.target
+      - name: Build Rust binary
         shell: bash
-        run: |
-          cp target/${{ matrix.target }}/release/fff_nvim.dll target/release/fff_nvim.dll
+        run: make build-e2e
+
+      - name: sccache stats
+        if: always()
+        shell: bash
+        run: sccache --show-stats
 
       - name: Verify Windows DLL has no unexpected dependencies
-        if: matrix.target
+        if: matrix.os == 'windows-latest'
         shell: pwsh
         run: |
           # Find dumpbin via vswhere (always available on GitHub Actions Windows runners)
@@ -72,10 +95,6 @@ jobs:
             exit 1
           }
 
-      - name: Build Rust binary
-        if: ${{ !matrix.target }}
-        run: cargo build --release -p fff-nvim --features zlob
-
       - name: Install Neovim
         uses: rhysd/action-setup-vim@v1
         with:
@@ -90,14 +109,36 @@ jobs:
         shell: bash
         run: make test-lua
 
+      - name: Dump fff trace log on failure
+        if: failure()
+        shell: bash
+        run: |
+          # init_tracing writes session files named fff-test+<ts>+<pid>.log
+          found=0
+          for f in fff-test*.log; do
+            [ -f "$f" ] || continue
+            found=1
+            echo "=== $f ==="
+            cat "$f"
+          done
+          if [ "$found" = 0 ]; then
+            echo "(no log file produced)"
+          fi
+
       - name: Run version resolution tests
         shell: bash
         run: make test-version
 
-      - name: Run bun tests
+      - name: Run non windows tests
         shell: bash
         if: ${{ matrix.os != 'windows-latest' }}
-        run: make test-bun
+        run: |
+          make test-bun
+          make test-c-api
+
+      - name: Verify bun --compile 
+        shell: bash
+        run: make test-bun-compile
 
       - name: Install Node.js
         if: ${{ matrix.os != 'ubuntu-latest' }}
@@ -107,8 +148,89 @@ jobs:
 
       - name: Install node dependencies
         shell: bash
-        run: cd packages/fff-node && npm install
+        run: cd packages && npm ci
 
       - name: Run node tests
         shell: bash
         run: make test-node
+
+  # Regression for https://github.com/dmtrKovalenko/fff/issues/480: build &
+  # run @ff-labs/fff-node end-to-end on real Alpine Linux (musl). Forces
+  # findBinary() through the npm-package resolver so detectLinuxLibc()
+  # actually runs.
+  alpine-musl:
+    name: e2e (alpine-musl)
+    runs-on: ubuntu-latest
+    container: node:22-alpine
+    continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }}
+    defaults:
+      run:
+        shell: sh
+    steps:
+      - name: Install build deps
+        run: apk add --no-cache git rust cargo musl-dev
+
+      - uses: actions/checkout@v5
+
+      # libgit2 refuses repos owned by a different user; checkout in a
+      # container can land at a uid mismatch, so opt every dir in.
+      - name: Mark workspace safe for git
+        run: git config --global --add safe.directory '*'
+
+      - name: Sanity check libc is musl
+        run: |
+          if ! ldd --version 2>&1 | grep -qi musl; then
+            echo "FAIL: container is not running musl libc"
+            exit 1
+          fi
+
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: alpine-musl-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            alpine-musl-cargo-
+
+      - name: Build libfff_c (musl)
+        run: cargo build --release -p fff-c
+
+      - name: Install workspace npm deps
+        working-directory: packages
+        run: npm ci
+
+      # Upstream @yuuang/ffi-rs-linux-x64-musl ships with libc:"glibc" in
+      # its package.json (a publishing bug in ffi-rs), so npm filters it
+      # out. Force-install it so the FFI runtime is present on Alpine.
+      - name: Install ffi-rs musl runtime
+        working-directory: packages
+        run: |
+          FFI_RS_VERSION=$(node -p "require('ffi-rs/package.json').version")
+          npm install --no-package-lock --no-save --force \
+            "@yuuang/ffi-rs-linux-x64-musl@${FFI_RS_VERSION}"
+
+      # Stage the freshly built libfff_c.so as the platform npm package
+      # so findBinary() resolves through the @ff-labs/fff-bin-* path —
+      # this is what exercises detectLinuxLibc().
+      - name: Stage musl bin package
+        run: |
+          PKG_DIR=packages/node_modules/@ff-labs/fff-bin-linux-x64-musl
+          mkdir -p "$PKG_DIR"
+          cp target/release/libfff_c.so "$PKG_DIR/libfff_c.so"
+          cat >"$PKG_DIR/package.json" <<'JSON'
+          { "name": "@ff-labs/fff-bin-linux-x64-musl", "version": "0.0.0" }
+          JSON
+
+      # fff-node builds with bun; the npm `bun` package ships musl binaries
+      - name: Install bun
+        run: npm install -g bun
+
+      - name: Build fff-node
+        working-directory: packages/fff-node
+        run: npm run build
+
+      - name: Run fff-node e2e suite
+        working-directory: packages/fff-node
+        run: node test/e2e.mjs

.github/workflows/lua.yml

@@ -3,8 +3,18 @@ name: Lua CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   lua-ls:

.github/workflows/nix.yml

@@ -3,8 +3,18 @@ name: Nix CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   check:

.github/workflows/oxc.yml

@@ -0,0 +1,46 @@
+name: Oxc
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "packages/**"
+      - .github/workflows/oxc.yml
+  pull_request:
+    paths:
+      - "packages/**"
+      - .github/workflows/oxc.yml
+
+env:
+  CLICOLOR: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
+jobs:
+  oxc:
+    name: Check packages with oxfmt and oxlint
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: packages
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install
+        run: bun install --frozen-lockfile
+
+      - name: oxfmt
+        run: bun run format:check
+
+      - name: oxlint
+        run: bun run lint -- -f github --deny-warnings

.github/workflows/panvimdoc.yaml

@@ -1,7 +1,8 @@
 on:
-  push:
-    branches-ignore:
-      - main
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
 name: docs
 
 jobs:
@@ -9,27 +10,76 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      pull-requests: write
     steps:
       - uses: actions/checkout@v5
         with:
-          # fetch last 2 commits required for auto force push back
+          ref: main
           fetch-depth: 2
 
+      - name: Extract Neovim section from README.md
+        run: |
+          awk '
+            /^<details id="neovim-plugin">/ { capture=1; next }
+            capture && /^<\/details>/        { capture=0; exit }
+            capture && /^<summary>$/         { next }
+            capture && /^<\/summary>$/       { next }
+            capture && /<h2>.*<\/h2>/ {
+              gsub(/<\/?h2>/, "")
+              sub(/^[[:space:]]+/, "")
+              print "# " $0
+              print ""
+              print "The best file search picker for Neovim. Frecency-ranked, typo-resistant, git-award, very fast."
+              print ""
+              next
+            }
+            capture { print }
+          ' README.md > .panvimdoc-input.md
+          test -s .panvimdoc-input.md
+
       - name: panvimdoc
         uses: kdheepak/panvimdoc@main
         with:
           vimdoc: fff.nvim
+          pandoc: .panvimdoc-input.md
           version: "Neovim >= 0.10.0"
           demojify: true
           treesitter: true
 
-      - name: Get last commit message
-        id: last-commit
-        run: |
-          echo "message=$(git log -1 --pretty=%s)" >> $GITHUB_OUTPUT
-          echo "author=$(git log -1 --pretty=\"%an <%ae>\")" >> $GITHUB_OUTPUT
+      - name: Cleanup intermediate file
+        run: rm -f .panvimdoc-input.md
 
-      - uses: stefanzweifel/git-auto-commit-action@v6
+      # panvimdoc stamps "Last change: <today>" every run, so a daily cron always
+      # produces a one-line diff. Skip the PR unless a non-date line changed.
+      - name: Detect real doc changes
+        id: docdiff
+        run: |
+          if git diff --quiet -I 'Last change:' -- doc/fff.nvim.txt; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create pull request
+        id: cpr
+        if: steps.docdiff.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
         with:
-          commit_author: ${{ steps.last-commit.outputs.author }}
-          commit_message: "chore: Update docs for - ${{ steps.last-commit.outputs.message }}"
+          branch: bot/regenerate-vimdoc
+          token: ${{ secrets.GUSTAV_PAT }}
+          delete-branch: true
+          title: "chore: regenerate Neovim vimdoc"
+          commit-message: |
+            chore: regenerate Neovim vimdoc
+
+            Co-authored-by: Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>
+          author: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>"
+          committer: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>"
+          body: Automated vimdoc regeneration from README.md, scribed by Gustav.
+          add-paths: doc/fff.nvim.txt
+
+      - name: Enable auto-merge
+        if: steps.cpr.outputs.pull-request-number
+        env:
+          GH_TOKEN: ${{ secrets.GUSTAV_PAT }}
+        run: gh pr merge --auto --squash "${{ steps.cpr.outputs.pull-request-number }}"

.github/workflows/python.yml

@@ -0,0 +1,70 @@
+name: Python CI
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+env:
+  CARGO_TERM_COLOR: always
+  MACOSX_DEPLOYMENT_TARGET: "13.0"
+  CARGO_PROFILE_RELEASE_LTO: thin
+  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
+  SCCACHE_GHA_ENABLED: "true"
+  RUSTC_WRAPPER: sccache
+  # fff-search alone exceeds the 600s default on windows, and the server sees
+  # no new requests while it compiles, so it would idle out mid-unit.
+  SCCACHE_IDLE_TIMEOUT: "0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
+jobs:
+  test:
+    name: Python bindings (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-python"
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.11.14"
+          enable-cache: true
+
+      - name: Build and test Python bindings
+        working-directory: packages/fff-python
+        shell: bash
+        run: |
+          # --no-install-project: uv would otherwise build the wheel via
+          # maturin's PEP 517 backend, then `maturin develop` builds it again.
+          uv sync --all-extras --no-install-project
+          uv run maturin develop --release
+          uv run pytest -v

.github/workflows/release.yaml

@@ -1,21 +1,37 @@
-name: Prebuild
+name: Build & Publish
 
 on:
   push:
-    branches: [main, fix/download-version]
+    branches: [main, fix/npm-pulish]
     tags:
       - "v*"
   pull_request:
+  workflow_dispatch:
+    inputs:
+      publish_pypi:
+        description: "Manually build and publish Python wheels to PyPI"
+        required: false
+        default: false
+        type: boolean
 
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
 jobs:
   build-nvim:
     name: Build Neovim ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_IDLE_TIMEOUT: "0"
     permissions:
       contents: read
+      id-token: write
     strategy:
       matrix:
         include:
@@ -76,6 +92,18 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      # Cache the per-target build dir and cargo-zigbuild binary. Keyed by
+      # target so matrix legs don't collide. See issue on slow release CI.
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: nvim-${{ matrix.target }}
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
@@ -88,7 +116,7 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android')
         run: |
-          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for Android (Termux)
@@ -102,13 +130,13 @@ jobs:
           export AR_aarch64_linux_android="$NDK_BIN/llvm-ar"
           export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang"
 
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Ad-hoc sign macOS binary
@@ -119,7 +147,7 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Upload artifacts
@@ -131,6 +159,10 @@ jobs:
   build-c:
     name: Build C FFI ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_IDLE_TIMEOUT: "0"
     permissions:
       contents: read
     strategy:
@@ -168,6 +200,7 @@ jobs:
           - os: ubuntu-latest
             target: aarch64-linux-android
             artifact_name: target/aarch64-linux-android/ci/libfff_c.so
+            npm_package: fff-bin-android-arm64
             lib_filename: libfff_c.so
             ext: so
 
@@ -207,6 +240,16 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: c-${{ matrix.target }}
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
@@ -219,7 +262,7 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android')
         run: |
-          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for Android (Termux)
@@ -232,13 +275,13 @@ jobs:
           export AR_aarch64_linux_android="$NDK_BIN/llvm-ar"
           export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang"
 
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-c --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Ad-hoc sign macOS binary
@@ -249,11 +292,10 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Prepare npm package
-        if: "!contains(matrix.target, 'android')"
         shell: bash
         run: |
           # Copy the built binary into the platform npm package directory
@@ -266,7 +308,6 @@ jobs:
           path: c-lib-${{ matrix.target }}.*
 
       - name: Upload npm package artifact
-        if: "!contains(matrix.target, 'android')"
         uses: actions/upload-artifact@v4
         with:
           name: npm-${{ matrix.npm_package }}
@@ -275,6 +316,10 @@ jobs:
   build-mcp:
     name: Build MCP ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_IDLE_TIMEOUT: "0"
     permissions:
       contents: read
     strategy:
@@ -320,6 +365,16 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: mcp-${{ matrix.target }}
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
@@ -332,13 +387,13 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu')
         run: |
-          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}"
 
       - name: Ad-hoc sign macOS binary
@@ -349,7 +404,7 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}.exe"
 
       - name: Upload artifact
@@ -358,16 +413,110 @@ jobs:
           name: mcp-${{ matrix.target }}
           path: fff-mcp-${{ matrix.target }}*
 
+  build-python:
+    name: Build Python wheels ${{ matrix.target }} (${{ matrix.os }})
+    # Wheels are release artifacts; PR validation uses the develop build in
+    # python.yml, so skip the cross-compile matrix on pull requests.
+    if: github.event_name != 'pull_request'
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            target: x86_64
+            container: "off"
+          - os: ubuntu-latest
+            target: aarch64
+            container: "off"
+          - os: macos-latest
+            target: x86_64
+          - os: macos-latest
+            target: aarch64
+          - os: windows-latest
+            target: x86_64
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install Zig
+        uses: mlugg/setup-zig@v2
+        with:
+          version: 0.16.0
+
+      - name: Install cargo-zigbuild
+        if: contains(matrix.os, 'ubuntu')
+        run: cargo install cargo-zigbuild
+
+      - name: Install aarch64 cross compiler
+        if: matrix.target == 'aarch64' && contains(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+          CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+          CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+          AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --no-default-features --features zlob
+          sccache: "true"
+          working-directory: packages/fff-python
+          container: ${{ matrix.container || '' }}
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-wheels-${{ matrix.os }}-${{ matrix.target }}
+          path: packages/fff-python/dist/
+
+  build-python-sdist:
+    name: Build Python sdist
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+          working-directory: packages/fff-python
+
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-sdist
+          path: packages/fff-python/dist/
+
   release:
     name: Release
-    needs: [build-nvim, build-c, build-mcp]
+    needs: [build-nvim, build-c, build-mcp, build-python, build-python-sdist]
     runs-on: ubuntu-latest
-    # do not create releases on the forks (no permissions)
-    if: github.event_name != 'pull_request' || github.event.pull_request.user.login == 'dmtrKovalenko'
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
     permissions:
       contents: write
     steps:
+      # we have to make sure that pushing a commit on this workflow triggers the CI for nightly neovim
       - uses: actions/checkout@v5
+        with:
+          token: ${{ secrets.GUSTAV_PAT || github.token }}
 
       - name: Install Lua
         uses: leafo/gh-actions-lua@v12
@@ -420,6 +569,20 @@ jobs:
             rmdir "$dir" 2>/dev/null || true
           done
 
+      - name: Move Python wheels to release directory
+        working-directory: ./binaries
+        run: |
+          mkdir -p python
+          for dir in python-wheels-*/ python-sdist/; do
+            [ -d "$dir" ] || continue
+            for file in "$dir"*; do
+              if [ -f "$file" ]; then
+                mv "$file" "python/$(basename "$file")"
+              fi
+            done
+            rmdir "$dir" 2>/dev/null || true
+          done
+
       - name: Remove npm package artifacts from release binaries
         working-directory: ./binaries
         run: |
@@ -429,7 +592,7 @@ jobs:
         working-directory: ./binaries
         run: |
           ls -la
-          for file in *; do
+          for file in * python/*; do
             if [ -f "$file" ] && [[ ! "$file" == *.sha256 ]]; then
               sha256sum "$file" > "${file}.sha256"
             fi
@@ -439,19 +602,32 @@ jobs:
         id: version
         run: lua scripts/determine-version.lua
 
+      # Nightlies publish to a permanent per-sha tag (release_tag == version) so
+      # pinned/stale installs always fetch the binary built for their own commit.
+      # The rolling `nightly` tag is also moved to HEAD for "give me latest" tooling.
+      - name: Move rolling nightly tag to current commit
+        if: steps.version.outputs.is_release != 'true'
+        run: |
+          git tag -f nightly "${{ github.sha }}"
+          git push -f origin refs/tags/nightly
+
       - name: Upload Release Assets
         uses: softprops/action-gh-release@v2
         with:
           name: "${{ steps.version.outputs.version }}"
-          tag_name: "${{ steps.version.outputs.is_release == 'true' && format('v{0}', steps.version.outputs.version) || steps.version.outputs.version }}"
+          tag_name: "${{ steps.version.outputs.release_tag }}"
           token: ${{ github.token }}
-          files: ./binaries/*
+          files: |
+            ./binaries/*
+            ./binaries/python/*
           draft: false
           prerelease: ${{ steps.version.outputs.is_release != 'true' }}
           generate_release_notes: ${{ steps.version.outputs.is_release == 'true' }}
           body: |
             ${{ steps.version.outputs.is_release == 'true' && format('Release {0}', steps.version.outputs.version) || format('Nightly release from commit: {0}', github.sha) }}
 
+            npm packages, rust crates and python wheels are available under this version ${{ steps.version.outputs.version }}
+
             ## Neovim Plugin
             - `{target}.so` / `.dylib` / `.dll` - Lua module for Neovim
 
@@ -461,21 +637,71 @@ jobs:
             ## MCP Server
             - `fff-mcp-{target}` - MCP server binary
 
-            Install with:
+            ## Python Package
+            - `python/*.whl` / `python/*.tar.gz` - Python wheels and sdist
+            - Install from PyPI: `pip install fff-search` (when published)
+
+            Update mcp via:
             ```sh
-            curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash
+            curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.sh | bash
             ```
 
+      - name: Bump Homebrew formula (uses local checksums)
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        run: make bump-homebrew-formula VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries
+
+      - name: Pin SHAs in install-mcp.sh (uses local checksums)
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        run: make bump-install-mcp-sh VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries
+
+      - name: Commit formula + installer bump to main
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          # Workflow runs on detached HEAD at the v* tag — explicit target needed.
+          branch: main
+          commit_message: "chore: bump fff-mcp release artifacts to v${{ steps.version.outputs.version }}"
+          file_pattern: "Formula/fff-mcp.rb install-mcp.sh"
+          commit_user_name: github-actions[bot]
+          commit_user_email: 41898282+github-actions[bot]@users.noreply.github.com
+
+  pypi-publish:
+    name: Publish Python wheels to PyPI
+    needs: [build-python, build-python-sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    environment:
+      name: pypi
+      url: https://pypi.org/p/fff-search
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Download Python wheels and sdist
+        uses: actions/download-artifact@v4
+        with:
+          pattern: python-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          skip-existing: true
+
   crates-publish:
     name: Publish Rust crates
     needs: [build-nvim, build-c, build-mcp]
     runs-on: ubuntu-latest
-    if: >-
-      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v')))
-      || (github.event_name == 'pull_request' && github.head_ref == 'main')
-
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v5
+      - uses: rust-lang/crates-io-auth-action@v1
+        id: auth
 
       - name: Install Lua
         uses: leafo/gh-actions-lua@v12
@@ -484,7 +710,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Install cargo-edit
-        run: cargo install cargo-edit
+        run: cargo install cargo-edit --force --locked
 
       - name: Determine version
         id: version
@@ -492,18 +718,17 @@ jobs:
 
       - name: Publish crates
         env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
         run: make publish-crates V="${{ steps.version.outputs.version }}"
 
   npm-publish:
     name: Publish npm packages
     needs: [build-c]
     runs-on: ubuntu-latest
-    if: >-
-      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v')))
-      || (github.event_name == 'pull_request' && github.head_ref == 'main')
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/npm-pulish' || startsWith(github.ref, 'refs/tags/v'))
     permissions:
       contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v5
 
@@ -516,6 +741,8 @@ jobs:
           node-version: "25"
           registry-url: "https://registry.npmjs.org"
 
+      - uses: oven-sh/setup-bun@v2
+
       - name: Determine version
         id: version
         run: lua scripts/determine-version.lua
@@ -527,8 +754,6 @@ jobs:
           path: ./npm-packages
 
       - name: Publish platform packages
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -541,14 +766,22 @@ jobs:
               make set-npm-version PKG="$pkg_dir" VERSION="$VERSION"
 
               cd "$pkg_dir"
-              npm publish --tag "$TAG" --access public || echo "Failed to publish ${pkg_name} (may already exist)"
+              npm publish --tag "$TAG" --access public --provenance
               cd -
             fi
           done
 
+      # Every install must happen before the first set-npm-version: the bump
+      # injects platform optionalDependencies that only resolve once published,
+      # and npm ci validates the whole workspace against the lockfile.
+      - name: Install and build JS packages
+        working-directory: packages
+        run: |
+          npm ci
+          (cd fff-bun && npm run build)
+          (cd fff-node && npm run build)
+
       - name: Publish bun package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -557,11 +790,9 @@ jobs:
           make set-npm-version PKG=packages/fff-bun VERSION="$VERSION"
 
           cd packages/fff-bun
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-bun (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance
 
       - name: Publish Node.js package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -570,13 +801,9 @@ jobs:
           make set-npm-version PKG=packages/fff-node VERSION="$VERSION"
 
           cd packages/fff-node
-          npm install
-          npm run build
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-node (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance
 
       - name: Publish pi-fff package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -585,4 +812,4 @@ jobs:
           make set-npm-version PKG=packages/pi-fff VERSION="$VERSION"
 
           cd packages/pi-fff
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/pi-fff (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance

.github/workflows/rust.yml

@@ -3,14 +3,29 @@ name: Rust CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 env:
   CARGO_TERM_COLOR: always
   # Ensure consistent macOS deployment target across all compiled objects
   # (Rust, cc-compiled C code, and Zig-compiled zlob) to avoid linker warnings
   MACOSX_DEPLOYMENT_TARGET: "13"
+  # RUSTC_WRAPPER is set per job, since cargo fmt runs without sccache.
+  SCCACHE_GHA_ENABLED: "true"
+  # fff-search alone exceeds the 600s default on windows, and the server sees
+  # no new requests while it compiles, so it would idle out mid-unit.
+  SCCACHE_IDLE_TIMEOUT: "0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   test:
@@ -40,11 +55,19 @@ jobs:
           cache-key: "v1-rust"
           components: rustfmt, clippy
 
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Run tests
-        run: cargo test --features zlob --workspace --exclude fff-nvim
+        env:
+          RUSTC_WRAPPER: sccache
+        # fff-python requires full python o3 machinery which is very slow
+        run: cargo test --no-default-features --features zlob --workspace --exclude fff-nvim --exclude fff-python
 
   stress-test:
-    name: Stress Test (Watcher + Git)
+    name: Fuzz Tests
     runs-on: ${{ matrix.os }}
     strategy:
       # Keep going after one OS fails so we can see whether a bug
@@ -55,6 +78,11 @@ jobs:
     # Long-running; don't let a stuck watcher thread burn a full CI
     # timeout. Two scenarios should finish well under this limit.
     timeout-minutes: 20
+    env:
+      FFF_STRESS_CASES: "5"
+      FFF_STRESS_MIN_OPS: "30"
+      FFF_STRESS_MAX_OPS: "60"
+      RUSTC_WRAPPER: sccache
     steps:
       - uses: actions/checkout@v5
 
@@ -71,21 +99,22 @@ jobs:
           cache-key: "v1-rust-stress-${{ matrix.os }}"
           components: rustfmt, clippy
 
-      - name: Stress test (seeded / deterministic)
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
+      - name: Stress test seeded
         shell: bash
         run: make test-stress-seeded
-        env:
-          FFF_STRESS_CASES: "3"
-          FFF_STRESS_MIN_OPS: "30"
-          FFF_STRESS_MAX_OPS: "50"
 
-      - name: Stress test (random / fuzzy)
+      - name: Stress test random
         shell: bash
         run: make test-stress-random
-        env:
-          FFF_STRESS_CASES: "5"
-          FFF_STRESS_MIN_OPS: "30"
-          FFF_STRESS_MAX_OPS: "60"
+
+      - name: Stress test regressions
+        shell: bash
+        run: make test-stress-regressions
 
       - name: Upload proptest regressions on failure
         if: failure()
@@ -95,6 +124,39 @@ jobs:
           path: crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions
           if-no-files-found: ignore
 
+  build-i686:
+    name: Build i686-unknown-linux-gnu
+    runs-on: ubuntu-latest
+    # Verifies that fff-search compiles on 32-bit x86, where std::arch::x86_64
+    # is unavailable. SIMD paths are disabled on this target; only the scalar
+    # fallback should build. See issue #656.
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install cross toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-multilib g++-multilib
+
+      - name: Install Rust (i686 target)
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          target: i686-unknown-linux-gnu
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-i686"
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
+      - name: Build fff-search for i686
+        env:
+          RUSTC_WRAPPER: sccache
+        run: cargo build -p fff-search --target i686-unknown-linux-gnu
+
   fmt:
     name: cargo fmt
     runs-on: ubuntu-latest
@@ -125,6 +187,13 @@ jobs:
         with:
           toolchain: stable
           components: clippy
-          
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Run clippy
-        run: cargo clippy -- -D warnings
+        env:
+          RUSTC_WRAPPER: sccache
+        run: cargo clippy --no-default-features --features zlob -- -D warnings

.github/workflows/spelling.yaml

@@ -12,6 +12,10 @@ on:
 env:
   CLICOLOR: 1
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
 jobs:
   spelling:
     name: Spell Check with Typos

.github/workflows/stylua.yaml

@@ -20,6 +20,10 @@ on:
 env:
   CLICOLOR: 1
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
 jobs:
   stylua:
     name: Check lua files using Stylua

.gitignore

@@ -16,6 +16,7 @@ result
 *.data
 node_modules/
 crates/fff-notify-debouncer-full/
+packages/fff-bun/glob-bench-bin
 
 dist/
 scripts/benchmark-results/
@@ -24,6 +25,41 @@ scripts/benchmark-results/
 *.dylib
 *.so
 *.dll
+*.pdb
 
 # Instruments traces
 *.trace/
+
+# Test logs
+fff-test.log
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyd
+*.egg-info/
+*.egg
+.eggs/
+build/
+*.whl
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+# uv
+# Testing / linting
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~

.luacheckrc

@@ -1,4 +1,4 @@
--- luacheck configuration for fff.nvim
+-- luacheck configuration for fff
 -- https://luacheck.readthedocs.io/en/stable/config.html
 
 -- Neovim globals

.luarc.json

@@ -5,14 +5,14 @@
   },
   "workspace": {
     "library": [
-      "/opt/homebrew/share/nvim/runtime/lua/vim/_meta",
-      "/opt/homebrew/share/nvim/runtime/lua/vim/shared.lua",
+      "$VIMRUNTIME/lua",
       "${3rd}/luv/library",
       "${3rd}/busted/library"
     ],
     "checkThirdParty": false
   },
   "diagnostics": {
+    "globals": ["vim"],
     "severity": {
       "undefined-global": "Error",
       "undefined-field": "Warning",

AGENTS.md

@@ -31,6 +31,13 @@ When doing code make sure to REDUCE SIZE OF COMMENTS. This is very important. Ev
 - Do not make public structs if something can be private
 
 
+## Style guide
+
+- NO MODULES COMMENTS
+- NO TOP FILE COMMENTS
+- NO COMMENT LONGER THAN 2 LINES UNLESS ASKED EXPLICITLY
+- UTILITY FUNCTIONS GO INTO THE END OF FILE
+
 ## Architecture
 
 Everything that is performance critical happens in rust world, everything that is neovim specific happens in the lua code.

Cargo.lock

@@ -91,9 +91,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.102"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
 
 [[package]]
 name = "arrayref"
@@ -139,17 +139,11 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "bindet"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5afee99ef5f7527f3944f2bf4f5d443749fa47d43eb1d4f83a36e839be7900a3"
-
 [[package]]
 name = "bindgen"
-version = "0.70.1"
+version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
  "bitflags 2.11.0",
  "cexpr",
@@ -160,7 +154,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash 1.1.0",
+ "rustc-hash",
  "shlex",
  "syn",
 ]
@@ -446,9 +440,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -639,20 +633,17 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "fff-c"
-version = "0.7.1"
+version = "0.10.5"
 dependencies = [
  "fff-query-parser",
  "fff-search",
  "git2",
- "mimalloc",
- "serde",
  "serde_json",
- "tracing",
 ]
 
 [[package]]
 name = "fff-grep"
-version = "0.7.1"
+version = "0.10.5"
 dependencies = [
  "bstr",
  "memchr",
@@ -660,7 +651,7 @@ dependencies = [
 
 [[package]]
 name = "fff-mcp"
-version = "0.7.1"
+version = "0.10.5"
 dependencies = [
  "clap",
  "fff-query-parser",
@@ -671,61 +662,60 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
+ "tempfile",
  "tokio",
  "tracing",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "fff-notify-debouncer-full"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c6f0c16164d10c082af931377766f5495440bee66a9a841bbcea1af5de8878c"
+checksum = "29a4ebea7b8a2840cd59358bbf396f6f04313ce8eae84ac79703ce80298b8731"
 dependencies = [
  "file-id",
  "log",
  "notify",
  "notify-types",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "walkdir",
 ]
 
 [[package]]
 name = "fff-nvim"
-version = "0.7.1"
+version = "0.10.5"
 dependencies = [
  "ahash",
- "blake3",
  "chrono",
  "criterion",
  "ctrlc",
- "dirs",
- "fff-notify-debouncer-full",
  "fff-query-parser",
  "fff-search",
  "git2",
- "glidesort",
- "heed",
  "ignore",
  "mimalloc",
  "mlua",
- "neo_frizbee",
- "notify",
  "once_cell",
- "parking_lot",
- "pathdiff",
  "rand 0.8.5",
- "rayon",
- "serde",
- "smallvec",
- "smartstring",
- "tempfile",
- "thiserror 2.0.18",
  "tracing",
+ "tracing-subscriber",
+ "zlob",
+]
+
+[[package]]
+name = "fff-python"
+version = "0.10.5"
+dependencies = [
+ "fff-query-parser",
+ "fff-search",
+ "git2",
+ "pyo3",
 ]
 
 [[package]]
 name = "fff-query-parser"
-version = "0.7.1"
+version = "0.10.5"
 dependencies = [
  "criterion",
  "zlob",
@@ -733,13 +723,11 @@ dependencies = [
 
 [[package]]
 name = "fff-search"
-version = "0.7.1"
+version = "0.10.5"
 dependencies = [
  "ahash",
  "aho-corasick",
- "bindet",
  "blake3",
- "chrono",
  "criterion",
  "ctor",
  "dirs",
@@ -759,7 +747,6 @@ dependencies = [
  "mimalloc",
  "neo_frizbee",
  "notify",
- "once_cell",
  "parking_lot",
  "pathdiff",
  "proptest",
@@ -768,12 +755,10 @@ dependencies = [
  "regex",
  "regex-syntax",
  "serde",
- "serde_json",
+ "signal-hook-registry",
  "smallvec",
- "smartstring",
  "tempfile",
  "thiserror 2.0.18",
- "toml",
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
@@ -942,15 +927,14 @@ dependencies = [
 
 [[package]]
 name = "git2"
-version = "0.20.4"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
+checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e"
 dependencies = [
  "bitflags 2.11.0",
  "libc",
  "libgit2-sys",
  "log",
- "url",
 ]
 
 [[package]]
@@ -1220,6 +1204,15 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "inotify"
 version = "0.11.1"
@@ -1275,15 +1268,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.17"
@@ -1350,9 +1334,9 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
 
 [[package]]
 name = "libgit2-sys"
-version = "0.18.3+1.9.2"
+version = "0.18.7+1.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
+checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89"
 dependencies = [
  "cc",
  "libc",
@@ -1457,13 +1441,22 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
 
 [[package]]
 name = "memmap2"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
 dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mimalloc"
 version = "0.1.48"
@@ -1504,7 +1497,7 @@ dependencies = [
  "mlua_derive",
  "num-traits",
  "parking_lot",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "rustversion",
 ]
 
@@ -1533,13 +1526,9 @@ dependencies = [
 
 [[package]]
 name = "neo_frizbee"
-version = "0.10.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b216854d8c3752f8dbda56703c708de44bccb0542ca3924976c5977d6fb0724"
-dependencies = [
- "itertools 0.14.0",
- "raw-cpuid",
-]
+checksum = "7a2f6120a8da26bea3587731072111062c5d8c51ca3a3a75a716bd8b735d5882"
 
 [[package]]
 name = "nix"
@@ -1565,9 +1554,9 @@ dependencies = [
 
 [[package]]
 name = "notify"
-version = "9.0.0-rc.3"
+version = "9.0.0-rc.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "783683ce6e1059e3747190a6c05688db21e07a846bf90babb351365f9133fe3e"
+checksum = "b44b771d4dd781ef14c84078693e67495da6b47f609f72e8a4da8420a861240e"
 dependencies = [
  "bitflags 2.11.0",
  "inotify",
@@ -1807,6 +1796,12 @@ dependencies = [
  "plotters-backend",
 ]
 
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -1867,6 +1862,69 @@ dependencies = [
  "unarray",
 ]
 
+[[package]]
+name = "pyo3"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -1962,15 +2020,6 @@ dependencies = [
  "rand_core 0.9.5",
 ]
 
-[[package]]
-name = "raw-cpuid"
-version = "11.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
-dependencies = [
- "bitflags 2.11.0",
-]
-
 [[package]]
 name = "rayon"
 version = "1.11.0"
@@ -2062,9 +2111,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
 
 [[package]]
 name = "rmcp"
-version = "1.2.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba6b9d2f0efe2258b23767f1f9e0054cfbcac9c2d6f81a031214143096d7864f"
+checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e"
 dependencies = [
  "async-trait",
  "base64",
@@ -2084,9 +2133,9 @@ dependencies = [
 
 [[package]]
 name = "rmcp-macros"
-version = "1.2.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab9d95d7ed26ad8306352b0d5f05b593222b272790564589790d210aa15caa9e"
+checksum = "6aefac48c364756e97f04c0401ba3231e8607882c7c1d92da0437dc16307904d"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -2095,12 +2144,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.2"
@@ -2238,15 +2281,6 @@ dependencies = [
  "zmij",
 ]
 
-[[package]]
-name = "serde_spanned"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -2290,18 +2324,6 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
-[[package]]
-name = "smartstring"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
-dependencies = [
- "autocfg",
- "serde",
- "static_assertions",
- "version_check",
-]
-
 [[package]]
 name = "socket2"
 version = "0.6.3"
@@ -2318,12 +2340,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -2361,6 +2377,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
 [[package]]
 name = "tempfile"
 version = "3.27.0"
@@ -2515,47 +2537,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "toml"
-version = "0.8.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
-dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_write",
- "winnow",
-]
-
-[[package]]
-name = "toml_write"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
-
 [[package]]
 name = "tracing"
 version = "0.1.44"
@@ -2647,6 +2628,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
 [[package]]
 name = "url"
 version = "2.5.8"
@@ -3051,15 +3038,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
-[[package]]
-name = "winnow"
-version = "0.7.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "wit-bindgen"
 version = "0.51.0"
@@ -3259,9 +3237,9 @@ dependencies = [
 
 [[package]]
 name = "zlob"
-version = "1.3.3"
+version = "1.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f3522fa9701b74ec72758aedb96da278f6e0533bc16b6a79090bfb465d4661"
+checksum = "f0f010d9b10495cc0037cb47cedc143c39c7fa2d59f752dca19036fd0234a210"
 dependencies = [
  "bindgen",
  "bitflags 2.11.0",

Cargo.toml

@@ -4,25 +4,29 @@ members = [
   "crates/fff-core",
   "crates/fff-mcp",
   "crates/fff-nvim",
+  "crates/fff-python",
   "crates/fff-query-parser",
   "crates/fff-grep",
 ]
+
 resolver = "2"
 
+[workspace.lints.clippy]
+module_inception = "allow"
+
 [workspace.dependencies]
-fff-grep = { version = "0.7.1", path = "crates/fff-grep" }
-fff-query-parser = { version = "0.7.1", path = "crates/fff-query-parser", default-features = false }
+fff-grep = { version = "0.10.5", path = "crates/fff-grep" }
+fff-query-parser = { version = "0.10.5", path = "crates/fff-query-parser", default-features = false }
 
 # Shared dependencies
 ahash = "0.8"
-bindet = "0.3"
 blake3 = "1.8.2"
 chrono = { version = "0.4", features = ["serde"] }
 ctrlc = "3.4.2"
 dirs = "5.0"
 dunce = "1.0"
 # git2 - base config without TLS (each crate adds platform-specific TLS)
-git2 = { version = "0.20.2", default-features = false, features = [
+git2 = { version = "0.21.0", default-features = false, features = [
   "vendored-libgit2",
 ] }
 glidesort = "0.1"
@@ -31,12 +35,13 @@ heed = "0.22.0"
 ignore = "0.4.22"
 memmap2 = "0.9"
 mimalloc = "0.1.47"
-zlob = "1.3.3"
+signal-hook-registry = "1.4"
+zlob = { version = "=1.6.3" }
 
 mlua = { version = "0.11.1", features = ["module", "luajit"] }
-neo_frizbee = { version = "0.10.1", features = ["match_end_col"] }
+neo_frizbee = { version = "0.11.0", features = ["match_end_col"] }
 notify = { version = "9.0.0-rc.3" }
-notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.3" }
+notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.4" }
 once_cell = "1.20.2"
 parking_lot = "0.12"
 pathdiff = "0.2.1"
@@ -51,7 +56,7 @@ tracing = "0.1"
 opt-level = 3
 lto = "fat"
 codegen-units = 1
-strip = true
+strip = "debuginfo"
 
 [profile.ci]
 inherits = "release"

Formula/fff-mcp.rb

@@ -0,0 +1,60 @@
+# Originally authored by @jellydn (https://github.com/jellydn/homebrew-tap).
+# Maintained in-repo; auto-bumped by .github/workflows/release.yaml on stable releases.
+class FffMcp < Formula
+  desc "Fast file search toolkit for AI agents (MCP server)"
+  homepage "https://github.com/dmtrKovalenko/fff"
+  license "MIT"
+  version "0.10.5"
+
+  LIVECHECK_REPO = "dmtrKovalenko/fff".freeze
+  RELEASE_BASE = "https://github.com/dmtrKovalenko/fff/releases/download".freeze
+
+  on_macos do
+    on_arm do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-apple-darwin"
+      sha256 "2f1a75c2401e5df7f5a10827f9ce9fe8401f290ff99c3f3f1423c7d7b156d849"
+    end
+
+    on_intel do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-apple-darwin"
+      sha256 "c81e22a7fb657731a7441be877c622a110eb7e7dbc94e3b57d79c9cea91f38db"
+    end
+  end
+
+  on_linux do
+    on_arm do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-unknown-linux-gnu"
+      sha256 "48f5af0d9e6ef4506c5cccd3f4571da4f0acc9b9c0967832167e49fe651f8ee5"
+    end
+
+    on_intel do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-unknown-linux-gnu"
+      sha256 "0408cbac06303b3bf2041181089d8b4974b527883242bdcf8ba599b41c003973"
+    end
+  end
+
+  livecheck do
+    url "https://github.com/#{LIVECHECK_REPO}/releases/latest"
+    strategy :github_latest
+  end
+
+  def install
+    if OS.mac?
+      if Hardware::CPU.arm?
+        bin.install "fff-mcp-aarch64-apple-darwin" => "fff-mcp"
+      elsif Hardware::CPU.intel?
+        bin.install "fff-mcp-x86_64-apple-darwin" => "fff-mcp"
+      end
+    elsif OS.linux?
+      if Hardware::CPU.arm?
+        bin.install "fff-mcp-aarch64-unknown-linux-gnu" => "fff-mcp"
+      elsif Hardware::CPU.intel?
+        bin.install "fff-mcp-x86_64-unknown-linux-gnu" => "fff-mcp"
+      end
+    end
+  end
+
+  test do
+    system bin/"fff-mcp", "--healthcheck"
+  end
+end

Makefile

@@ -1,4 +1,5 @@
 PLENARY_DIR ?= ../plenary.nvim
+MINI_DIR ?= ../mini.nvim
 
 PREFIX ?= /usr/local
 LIBDIR ?= $(PREFIX)/lib
@@ -8,15 +9,49 @@ INCLUDEDIR ?= $(PREFIX)/include
 STRESS_RUSTFLAGS := --cfg stress
 FFF_STRESS_DEFAULT_SEED ?= 0xDEADBEEFCAFEBABE
 
-.PHONY: build build-c-lib install uninstall test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random
+SHELL := bash
+# Order matters: `-c` must be last so bash treats the recipe as the script
+# string rather than the literal `-o` / `pipefail` tokens.
+.SHELLFLAGS := -o pipefail -euc
+
+.PHONY: build build-c-lib install uninstall test test-rust test-rescan test-rescan-known-defects rescan-probe test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
 
 all: format test lint
 
+SYNC_API_SRC := packages/shared/fff-api.ts
+SYNC_API_TARGETS := packages/fff-node/src/fff-api.ts packages/fff-bun/src/fff-api.ts
+SYNC_API_BANNER := // ----------------------------------------------------------------------------\n// GENERATED FILE - DO NOT EDIT.\n// Copied from: ${SYNC_API_SRC}\n// Run make sync-js-api from the repo root to regenerate.\n// ----------------------------------------------------------------------------\n\n
+
+sync-js-api:
+	@for target in $(SYNC_API_TARGETS); do \
+		printf '$(SYNC_API_BANNER)' > "$$target"; \
+		cat $(SYNC_API_SRC) >> "$$target"; \
+		echo "synced: $$target"; \
+	done
+
+sync-js-api-check:
+	@status=0; \
+	for target in $(SYNC_API_TARGETS); do \
+		tmp=$$(mktemp); \
+		printf '$(SYNC_API_BANNER)' > "$$tmp"; \
+		cat $(SYNC_API_SRC) >> "$$tmp"; \
+		if ! cmp -s "$$tmp" "$$target"; then \
+			echo "out of date: $$target (run make sync-js-api)"; status=1; \
+		fi; \
+		rm -f "$$tmp"; \
+	done; \
+	exit $$status
+
 build:
-	cargo build --release --features zlob
+	cargo build --release --no-default-features --features zlob
+
+# Only the crates the e2e suites load (nvim lua tests + C/bun/node FFI tests),
+# skipping fff-python (pyo3) and fff-mcp (tokio/rmcp) which e2e never touches.
+build-e2e:
+	cargo build --release -p fff-nvim -p fff-c --no-default-features --features zlob
 
 build-c-lib:
-	cargo build --release -p fff-c --features zlob
+	cargo build --release -p fff-c --no-default-features --features zlob
 
 header:
 	cbindgen --config crates/fff-c/cbindgen.toml --crate fff-c --output crates/fff-c/include/fff.h
@@ -54,77 +89,192 @@ test-setup:
 		echo "Cloning plenary.nvim..."; \
 		git clone --depth 1 https://github.com/nvim-lua/plenary.nvim $(PLENARY_DIR); \
 	fi
+	@if [ ! -d "$(MINI_DIR)" ]; then \
+		echo "Cloning mini.nvim..."; \
+		git clone --depth 1 https://github.com/echasnovski/mini.nvim $(MINI_DIR); \
+	fi
 
 test-rust:
-	cargo test --workspace --features zlob --exclude fff-nvim
+	cargo test --workspace --no-default-features --features zlob --exclude fff-nvim --exclude fff-python
 
-test-lua: test-setup build
+# Watcher rescan harness: asserts that editing, build output, git activity and
+# preview reads all stay on the incremental path instead of re-walking the tree.
+test-rescan:
+	cargo test -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- rescan
+
+# Live probe for watcher rescan requests and their causes.
+# Usage: make rescan-probe DIR=~/some/repo [SECONDS=120]
+rescan-probe:
+	cargo run --release -p fff-nvim --bin rescan_probe \
+		--no-default-features --features zlob,rescan-stats -- \
+		$(or $(DIR),.) $(if $(SECONDS),--seconds $(SECONDS),)
+
+# The same harness, restricted to cases that currently fail on purpose. Each
+# `#[ignore]` reason names the code that causes the unnecessary rescan.
+test-rescan-known-defects:
+	cargo test --no-fail-fast -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- --ignored --nocapture
+
+CC ?= cc
+CFLAGS ?= -O0 -g -Wall -Wextra -std=c99
+TARGET_DIR ?= target/release
+SMOKE_BIN := $(TARGET_DIR)/fff_c_smoke
+SMOKE_SRC := crates/fff-c/tests/smoke.c
+SMOKE_INCLUDE := crates/fff-c/include
+
+test-c-smoke: build-e2e
+	$(CC) $(CFLAGS) -I $(SMOKE_INCLUDE) -L $(TARGET_DIR) \
+		-Wl,-rpath,@loader_path/../target/release \
+		-Wl,-rpath,$$(pwd)/$(TARGET_DIR) \
+		$(SMOKE_SRC) -lfff_c -o $(SMOKE_BIN)
+	$(SMOKE_BIN) .
+
+# Alias kept for the `external-tests.yml` workflow naming.
+test-c-api: test-c-smoke
+
+# neovim instance swallows internal crashes and doesn't rise the the error exiting silently
+# so check the stdout in case the sigsegv coming out of fff was printed (actual regression).
+# Output is streamed live via `tee`; pipefail (set above) propagates nvim's exit.
+test-lua: test-setup build-e2e
+	@logfile=$$(mktemp); \
+	trap 'rm -f "$$logfile"' EXIT; \
 	nvim --headless -u tests/minimal_init.lua \
-		-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1
+		-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1 \
+		| tee "$$logfile"; \
+	if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \
+		echo ""; \
+		echo "FAIL: native crash detected during lua tests"; \
+		exit 1; \
+	fi
+
+test-lua-snap: test-setup build-e2e
+	@logfile=$$(mktemp); \
+	trap 'rm -f "$$logfile"' EXIT; \
+	nvim --headless -u tests/minimal_init.lua \
+		-c "lua local ok,err=pcall(require('mini.test').run_file,'tests/picker_ui_snap.lua'); if not ok then io.stderr:write('mini.test failed to load: '..tostring(err)..'\\n'); vim.cmd('cquit 2') end" 2>&1 \
+		| tee "$$logfile"; \
+	if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \
+		echo ""; \
+		echo "FAIL: native crash detected during snapshot tests"; \
+		exit 1; \
+	fi
 
 test-version: test-setup
 	nvim --headless -u tests/minimal_init.lua \
 		-c "PlenaryBustedFile tests/version_spec.lua" 2>&1
 
-prepare-bun: build
+prepare-bun: build-e2e sync-js-api
 	mkdir -p packages/fff-bun/bin
-	cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null; \
-	cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null; \
-	cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null; \
-	true
+	cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null || true; \
+	cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null || true; \
+	cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null || true
 
-prepare-node: build
+prepare-node: build-e2e sync-js-api
 	mkdir -p packages/fff-node/bin
-	cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null; \
-	cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null; \
-	cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null; \
-	true
+	cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null || true; \
+	cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null || true; \
+	cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null || true
 
 test-bun: prepare-bun
-	cd packages/fff-bun && bun test src/
+	cd packages/fff-bun && bun test test/
 	cd packages/pi-fff && bun test test/
 
+# Same as prepare-bun but puts the compiled binary into the actual npm package location
+prepare-bun-packaged: prepare-bun
+	@machine=$$(uname -m); \
+	case "$$machine" in \
+	  x86_64|amd64) arch=x64 ;; \
+	  aarch64|arm64) arch=arm64 ;; \
+	  *) echo "unsupported arch: $$machine" >&2; exit 1 ;; \
+	esac; \
+	case "$$(uname -s)" in \
+	  Darwin) lib=libfff_c.dylib; pkg=fff-bin-darwin-$$arch ;; \
+	  Linux) lib=libfff_c.so; \
+	    if ldd --version 2>&1 | grep -qi musl; then libc=musl; else libc=gnu; fi; \
+	    pkg=fff-bin-linux-$$arch-$$libc ;; \
+	  MINGW*|MSYS*|CYGWIN*|Windows_NT) lib=fff_c.dll; pkg=fff-bin-win32-$$arch ;; \
+	  *) echo "unsupported OS: $$(uname -s)" >&2; exit 1 ;; \
+	esac; \
+	src=target/release/$$lib; \
+	[ -f "$$src" ] || { echo "missing built library: $$src" >&2; exit 1; }; \
+	dest=packages/fff-bun/node_modules/@ff-labs/$$pkg; \
+	rm -rf "$$dest"; mkdir -p "$$dest"; \
+	cp "$$src" "$$dest/$$lib"; \
+	printf '{ "name": "@ff-labs/%s", "version": "0.0.0", "main": "%s" }\n' "$$pkg" "$$lib" > "$$dest/package.json"
+
+# Compile a bun example to a standalone executable and run it. Verifies the
+# native libfff_c is embedded + loaded from a `bun build --compile` binary.
+# The staged bin package is removed before running so success proves the lib
+# was embedded, not resolved from disk.
+test-bun-compile: prepare-bun-packaged
+	cd packages/fff-bun && \
+		if [ "$$(uname -s)" = "Linux" ]; then \
+		  if ldd --version 2>&1 | grep -qi musl; then DEFINE='--define FFF_LIBC="musl"'; \
+		  else DEFINE='--define FFF_LIBC="gnu"'; fi; \
+		else DEFINE=""; fi; \
+		bun build --compile $$DEFINE ./examples/glob-bench.ts --outfile ./glob-bench-bin && \
+		EXE=./glob-bench-bin; [ -f "$$EXE.exe" ] && EXE="$$EXE.exe"; \
+		rm -rf bin node_modules/@ff-labs; \
+		"$$EXE" . '**/*.ts' 1 | tee /tmp/fff-compile-e2e.log && \
+		grep -q 'fff.glob' /tmp/fff-compile-e2e.log
+	rm -f packages/fff-bun/glob-bench-bin packages/fff-bun/glob-bench-bin.exe
+
 test-node: prepare-node
-	cd packages/fff-node && npm run build && node test/e2e.mjs
+	cd packages/fff-node && npm run build && node test/e2e.mjs && node test/watch.mjs
 
-test: test-rust test-lua test-version test-bun test-node
+test-js: test-bun test-node
 
+# Bug pinning stress test script over fff-node for issue #515
+# Just keep it untouched because it's good enough + some stress for SDK
+FFF_STRESS_ITERS ?= 50
+test-node-stress: prepare-node
+	cd packages/fff-node && npm run build && \
+		FFF_STRESS_ITERS=$(FFF_STRESS_ITERS) node test/stress-515.mjs
+
+test: test-rust test-lua test-lua-snap test-version test-bun test-node test-node-stress
 
 test-stress-seeded:
 	FFF_STRESS_SEED="$${FFF_STRESS_SEED:-$(FFF_STRESS_DEFAULT_SEED)}" \
 	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
-	cargo test \
+	cargo test --release \
 		-p fff-search \
 		--test fuzz_git_watcher_stress \
-		--features zlob \
+		--no-default-features --features zlob \
 		-- --nocapture stress_seeded
 
 test-stress-random:
 	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
-	cargo test \
+	cargo test --release \
 		-p fff-search \
 		--test fuzz_git_watcher_stress \
-		--features zlob \
+		--no-default-features --features zlob \
 		-- --nocapture stress_random
 
-test-stress: test-stress-seeded test-stress-random
+test-stress-regressions:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_regression stress_merge_conflict_convergence
+
+test-stress-repos:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_real_repos \
+		--no-default-features --features zlob \
+		-- --nocapture
+
+test-stress: test-stress-seeded test-stress-random test-stress-regressions test-stress-repos
 
 # Update version in a package.json, including optionalDependencies.
 # Usage: make set-npm-version PKG=packages/fff-bun VERSION=1.0.0-nightly.abc1234
 set-npm-version:
 	@test -n "$(PKG)" || (echo "PKG is required" && exit 1)
 	@test -n "$(VERSION)" || (echo "VERSION is required" && exit 1)
-	node -e " \
-		const fs = require('fs'); \
-		const pkg = JSON.parse(fs.readFileSync('$(PKG)/package.json', 'utf8')); \
-		pkg.version = '$(VERSION)'; \
-		if (pkg.optionalDependencies) { \
-			for (const dep of Object.keys(pkg.optionalDependencies)) { \
-				pkg.optionalDependencies[dep] = '$(VERSION)'; \
-			} \
-		} \
-		fs.writeFileSync('$(PKG)/package.json', JSON.stringify(pkg, null, 2) + '\n'); \
-	"
+	node scripts/set-npm-version.mjs "$(PKG)" "$(VERSION)"
 	@echo "Set $(PKG) to $(VERSION)"
 
 format-rust:
@@ -132,26 +282,80 @@ format-rust:
 format-lua:
 	stylua .
 format-ts:
-	bun format
+	cd packages && bun format
 
 format: format-rust format-lua format-ts
 
 lint-rust:
-	cargo clippy --workspace --features zlob -- -D warnings
+	cargo clippy --workspace --no-default-features --features zlob -- -D warnings
 lint-lua:
 	 ~/.luarocks/bin/luacheck .
 lint-ts:
-	bun lint
+	cd packages && bun lint
 
 lint: lint-rust lint-lua lint-ts
 
 check: format lint
 
+FFF_RELEASE_REPO ?= dmtrKovalenko/fff
+FFF_FORMULA_PATH ?= Formula/fff-mcp.rb
+FFF_INSTALL_SCRIPT_PATH ?= install-mcp.sh
+
+# Read the sha256 for $1 (filename, no .sha256 suffix). Reads from
+# BINARIES_DIR/$1.sha256 when set; otherwise curls the GitHub release.
+define fff_fetch_sha
+	if [ -n "$$BINARIES_DIR" ]; then \
+		awk '{print $$1}' "$$BINARIES_DIR/$$1.sha256" \
+			|| { echo "Missing checksum file: $$BINARIES_DIR/$$1.sha256" >&2; exit 1; }; \
+	else \
+		curl -fsSL "https://github.com/$(FFF_RELEASE_REPO)/releases/download/v$(VERSION)/$$1.sha256" \
+			| awk '{print $$1}'; \
+	fi
+endef
+
+bump-homebrew-formula:
+	@test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-homebrew-formula VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1)
+	@export BINARIES_DIR="$(BINARIES_DIR)"; \
+	fetch_sha() { $(fff_fetch_sha); }; \
+	sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \
+	sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \
+	sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-gnu)"; \
+	sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-gnu)"; \
+	sed -i.bak \
+		-e 's/^  version "[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*"$$/  version "$(VERSION)"/' \
+		-e '/fff-mcp-aarch64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_arm"'"/;}' \
+		-e '/fff-mcp-x86_64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_intel"'"/;}' \
+		-e '/fff-mcp-aarch64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_arm"'"/;}' \
+		-e '/fff-mcp-x86_64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_intel"'"/;}' \
+		"$(FFF_FORMULA_PATH)" && rm -f "$(FFF_FORMULA_PATH).bak"; \
+	echo "Bumped $(FFF_FORMULA_PATH) to v$(VERSION)"
+
+bump-install-mcp-sh:
+	@test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-install-mcp-sh VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1)
+	@export BINARIES_DIR="$(BINARIES_DIR)"; \
+	fetch_sha() { $(fff_fetch_sha); }; \
+	sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-musl)"; \
+	sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-musl)"; \
+	sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \
+	sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \
+	sha_win_intel="$$(fetch_sha fff-mcp-x86_64-pc-windows-msvc.exe)"; \
+	sha_win_arm="$$(fetch_sha fff-mcp-aarch64-pc-windows-msvc.exe)"; \
+	sed -i.bak \
+		-e 's|^PINNED_RELEASE_TAG=".*"|PINNED_RELEASE_TAG="v$(VERSION)"|' \
+		-e 's|^SHA256_X86_64_UNKNOWN_LINUX_MUSL=".*"|SHA256_X86_64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_intel"'"|' \
+		-e 's|^SHA256_AARCH64_UNKNOWN_LINUX_MUSL=".*"|SHA256_AARCH64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_arm"'"|' \
+		-e 's|^SHA256_X86_64_APPLE_DARWIN=".*"|SHA256_X86_64_APPLE_DARWIN="'"$$sha_darwin_intel"'"|' \
+		-e 's|^SHA256_AARCH64_APPLE_DARWIN=".*"|SHA256_AARCH64_APPLE_DARWIN="'"$$sha_darwin_arm"'"|' \
+		-e 's|^SHA256_X86_64_PC_WINDOWS_MSVC=".*"|SHA256_X86_64_PC_WINDOWS_MSVC="'"$$sha_win_intel"'"|' \
+		-e 's|^SHA256_AARCH64_PC_WINDOWS_MSVC=".*"|SHA256_AARCH64_PC_WINDOWS_MSVC="'"$$sha_win_arm"'"|' \
+		"$(FFF_INSTALL_SCRIPT_PATH)" && rm -f "$(FFF_INSTALL_SCRIPT_PATH).bak"; \
+	echo "Bumped $(FFF_INSTALL_SCRIPT_PATH) tag + checksums to v$(VERSION)"
+
 CRATES_TO_PUBLISH= fff-grep fff-query-parser fff-search
 
 publish-crates:
 	@test -n "$(V)" || (echo "V is required. Usage: make publish-crates V=0.2.0" && exit 1)
-	cargo install cargo-edit
+	cargo install cargo-edit --force --locked
 	cargo set-version $(V) || exit 1;
 	@for crate in $(CRATES_TO_PUBLISH); do \
 		cargo publish -p $$crate --allow-dirty $$(if [ -n "$$CI" ]; then echo "--no-verify"; fi) || exit 1; \

README.md

@@ -1,12 +1,17 @@
-<img alt="FFF" src="./assets/logo-orange.png" width="300">
+<a href="./assets/logo-orange.png"><img alt="FFF" src="./assets/logo-orange.png" width="300"></a>
 
 <p>
   <i>A file search toolkit for humans and AI agents. Really fast.</i>
 </p>
 
-Typo-resistant path and content search, frecency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once.
+Typo-resistant path and content search, frequency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once.
+
+Powers file search in [opencode](http://github.com/anomalyco/opencode/), [nushell](https://github.com/nushell/nushell), and many more amazing projects!
 
 Originally started as [Neovim plugin](#neovim-plugin) people loved, but it turned out that plenty of AI harnesses and code editors need the same thing: accurate, fast file search as a library. That is what fff is.
+<p>
+<a href="https://trendshift.io/repositories/26711?utm_source=repository-badge&amp;utm_medium=badge&amp;utm_campaign=badge-repository-26711" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26711" alt="dmtrKovalenko%2Ffff | Trendshift" width="250" height="55"/></a>
+</p>
 
 ---
 
@@ -32,12 +37,46 @@ curl -L https://dmtrkovalenko.dev/install-fff-mcp.sh | bash
 Windows (PowerShell):
 
 ```powershell
-irm https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.ps1 | iex
+irm https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.ps1 | iex
 ```
 
-The scripts live at [`install-mcp.sh`](./install-mcp.sh) and [`install-mcp.ps1`](./install-mcp.ps1) if you want to read them first.
+The scripts live at [`install-mcp.sh`](./install-mcp.sh) and [`install-mcp.ps1`](./install-mcp.ps1) if you want to read them first. They print the exact wiring instructions for your client.
 
-It prints the exact wiring instructions for your client. Once the server is connected, ask the agent to "use fff" and it picks up the `ffgrep`, `fffind`, and `fff-multi-grep` tools.
+### Homebrew (macOS / Linux)
+
+```bash
+brew install dmtrKovalenko/fff/fff-mcp
+brew upgrade fff-mcp   # after new stable releases
+```
+
+Formula lives in [`Formula/fff-mcp.rb`](./Formula/fff-mcp.rb) in this repo and is **auto-bumped on every stable release** (see `bump-homebrew-formula` in [`.github/workflows/release.yaml`](./.github/workflows/release.yaml)). Installs the prebuilt `fff-mcp` binary from [GitHub releases](https://github.com/dmtrKovalenko/fff/releases).
+
+### Codex setup
+
+Register the installed binary using its absolute path, since Codex desktop sessions may not inherit your interactive shell's `PATH`.
+
+Homebrew:
+
+```bash
+codex mcp add fff -- "$(brew --prefix)/bin/fff-mcp"
+```
+
+One-line installer:
+
+```bash
+codex mcp add fff -- "$HOME/.local/bin/fff-mcp"
+```
+
+This creates an entry in `~/.codex/config.toml` similar to:
+
+```toml
+[mcp_servers.fff]
+command = "/opt/homebrew/bin/fff-mcp"
+```
+
+Use the actual installed path for your system, then restart Codex or start a new task so it loads the server.
+
+Once the server is connected, ask the agent to "use fff" and it picks up the `ffgrep`, `fffind`, and `fff-multi-grep` tools.
 
 ### Recommended agent prompt
 
@@ -81,7 +120,7 @@ Three operating modes, switchable at runtime with `/fff-mode`:
 | `tools-only`             | Only tool injection. Keeps pi's native editor autocomplete.                       |
 | `override`               | Replaces pi's built-in `grep`, `find`, and `multi_grep` with FFF implementations. |
 
-Env vars: `PI_FFF_MODE`, `FFF_FRECENCY_DB`, `FFF_HISTORY_DB`. Flags: `--fff-mode`, `--fff-frecency-db`, `--fff-history-db`.
+Env vars: `PI_FFF_MODE`, `FFF_FRECENCY_DB`, `FFF_HISTORY_DB`. Flags: `--fff-mode`, `--fff-frecency-db`, `--fff-history-db`. The databases default to your existing fff.nvim ones when present, otherwise `~/.pi/agent/fff/`.
 
 ### Agent-facing tools
 
@@ -102,7 +141,7 @@ The Pi extension swaps pi's native tools for FFF implementations and feeds the i
 
 <details id="neovim-plugin">
 <summary>
-<h2>Neovim plugin</h2>
+<h2>fff.nvim</h2>
 </summary>
 
 Demo on the Linux kernel repo (100k files, 8GB):
@@ -114,8 +153,9 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
 #### lazy.nvim
 
 ```lua
+-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:Lazy clean`
 {
-  'dmtrKovalenko/fff.nvim',
+  'dmtrKovalenko/fff',
   build = function()
     -- downloads a prebuilt binary or falls back to cargo build
     require("fff.download").download_or_build_binary()
@@ -136,9 +176,10 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
       function() require('fff').live_grep({ grep = { modes = { 'fuzzy', 'plain' } } }) end,
       desc = 'Live fffuzy grep',
     },
-    { "fc",
-      function() require('fff').live_grep({ query = vim.fn.expand("<cword>") }) end,
-      desc = 'Search current word',
+    { "fw",
+      function() require('fff').live_grep_under_cursor() end,
+      mode = { 'n', 'x' },
+      desc = 'Search current word / selection',
     },
   },
 }
@@ -147,13 +188,14 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
 #### vim.pack
 
 ```lua
-vim.pack.add({ 'https://github.com/dmtrKovalenko/fff.nvim' })
+-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:packdel fff.nvim`
+vim.pack.add({ 'https://github.com/dmtrKovalenko/fff' })
 
 vim.api.nvim_create_autocmd('PackChanged', {
   callback = function(ev)
     local name, kind = ev.data.spec.name, ev.data.kind
-    if name == 'fff.nvim' and (kind == 'install' or kind == 'update') then
-      if not ev.data.active then vim.cmd.packadd('fff.nvim') end
+    if name == 'fff' and (kind == 'install' or kind == 'update') then
+      if not ev.data.active then vim.cmd.packadd('fff') end
       require('fff.download').download_or_build_binary()
     end
   end,
@@ -172,12 +214,69 @@ vim.keymap.set('n', 'ff', function() require('fff').find_files() end, { desc = '
 ```lua
 require('fff').find_files()                        -- find files in current repo
 require('fff').live_grep()                         -- live content grep
+require('fff').live_grep_under_cursor()            -- grep <cword> in normal, selection in visual
 require('fff').scan_files()                        -- force rescan
 require('fff').refresh_git_status()                -- refresh git status
 require('fff').find_files_in_dir(path)             -- find in a specific dir
 require('fff').change_indexing_directory(new_path) -- change root
+
+-- Programmatic search (no UI). Useful for plugin integrations.
+require('fff').file_search(query, opts)            -- fuzzy search files / dirs / mixed
+require('fff').content_search(query, opts)         -- programmatic grep
 ```
 
+#### `file_search(query, opts)`
+
+Returns a structured result `{ items, scores, total_matched, total_files?, total_dirs?, location? }`. Each item has a `type` field (`"file"` or `"directory"`) and `name` / `relative_path`. File items also expose `size`, `modified`, `git_status`, `is_binary`, and frecency scores.
+
+```lua
+local r = require('fff').file_search('button', {
+  mode             = 'mixed',  -- 'files' (default) | 'directories' | 'mixed'
+  max_results      = 50,
+  page             = 0,        -- 0-based pagination
+  current_file     = nil,      -- path to deprioritize for distance scoring
+  max_threads      = 4,
+  cwd              = nil,      -- switch indexed root if different (see below)
+  wait_for_index_ms = nil,     -- override the default scan wait timeout
+})
+for _, item in ipairs(r.items) do
+  print(item.type, item.relative_path)
+end
+```
+
+#### `content_search(query, opts)`
+
+Returns a `GrepResult` `{ items, total_matched, total_files_searched, total_files, filtered_file_count, next_file_offset, regex_fallback_error? }`. Each match item has `relative_path`, `name`, `line_number`, `col`, `line_content`, `match_ranges`, plus the same file metadata as `file_search`.
+
+```lua
+local r = require('fff').content_search('TODO', {
+  mode                  = 'plain',  -- 'plain' (default) | 'regex' | 'fuzzy'
+  max_file_size         = 10 * 1024 * 1024,
+  max_matches_per_file  = 100,
+  smart_case            = true,
+  page_size             = 50,
+  file_offset           = 0,
+  time_budget_ms        = 0,
+  trim_whitespace       = false,
+  cwd                   = nil,      -- switch indexed root if different
+  wait_for_index_ms     = nil,      -- override the default scan wait timeout
+})
+for _, m in ipairs(r.items) do
+  print(string.format('%s:%d %s', m.relative_path, m.line_number, m.line_content))
+end
+```
+
+Both functions accept the same constraint syntax as the UI pickers (e.g. `git:modified`, `*.rs`, `!test/`, glob patterns).
+
+#### `cwd` and indexing
+
+Both `file_search` and `content_search` honour an optional `cwd` field. The first call to either function lazily initialises the picker at `config.base_path` (your Neovim cwd by default).
+
+- If `cwd` matches the currently indexed root, the call returns immediately against the existing index.
+- If `cwd` differs, the picker is re-indexed at the new root and the call **blocks** (default up to 10 s) until the new picker is installed and its initial scan completes — so callers always get results from the right tree.
+- If the index is still warming up after a `change_indexing_directory`, you can pass `wait_for_index_ms = N` to block for up to `N` ms regardless of whether `cwd` triggered the swap. Pass `0` to skip waiting entirely (useful for fire-and-forget calls where partial results are acceptable).
+- Invalid or non-existent `cwd` paths return an empty result and emit an error via `vim.notify`.
+
 ### Commands
 
 - `:FFFScan`. Rescan files.
@@ -200,15 +299,30 @@ require('fff').setup({
   max_threads = 4,
   lazy_sync = true,
   prompt_vim_mode = false,
+  follow_symlinks = false,
+  -- Allow indexing the user's $HOME directory. Enabled by default.
+  -- Disable if you strictly sure you don't want this, as it makes whole fff error hard
+  enable_home_dir_scanning = true,
+  -- Allow indexing a filesystem root (e.g. `/`, `C:\`). Disabled by default
+  enable_fs_root_scanning = false,
   layout = {
     height = 0.8,
     width = 0.8,
     prompt_position = 'bottom',   -- or 'top'
     preview_position = 'right',   -- 'left' | 'right' | 'top' | 'bottom'
     preview_size = 0.5,
+    -- Border style for the picker windows. Leave unset (nil) to follow the
+    -- global `vim.o.winborder`; set it to override fff's borders independently.
+    border = nil, -- 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | 'none'
+    -- border = {
+    --   { ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' },
+    --   { ' ', ' ', ' ', ' ', ' ' },
+    -- },
+
     flex = { size = 130, wrap = 'top' },
+    min_list_height = 10, --  do not display anything except the list below this threshold
     show_scrollbar = true,
-    path_shorten_strategy = 'middle_number', -- 'middle_number' | 'middle' | 'end'
+    path_shorten_strategy = 'middle_number', -- 'middle_number' | 'middle' | 'end' | 'start'
     anchor = 'center',
   },
   preview = {
@@ -238,6 +352,10 @@ require('fff').setup({
     preview_scroll_down = '<C-d>',
     toggle_debug = '<F2>',
     cycle_grep_modes = '<S-Tab>',
+    insert_newline_escape = '<C-CR>',
+    -- grep mode only: jump cursor to first match of next/prev file group
+    grep_jump_to_next_file = { '<C-A-n>', '<A-Down>' },
+    grep_jump_to_prev_file = { '<C-A-p>', '<A-Up>' },
     cycle_previous_query = '<C-Up>',
     toggle_select = '<Tab>',
     send_to_quickfix = '<C-q>',
@@ -257,6 +375,13 @@ require('fff').setup({
   git = {
     status_text_color = false, -- true to color filenames by git status
   },
+  file_picker = {
+    fuzzy_query_highlighting = false, -- true to highlight fuzzy query matches in file picker results
+  },
+  select = {
+    -- Return winid to open the chosen file in, or nil to open in the original window
+    select_window = function(current_buf, action) --[[ default impl ]] end,
+  },
   grep = {
     max_file_size = 10 * 1024 * 1024,
     max_matches_per_file = 100,
@@ -264,12 +389,31 @@ require('fff').setup({
     time_budget_ms = 150,
     modes = { 'plain', 'regex', 'fuzzy' },
     trim_whitespace = false,
+    enable_filename_constraint = false, -- treat filename-like tokens (e.g. `score.rs`) in a grep query as a file-path filter scoping the search; off = searched as literal text
+    location_format = ':%d:%d', -- printf format for line:col prefix in grep results, e.g. ':%d' for line-only
+  },
+  debug = {
+    enabled = false, -- show the file info panel next to the preview
+    show_scores = false, -- inline scores in the file list
+    -- Per-section toggles for the file info panel. Accepts a boolean shorthand
+    -- (`show_file_info = true|false`) to flip everything at once. The panel
+    -- adapts to width: narrow renders sections vertically, wide renders them
+    -- as a two-column grid. Disable a section to also shrink the panel.
+    show_file_info = {
+      file_info = true, -- size, type, git status, frecency
+      score_breakdown = true, -- total + match type, bonuses, modifiers, penalty
+      -- modified + accessed timestamps; pass a table to hide individual rows:
+      --   timings = { modified = false, accessed = true }
+      timings = true,
+      full_path = true, -- relative path at the bottom (wraps if too long)
+    },
   },
-  debug = { enabled = false, show_scores = false },
   logging = {
-    enabled = true,
+    -- logs will be written in a parent directory of this file path in files like
+    -- `<stem>+<UTC-timestamp>+<pid>.<ext>`. Run :FFFOpenLog to open current one
     log_file = vim.fn.stdpath('log') .. '/fff.log',
     log_level = 'info',
+    retain_runs = 20,
   },
 })
 ```
@@ -291,7 +435,7 @@ Both find and grep accept these tokens to refine a query:
 
 - `git:modified`. One of `modified`, `staged`, `deleted`, `renamed`, `untracked`, `ignored`.
 - `test/`. Any deeply nested children of `test/`.
-- `!something`, `!test/`, `!git:modified`. Exclusion.
+- `!something`, `!test/`, `!git:modified`. Exclusion. Text exclusions need at least 3 alphanumeric-containing characters, so operators like `!=` or `!==` work.
 - `./**/*.{rs,lua}`. Any valid glob, powered by [zlob](https://github.com/dmtrKovalenko/zlob).
 
 Grep-only:
@@ -301,6 +445,20 @@ Grep-only:
 
 Mix freely: `git:modified src/**/*.rs !src/**/mod.rs user controller`.
 
+### Open in invoking window
+
+By default fff.nvim will try to open a file in the most suitable window, so any non-file buffers are not affected. You can customize or disable this by providing:
+
+```lua
+require('fff').setup({
+  select = {
+    select_window = function(_current_buf, _action) return nil end,
+  },
+})
+```
+
+Caveat: the chosen file replaces the buffer in the invoking window even if it's a non-modifiable / special buftype. `winfixbuf` windows still fall back to `:split` to avoid `E1513`.
+
 ### Multi-select and quickfix
 
 - `<Tab>`. Toggle selection (shows a thick `▊` in the signcolumn).
@@ -310,6 +468,51 @@ Mix freely: `git:modified src/**/*.rs !src/**/mod.rs user controller`.
 
 Sign-column indicators are on by default. To color filename text by git status, set `git.status_text_color = true` and adjust the `hl.git_*` groups. See `:help fff.nvim` for the full list.
 
+### Float colors
+
+The picker maps its float content to `NormalFloat` (via `hl.normal`) and the border to `FloatBorder`. Default `FloatBorder` links to `NormalFloat`, so border and content share a background out of the box and the picker reads as a single popup. Override `hl.normal = 'Normal'` to make the picker blend with the editor instead.
+
+For finer control, set `hl.winhl` to override the per-window `winhighlight`. It accepts either a single string applied to every picker window, or a table with optional `prompt`, `list`, `preview`, and `file_info` keys. Missing keys fall back to the default built from `hl.normal`, `hl.border`, and `hl.title`.
+
+```lua
+-- Apply the same winhighlight to all picker windows
+hl = { winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title' }
+
+-- Or override specific windows only
+hl = {
+  winhl = {
+    prompt  = 'Normal:Pmenu,FloatBorder:FloatBorder',
+    list    = 'Normal:NormalFloat,FloatBorder:FloatBorder',
+    preview = 'Normal:NormalFloat,FloatBorder:FloatBorder',
+  },
+}
+```
+
+### File info panel
+
+Enable with `debug.enabled = true`. The panel sits above the preview and shows
+file metadata, score breakdown, timestamps and the full absolute path. It
+adapts to the panel width: at narrow widths sections stack vertically (B2),
+at wide widths sections render as a two-column grid (H2). Each section can be
+disabled individually via `debug.show_file_info`.
+
+Customise the panel via `hl`:
+
+| key                     | default           | used for                           |
+| ----------------------- | ----------------- | ---------------------------------- |
+| `file_info_section`     | `Title`           | section header label               |
+| `file_info_separator`   | `FloatBorder`     | dashes that act as section borders |
+| `file_info_label`       | `Comment`         | row labels (Size, Type, Git, ...)  |
+| `file_info_value`       | `Normal` fg       | plain values                       |
+| `file_info_value_dim`   | `NonText`         | dim values, separators inside rows |
+| `file_info_size`        | `Number`          | file size value                    |
+| `file_info_type`        | `Type`            | filetype value                     |
+| `file_info_path`        | `Directory`       | full path                          |
+| `file_info_total_score` | bold + `Number`   | total score (bold)                 |
+| `file_info_match_type`  | bold + `Special`  | match type (bold)                  |
+| `file_info_score_pos`   | `DiagnosticOk`    | positive score components          |
+| `file_info_score_neg`   | `DiagnosticError` | negative score components          |
+
 ### File filtering
 
 FFF honours `.gitignore`. For picker-only ignores that do not touch git, add a sibling `.ignore` file:
@@ -324,7 +527,9 @@ Run `:FFFScan` to force a rescan.
 ### Troubleshooting
 
 - `:FFFHealth` verifies picker init, optional dependencies, and DB connectivity.
-- `:FFFOpenLog` opens the log file.
+- `:FFFOpenLog` opens the current session's log file.
+- Historical log files are stored near the main log file `<state>/log/fff+<UTC-timestamp>+<pid>.log` (up to 20 files)
+- For a crash backtrace, run `lldb -- nvim` or `gdb -- nvim` and reproduce
 
 </details>
 
@@ -357,6 +562,9 @@ const hits = finder.value.grep("GetOffTheRecordProfile", {
   classifyDefinitions: true,
 });
 
+// Run extremely fast glob matching which is significantly (10-100 times) faster than Bun's and Node implementation
+const rustFiles = finder.value.glob("**/*.rs", { pageSize: 100 });
+
 finder.value.destroy();
 ```
 
@@ -401,9 +609,14 @@ make build-c-lib
 cargo build --release -p fff-c --features zlob
 ```
 
+> The `zlob` feature (requires the [Zig](https://ziglang.org) toolchain) switches both
+> glob matching **and** filesystem traversal to [zlob](https://github.com/dmtrKovalenko/zlob)'s
+> native parallel walker. Without it, the default build uses the pure-Rust
+> [`ignore`](https://crates.io/crates/ignore) (ripgrep) walker and `globset`.
+
 The output is a `cdylib` (`libfff_c.so` / `libfff_c.dylib` / `fff_c.dll`). The header lives at [`crates/fff-c/include/fff.h`](./crates/fff-c/include/fff.h).
 
-Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff.nvim/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages.
+Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages.
 
 ### Install
 
@@ -462,6 +675,35 @@ int main(void) {
 }
 ```
 
+### Versioned options struct (preferred)
+
+For instance creation use [`FffCreateOptions`](./crates/fff-c/include/fff.h) — a
+versioned struct that evolves without ABI breaks. C99 designated
+initializers keep call sites readable and zero-init unspecified fields:
+
+```c
+FffResult *res = fff_create_instance_with(&(FffCreateOptions){
+    .version = FFF_CREATE_OPTIONS_VERSION,
+    .base_path = "/path/to/repo",
+    .ai_mode = true,
+    .watch = true,
+    .enable_fs_root_scanning = false,   // off by default
+    .enable_home_dir_scanning = false,  // off by default
+});
+```
+
+### Glob-only search
+
+`fff_glob` filters indexed files by a single glob pattern, ranks by frecency,
+paginates — bypasses the regular query parser entirely. Use this when you
+already have a literal glob (`*.rs`, `**/*.test.ts`, `src/**`) and don't want
+fuzzy matching layered on top.
+
+```c
+FffResult *res = fff_glob(handle, "**/*.rs", "", 0, 0, 100);
+// FffSearchResult in res->handle, free with fff_free_search_result.
+```
+
 ### Notes
 
 - Every function returning `FffResult*` allocates with Rust's `Box`. Free with `fff_free_result`, do not use malloc's free
@@ -474,6 +716,75 @@ Source: [`crates/fff-c/`](./crates/fff-c/).
 
 Stable C ABI. Bind from C/C++, Zig, Go via cgo, Python via ctypes, or anything with C FFI.
 
+<details id="python-bindings">
+<summary>
+<h2>Python bindings</h2>
+</summary>
+
+### Install
+
+```bash
+pip install fff-search
+```
+
+Or build and install from source:
+
+```bash
+cd packages/fff-python
+uv sync --all-extras
+uv run maturin develop --release
+```
+
+### Basic usage
+
+```python
+from fff import FileFinder
+
+with FileFinder("/path/to/project", watch=False) as finder:
+    finder.wait_for_scan_blocking(timeout_ms=5000)
+
+    result = finder.search("main")
+    for item, score in zip(result.items, result.scores):
+        print(f"{item.relative_path}: {score.total}")
+
+    hits = finder.grep("class Profile", mode="plain", before_context=1, after_context=1)
+```
+
+### Async usage
+
+`wait_for_scan` is a coroutine that polls scan status and yields to the event
+loop, so it never blocks other tasks. Use `wait_for_scan_blocking` from
+synchronous code.
+
+```python
+import asyncio
+from fff import FileFinder
+
+async def main():
+    with FileFinder("/path/to/project", watch=False) as finder:
+        await finder.wait_for_scan(timeout_ms=5000)
+        result = finder.search("main")
+        print(result)
+
+asyncio.run(main())
+```
+
+### What you get
+
+- `search`, `glob`, `directory_search`, `mixed_search` — frecency-ranked fuzzy file/dir search
+- `grep` / `multi_grep` — plain, regex, or fuzzy content search with context lines and cursor pagination
+- `track_query` / `get_historical_query` — optional frecency and query-history databases
+- `reindex`, `refresh_git_status`, `scan_progress`, `health_check` — lifecycle and diagnostics
+
+Typed result objects (`FileItem`, `Score`, `GrepMatch`, …) with `py.typed`
+stubs included. Ships as an `abi3` wheel compatible with Python 3.10+.
+
+Source: [`packages/fff-python/`](./packages/fff-python/).
+
+</details>
+
+Native Python bindings built with PyO3. Use them for notebooks, agent scripts, or any Python tool that needs fast file search.
+
 ---
 
 ## What is FFF and why use it over ripgrep or fzf?
@@ -482,7 +793,7 @@ FFF is a file search library, not a CLI. Ripgrep and fzf are great tools, but th
 
 FFF keeps the index and the file cache resident in one long-lived process and exposes the same Rust core through four thin layers: a native crate (`fff-search`), a C library (`libfff_c`), a Node/Bun SDK (`@ff-labs/fff-node`), and an MCP server. You call `FileFinder.create()` once, then every subsequent search hits warm memory. On a 500k-file Chromium checkout, that is the difference between 3-9 **SECONDS** per ripgrep spawn and sub-10 ms per FFF query.
 
-Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema".
+Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm. It is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "\*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema".
 
 ### Why a programmatic API matters
 
@@ -495,7 +806,7 @@ Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it
 ### What the core actually does
 
 - **Frecency-ranked fuzzy matching.** Every indexed file carries an access score and a modification score. Searches rank files you have opened recently and frequently above cold results. This is the same idea as VS Code's recently-opened list, but applied to every search result, not just a sidebar.
-- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghm/frizbee)-derived core) that survives dropped characters and reorderings.
+- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghen/frizbee)-derived core) that survives dropped characters and reorderings.
 - **Content grep with three modes.** Plain literal (SIMD memmem), regex (the Rust `regex` crate), and fuzzy (Smith-Waterman per line). Auto-detects which mode to use from the pattern, falls back to fuzzy when a plain search returns zero hits.
 - **Multi-pattern OR search.** SIMD Aho-Corasick for "find any of these 20 identifiers at once", which is faster than regex alternation and a lot faster than 20 separate ripgrep runs.
 - **Background file watcher.** The index updates as files change. You never pay for a rescan on the hot path.
@@ -516,7 +827,6 @@ Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it
 
 Yes, fff fundamentally requires more memory than calling a single child process. That is the primary source of the speedup. In practice, alongside one of the most popular file search pickers for Neovim, [fff ends up using less RAM than a burst of ripgrep invocations](https://x.com/neogoose_btw/status/2041606853155811442).
 
-
 FFF also keeps a content index, around 360 bytes per indexed file, so roughly 36 MB for a 100k-file repo. Not every file is indexed - binaries, oversized files, and anything not eligible for grep are skipped. If even that footprint is too much, the index can be backed by a memory-mapped file instead of anonymous RAM.
 
 ### What this means in practice
@@ -541,7 +851,7 @@ If you are running one grep from a terminal, `rg` is still the right tool. If yo
 - `crates/fff-nvim` - Lua/mlua bindings for the Neovim plugin.
 - `crates/fff-mcp` - MCP server binary.
 - `packages/fff-node` - Node.js SDK (`@ff-labs/fff-node`).
-- `packages/fff-bun` - Bun SDK (`@ff-labs/fff-node`).
+- `packages/fff-bun` - Bun SDK (`@ff-labs/fff-bun`).
 - `packages/pi-fff` - pi extension (`@ff-labs/pi-fff`).
 - `lua/` - Neovim-side plugin code.
 
@@ -552,3 +862,15 @@ Bug reports and pull requests welcome. Agentic coding tools are welcome to be us
 ## License
 
 [MIT](./LICENSE) & open source forever.
+
+## FAQ
+
+### What does FFF stand for?
+
+There is intentionally no single canonical definition. Pick your favourite:
+
+- **F**ast **F**ile **F**inder
+- **F**uzzy **F**ile **F**inder
+- will search **F**iles **F**or **F**ood
+
+The brand hex is `#F87216`, not `#FFF`. Logo variants: [orange](./assets/logo-orange.png) · [dark](./assets/logo-dark.png) · [light](./assets/logo-light.png).

biome.json

@@ -1,35 +0,0 @@
-{
-  "$schema": "https://biomejs.dev/schemas/2.4.7/schema.json",
-  "files": {
-    "includes": ["packages/**/*.ts", "!packages/*/dist"],
-    "ignoreUnknown": true
-  },
-  "formatter": {
-    "enabled": true,
-    "indentStyle": "space",
-    "indentWidth": 2,
-    "lineWidth": 90
-  },
-  "javascript": {
-    "formatter": {
-      "quoteStyle": "double",
-      "trailingCommas": "all",
-      "semicolons": "always"
-    }
-  },
-  "linter": {
-    "enabled": true,
-    "rules": {
-      "recommended": true,
-      "style": {
-        "noNonNullAssertion": "off"
-      },
-      "suspicious": {
-        "noExplicitAny": "off"
-      },
-      "complexity": {
-        "noForEach": "off"
-      }
-    }
-  }
-}

bun.lock

@@ -1,148 +0,0 @@
-{
-  "lockfileVersion": 1,
-  "configVersion": 1,
-  "workspaces": {
-    "": {
-      "devDependencies": {
-        "@biomejs/biome": "^2.4.4",
-      },
-    },
-    "packages/fff-bun": {
-      "name": "@ff-labs/fff-bun",
-      "version": "0.1.37",
-      "bin": {
-        "fff": "./scripts/cli.ts",
-        "fff-demo": "./examples/search.ts",
-        "fff-grep": "./examples/grep.ts",
-      },
-      "devDependencies": {
-        "@types/bun": "^1.3.8",
-        "typescript": "^5.0.0",
-      },
-      "optionalDependencies": {
-        "@ff-labs/fff-bun-darwin-arm64": "0.0.0",
-        "@ff-labs/fff-bun-darwin-x64": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-musl": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-musl": "0.0.0",
-        "@ff-labs/fff-bun-win32-arm64": "0.0.0",
-        "@ff-labs/fff-bun-win32-x64": "0.0.0",
-      },
-      "peerDependencies": {
-        "bun": ">=1.0.0",
-      },
-    },
-    "packages/fff-node": {
-      "name": "@ff-labs/fff-node",
-      "version": "0.1.37",
-      "bin": {
-        "fff-node": "./dist/scripts/cli.js",
-      },
-      "dependencies": {
-        "ffi-rs": "^1.0.0",
-      },
-      "devDependencies": {
-        "@types/node": "^22.0.0",
-        "typescript": "^5.0.0",
-      },
-      "optionalDependencies": {
-        "@ff-labs/fff-bun-darwin-arm64": "0.0.0",
-        "@ff-labs/fff-bun-darwin-x64": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-musl": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-musl": "0.0.0",
-        "@ff-labs/fff-bun-win32-arm64": "0.0.0",
-        "@ff-labs/fff-bun-win32-x64": "0.0.0",
-      },
-    },
-  },
-  "packages": {
-    "@biomejs/biome": ["@biomejs/biome@2.4.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.4", "@biomejs/cli-darwin-x64": "2.4.4", "@biomejs/cli-linux-arm64": "2.4.4", "@biomejs/cli-linux-arm64-musl": "2.4.4", "@biomejs/cli-linux-x64": "2.4.4", "@biomejs/cli-linux-x64-musl": "2.4.4", "@biomejs/cli-win32-arm64": "2.4.4", "@biomejs/cli-win32-x64": "2.4.4" }, "bin": { "biome": "bin/biome" } }, "sha512-tigwWS5KfJf0cABVd52NVaXyAVv4qpUXOWJ1rxFL8xF1RVoeS2q/LK+FHgYoKMclJCuRoCWAPy1IXaN9/mS61Q=="],
-
-    "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jZ+Xc6qvD6tTH5jM6eKX44dcbyNqJHssfl2nnwT6vma6B1sj7ZLTGIk6N5QwVBs5xGN52r3trk5fgd3sQ9We9A=="],
-
-    "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dh1a/+W+SUCXhEdL7TiX3ArPTFCQKJTI1mGncZNWfO+6suk+gYA4lNyJcBB+pwvF49uw0pEbUS49BgYOY4hzUg=="],
-
-    "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-V/NFfbWhsUU6w+m5WYbBenlEAz8eYnSqRMDMAW3K+3v0tYVkNyZn8VU0XPxk/lOqNXLSCCrV7FmV/u3SjCBShg=="],
-
-    "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-+sPAXq3bxmFwhVFJnSwkSF5Rw2ZAJMH3MF6C9IveAEOdSpgajPhoQhbbAK12SehN9j2QrHpk4J/cHsa/HqWaYQ=="],
-
-    "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-R4+ZCDtG9kHArasyBO+UBD6jr/FcFCTH8QkNTOCu0pRJzCWyWC4EtZa2AmUZB5h3e0jD7bRV2KvrENcf8rndBg=="],
-
-    "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-gGvFTGpOIQDb5CQ2VC0n9Z2UEqlP46c4aNgHmAMytYieTGEcfqhfCFnhs6xjt0S3igE6q5GLuIXtdQt3Izok+g=="],
-
-    "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-trzCqM7x+Gn832zZHgr28JoYagQNX4CZkUZhMUac2YxvvyDRLJDrb5m9IA7CaZLlX6lTQmADVfLEKP1et1Ma4Q=="],
-
-    "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.4", "", { "os": "win32", "cpu": "x64" }, "sha512-gnOHKVPFAAPrpoPt2t+Q6FZ7RPry/FDV3GcpU53P3PtLNnQjBmKyN2Vh/JtqXet+H4pme8CC76rScwdjDcT1/A=="],
-
-    "@ff-labs/fff-bun": ["@ff-labs/fff-bun@workspace:packages/fff-bun"],
-
-    "@ff-labs/fff-node": ["@ff-labs/fff-node@workspace:packages/fff-node"],
-
-    "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.3.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-PXgg5gqcS/rHwa1hF0JdM1y5TiyejVrMHoBmWY/DjtfYZoFTXie1RCFOkoG0b5diOOmUcuYarMpH7CSNTqwj+w=="],
-
-    "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.3.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-Nhssuh7GBpP5PiDSOl3+qnoIG7PJo+ec2oomDevnl9pRY6x6aD2gRt0JE+uf+A8Om2D6gjeHCxjEdrw5ZHE8mA=="],
-
-    "@oven/bun-darwin-x64-baseline": ["@oven/bun-darwin-x64-baseline@1.3.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-w1gaTlqU0IJCmJ1X+PGHkdNU1n8Gemx5YKkjhkJIguvFINXEBB5U1KG82QsT65Tk4KyNMfbLTlmy4giAvUoKfA=="],
-
-    "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.3.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-OUgPHfL6+PM2Q+tFZjcaycN3D7gdQdYlWnwMI31DXZKY1r4HINWk9aEz9t/rNaHg65edwNrt7dsv9TF7xK8xIA=="],
-
-    "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.3.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-Ui5pAgM7JE9MzHokF0VglRMkbak3lTisY4Mf1AZutPACXWgKJC5aGrgnHBfkl7QS6fEeYb0juy1q4eRznRHOsw=="],
-
-    "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-bzUgYj/PIZziB/ZesIP9HUyfvh6Vlf3od+TrbTTyVEuCSMKzDPQVW/yEbRp0tcHO3alwiEXwJDrWrHAguXlgiQ=="],
-
-    "@oven/bun-linux-x64-baseline": ["@oven/bun-linux-x64-baseline@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-oqvMDYpX6dGJO03HgO5bXuccEsH3qbdO3MaAiAlO4CfkBPLUXz3N0DDElg5hz0L6ktdDVKbQVE5lfe+LAUISQg=="],
-
-    "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-poVXvOShekbexHq45b4MH/mRjQKwACAC8lHp3Tz/hEDuz0/20oncqScnmKwzhBPEpqJvydXficXfBYuSim8opw=="],
-
-    "@oven/bun-linux-x64-musl-baseline": ["@oven/bun-linux-x64-musl-baseline@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-/hOZ6S1VsTX6vtbhWVL9aAnOrdpuO54mAGUWpTdMz7dFG5UBZ/VUEiK0pBkq9A1rlBk0GeD/6Y4NBFl8Ha7cRA=="],
-
-    "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.3.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-GXbz2swvN2DLw2dXZFeedMxSJtI64xQ9xp9Eg7Hjejg6mS2E4dP1xoQ2yAo2aZPi/2OBPAVaGzppI2q20XumHA=="],
-
-    "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.3.10", "", { "os": "win32", "cpu": "x64" }, "sha512-qaS1In3yfC/Z/IGQriVmF8GWwKuNqiw7feTSJWaQhH5IbL6ENR+4wGNPniZSJFaM/SKUO0e/YCRdoVBvgU4C1g=="],
-
-    "@oven/bun-windows-x64-baseline": ["@oven/bun-windows-x64-baseline@1.3.10", "", { "os": "win32", "cpu": "x64" }, "sha512-gh3UAHbUdDUG6fhLc1Csa4IGdtghue6U8oAIXWnUqawp6lwb3gOCRvp25IUnLF5vUHtgfMxuEUYV7YA2WxVutw=="],
-
-    "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
-
-    "@types/node": ["@types/node@22.19.15", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg=="],
-
-    "@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="],
-
-    "@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="],
-
-    "@yuuang/ffi-rs-darwin-x64": ["@yuuang/ffi-rs-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-sI3LpQQ34SX4nyOHc5yxA7FSqs9qPEUMqW/y/wWo9cuyPpaHMFsi/BeOVYsnC0syp3FrY7gzn6RnD6PlXCktXg=="],
-
-    "@yuuang/ffi-rs-linux-arm-gnueabihf": ["@yuuang/ffi-rs-linux-arm-gnueabihf@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-1WkcGkJTlwh4ZA59htKI+RXhiL3oKiYwLv7PO8LUf6FuADK73s5GcXp67iakKu243uYu+qGYr4RHco4ySddYhQ=="],
-
-    "@yuuang/ffi-rs-linux-arm64-gnu": ["@yuuang/ffi-rs-linux-arm64-gnu@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-J2PwqviycZxaEVA0Bwv38LqGDGSB9A1DPN4iYginYJZSvTvKW8kh7Tis0HbZrX1YDKnY8hi3lt0N0tCTNPDH5Q=="],
-
-    "@yuuang/ffi-rs-linux-arm64-musl": ["@yuuang/ffi-rs-linux-arm64-musl@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Hn1W1hBPssTaqikU1Bqp1XUdDdOgbnYVIOtR++LVx66hhrtjf/xrIUQOhTm+NmOFDG16JUKXe1skfM4gpaqYwg=="],
-
-    "@yuuang/ffi-rs-linux-x64-gnu": ["@yuuang/ffi-rs-linux-x64-gnu@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-kW6e+oCYZPvpH2ppPsffA18e1aLowtmWTRjVlyHtY04g/nQDepQvDUkkcvInh9fW5jLna7PjHvktW1tVgYIj2A=="],
-
-    "@yuuang/ffi-rs-linux-x64-musl": ["@yuuang/ffi-rs-linux-x64-musl@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-HTwblAzruUS16nQPrez3ozvEHm1Xxh8J8w7rZYrpmAcNl1hzyOT8z/hY70M9Rt9fOqQ4Ovgor9qVy/U3ZJo0ZA=="],
-
-    "@yuuang/ffi-rs-win32-arm64-msvc": ["@yuuang/ffi-rs-win32-arm64-msvc@1.3.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-WeZkGl2BP1U4tRhEQH+FXLQS52N8obp74smK5AAGOfzPAT1pHkq6+dVkC1QCSIt7dHJs7SPtlnQw+5DkdZYlWA=="],
-
-    "@yuuang/ffi-rs-win32-ia32-msvc": ["@yuuang/ffi-rs-win32-ia32-msvc@1.3.1", "", { "os": "win32", "cpu": [ "x64", "ia32", ] }, "sha512-rNGgMeCH5mdeHiMiJgt7wWXovZ+FHEfXhU9p4zZBH4n8M1/QnEsRUwlapISPLpILSGpoYS6iBuq9/fUlZY8Mhg=="],
-
-    "@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="],
-
-    "bun": ["bun@1.3.10", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.3.10", "@oven/bun-darwin-x64": "1.3.10", "@oven/bun-darwin-x64-baseline": "1.3.10", "@oven/bun-linux-aarch64": "1.3.10", "@oven/bun-linux-aarch64-musl": "1.3.10", "@oven/bun-linux-x64": "1.3.10", "@oven/bun-linux-x64-baseline": "1.3.10", "@oven/bun-linux-x64-musl": "1.3.10", "@oven/bun-linux-x64-musl-baseline": "1.3.10", "@oven/bun-windows-aarch64": "1.3.10", "@oven/bun-windows-x64": "1.3.10", "@oven/bun-windows-x64-baseline": "1.3.10" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-S/CXaXXIyA4CMjdMkYQ4T2YMqnAn4s0ysD3mlsY4bUiOCqGlv28zck4Wd4H4kpvbekx15S9mUeLQ7Uxd0tYTLA=="],
-
-    "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
-
-    "ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="],
-
-    "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
-
-    "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
-
-    "bun-types/@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="],
-
-    "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
-  }
-}

crates/fff-c/Cargo.toml

@@ -1,23 +1,24 @@
 [package]
 name = "fff-c"
-version = "0.7.1"
+version = "0.10.5"
 edition = "2024"
 description = "Raw C api of FFF file finder"
 license = "MIT"
 
+[lints]
+workspace = true
+
 [lib]
 crate-type = ["cdylib"]
 
 [features]
-default = []
-zlob = ["fff/zlob"]
+default = ["ripgrep"] # use ripgrep base crates to avoid requiring zig for rust crate
+ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"]
+zlob = ["fff/zlob", "fff-query-parser/zlob"]
 
 [dependencies]
-mimalloc.workspace = true
-tracing.workspace = true
 git2.workspace = true
 
-fff = { package = "fff-search", path = "../fff-core" , version = "0.7.1" }
-fff-query-parser = { path = "../fff-query-parser" , version = "0.7.1" }
-serde = { version = "1.0", features = ["derive"] }
+fff = { package = "fff-search", path = "../fff-core", version = "0.10.5", default-features = false }
+fff-query-parser = { path = "../fff-query-parser", version = "0.10.5", default-features = false }
 serde_json = "1.0"

crates/fff-c/cbindgen.toml

@@ -26,3 +26,8 @@ include = [
 
 [fn]
 sort_by = "None"
+# Translate `#[deprecated]` on extern "C" fns into a real C compiler
+# attribute so callers get a warning when they use a removed/legacy entry.
+# `{}` is substituted with the Rust deprecation note as a C string literal
+# (already quoted) — do not wrap in extra quotes.
+deprecated_with_note = "__attribute__((deprecated({})))"

crates/fff-c/src/accessors.rs

@@ -1,42 +1,70 @@
-//! Stable accessor functions for `fff-c` FFI struct fields.
-//!
-//! # Why this exists
-//!
-//! `fff-c` exposes its result types as plain `#[repr(C)]` structs. External
-//! consumers (Emacs Lisp via `emacs-ffi`, Python `ctypes`, etc.) that access
-//! fields by hardcoding byte offsets break silently whenever the struct layout
-//! changes — a new field shifts every subsequent offset with no compile-time
-//! warning.
-//!
-//! These functions turn field access into a **stable named API**: callers bind
-//! to a symbol name once and are fully insulated from layout changes.
-//!
-//! # Usage from Emacs Lisp (example)
-//!
-//! ```elisp
-//! (define-ffi-function fff--grep-match-line-content
-//!   "fff_grep_match_get_line_content" :pointer [:pointer] fff--library)
-//!
-//! (ffi-get-c-string (fff--grep-match-line-content match-ptr))
-//! ```
-//!
-//! # Array iteration
-//!
-//! To walk result arrays use `fff_search_result_get_item`,
-//! `fff_grep_result_get_match`, and `fff_search_result_get_score` — these are
-//! defined in the main `lib.rs` FFI surface alongside the search functions.
+//! Stable accessor functions for `fff-c` FFI struct fields: a named API so
+//! FFI callers (Emacs Lisp, Python `ctypes`, etc.) don't hardcode struct byte
+//! offsets that break silently on layout changes. For array iteration use
+//! `fff_search_result_get_item` / `fff_grep_result_get_match` in `lib.rs`.
 
 use std::ffi::c_char;
 use std::ptr;
 
-use crate::ffi_types::{FffFileItem, FffGrepMatch, FffGrepResult, FffMatchRange, FffSearchResult};
+use crate::ffi_types::{
+    FffFileItem, FffGrepMatch, FffGrepResult, FffMatchRange, FffResult, FffSearchResult,
+};
+
+// ── FffResult ────────────────────────────────────────────────────────────────
+
+/// Returns whether the operation completed successfully. Returns `false` if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_success(result: *const FffResult) -> bool {
+    if result.is_null() {
+        return false;
+    }
+    unsafe { (*result).success }
+}
+
+/// Returns the operation error message, or null when there is no error or `result` is null.
+///
+/// Do not free the returned pointer. It remains valid until `fff_free_result` is called.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_error(result: *const FffResult) -> *const c_char {
+    if result.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*result).error }
+}
+
+/// Returns the result payload handle, or null if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_handle(result: *const FffResult) -> *mut std::ffi::c_void {
+    if result.is_null() {
+        return ptr::null_mut();
+    }
+    unsafe { (*result).handle }
+}
+
+/// Returns the result integer payload. Returns `0` if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_int_value(result: *const FffResult) -> i64 {
+    if result.is_null() {
+        return 0;
+    }
+    unsafe { (*result).int_value }
+}
 
 // ── FffFileItem ──────────────────────────────────────────────────────────────
 
-/// Returns the relative path of a file item (e.g. `"src/main.rs"`).
-///
-/// Returns null if `item` is null. The returned pointer is valid for the
-/// lifetime of the owning `FffSearchResult`; do not free it directly.
+/// Relative path of a file item (e.g. `"src/main.rs"`); null if `item` is null. Do not free.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -50,9 +78,7 @@ pub unsafe extern "C" fn fff_file_item_get_relative_path(
     unsafe { (*item).relative_path }
 }
 
-/// Returns the file-name component of a file item (e.g. `"main.rs"`).
-///
-/// Returns null if `item` is null. Do not free the returned pointer.
+/// File-name component of a file item (e.g. `"main.rs"`); null if `item` is null. Do not free.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -64,10 +90,8 @@ pub unsafe extern "C" fn fff_file_item_get_file_name(item: *const FffFileItem) -
     unsafe { (*item).file_name }
 }
 
-/// Returns the git status string for a file item (e.g. `"M "`, `"??"`)
-/// or null if git is unavailable, the file is untracked, or `item` is null.
-///
-/// Do not free the returned pointer.
+/// Git status string of a file item (e.g. `"M "`, `"??"`); null if git is unavailable,
+/// the file is untracked, or `item` is null. Do not free.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -79,7 +103,7 @@ pub unsafe extern "C" fn fff_file_item_get_git_status(item: *const FffFileItem)
     unsafe { (*item).git_status }
 }
 
-/// Returns the file size in bytes. Returns `0` if `item` is null.
+/// File size in bytes; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -91,8 +115,7 @@ pub unsafe extern "C" fn fff_file_item_get_size(item: *const FffFileItem) -> u64
     unsafe { (*item).size }
 }
 
-/// Returns the last-modified time as seconds since the UNIX epoch.
-/// Returns `0` if `item` is null.
+/// Last-modified time as seconds since the UNIX epoch; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -104,7 +127,7 @@ pub unsafe extern "C" fn fff_file_item_get_modified(item: *const FffFileItem) ->
     unsafe { (*item).modified }
 }
 
-/// Returns the combined frecency score. Returns `0` if `item` is null.
+/// Combined frecency score; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -116,7 +139,7 @@ pub unsafe extern "C" fn fff_file_item_get_total_frecency_score(item: *const Fff
     unsafe { (*item).total_frecency_score }
 }
 
-/// Returns the access-based frecency score. Returns `0` if `item` is null.
+/// Access-based frecency score; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -128,7 +151,7 @@ pub unsafe extern "C" fn fff_file_item_get_access_frecency_score(item: *const Ff
     unsafe { (*item).access_frecency_score }
 }
 
-/// Returns the modification-based frecency score. Returns `0` if `item` is null.
+/// Modification-based frecency score; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -142,7 +165,7 @@ pub unsafe extern "C" fn fff_file_item_get_modification_frecency_score(
     unsafe { (*item).modification_frecency_score }
 }
 
-/// Returns `true` if the file was detected as binary. Returns `false` if `item` is null.
+/// `true` if the file was detected as binary; `false` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -156,9 +179,7 @@ pub unsafe extern "C" fn fff_file_item_get_is_binary(item: *const FffFileItem) -
 
 // ── FffGrepMatch ─────────────────────────────────────────────────────────────
 
-/// Returns the relative path of the file containing this grep match.
-///
-/// Returns null if `m` is null. Do not free the returned pointer.
+/// Relative path of the file containing this grep match; null if `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -170,9 +191,7 @@ pub unsafe extern "C" fn fff_grep_match_get_relative_path(m: *const FffGrepMatch
     unsafe { (*m).relative_path }
 }
 
-/// Returns the file-name component of the file containing this grep match.
-///
-/// Returns null if `m` is null. Do not free the returned pointer.
+/// File-name component of the file containing this grep match; null if `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -184,10 +203,8 @@ pub unsafe extern "C" fn fff_grep_match_get_file_name(m: *const FffGrepMatch) ->
     unsafe { (*m).file_name }
 }
 
-/// Returns the git status string for the matched file (e.g. `"M "`, `"??"`)
-/// or null if git is unavailable, the file is untracked, or `m` is null.
-///
-/// Do not free the returned pointer.
+/// Git status string of the matched file (e.g. `"M "`, `"??"`); null if git is unavailable,
+/// the file is untracked, or `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -199,9 +216,7 @@ pub unsafe extern "C" fn fff_grep_match_get_git_status(m: *const FffGrepMatch) -
     unsafe { (*m).git_status }
 }
 
-/// Returns the full text content of the matched line.
-///
-/// Returns null if `m` is null. Do not free the returned pointer.
+/// Full text content of the matched line; null if `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -213,8 +228,7 @@ pub unsafe extern "C" fn fff_grep_match_get_line_content(m: *const FffGrepMatch)
     unsafe { (*m).line_content }
 }
 
-/// Returns the 1-based line number of the match within its file.
-/// Returns `0` if `m` is null.
+/// 1-based line number of the match within its file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -226,8 +240,7 @@ pub unsafe extern "C" fn fff_grep_match_get_line_number(m: *const FffGrepMatch)
     unsafe { (*m).line_number }
 }
 
-/// Returns the 0-based column of the match start within its line.
-/// Returns `0` if `m` is null.
+/// 0-based column of the match start within its line; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -239,8 +252,7 @@ pub unsafe extern "C" fn fff_grep_match_get_col(m: *const FffGrepMatch) -> u32 {
     unsafe { (*m).col }
 }
 
-/// Returns the byte offset of the match start from the beginning of the file.
-/// Returns `0` if `m` is null.
+/// Byte offset of the match start from the beginning of the file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -252,7 +264,7 @@ pub unsafe extern "C" fn fff_grep_match_get_byte_offset(m: *const FffGrepMatch)
     unsafe { (*m).byte_offset }
 }
 
-/// Returns the file size in bytes for the matched file. Returns `0` if `m` is null.
+/// File size in bytes of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -264,8 +276,7 @@ pub unsafe extern "C" fn fff_grep_match_get_size(m: *const FffGrepMatch) -> u64
     unsafe { (*m).size }
 }
 
-/// Returns the combined frecency score for the matched file.
-/// Returns `0` if `m` is null.
+/// Combined frecency score of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -277,8 +288,7 @@ pub unsafe extern "C" fn fff_grep_match_get_total_frecency_score(m: *const FffGr
     unsafe { (*m).total_frecency_score }
 }
 
-/// Returns the access-based frecency score for the matched file.
-/// Returns `0` if `m` is null.
+/// Access-based frecency score of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -290,8 +300,7 @@ pub unsafe extern "C" fn fff_grep_match_get_access_frecency_score(m: *const FffG
     unsafe { (*m).access_frecency_score }
 }
 
-/// Returns the modification-based frecency score for the matched file.
-/// Returns `0` if `m` is null.
+/// Modification-based frecency score of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -305,8 +314,7 @@ pub unsafe extern "C" fn fff_grep_match_get_modification_frecency_score(
     unsafe { (*m).modification_frecency_score }
 }
 
-/// Returns the last-modified time as seconds since the UNIX epoch for the matched file.
-/// Returns `0` if `m` is null.
+/// Last-modified time of the matched file as seconds since the UNIX epoch; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -318,8 +326,7 @@ pub unsafe extern "C" fn fff_grep_match_get_modified(m: *const FffGrepMatch) ->
     unsafe { (*m).modified }
 }
 
-/// Returns the number of highlight ranges in this match. Returns `0` if `m` is null.
-///
+/// Number of highlight ranges in this match; `0` if `m` is null.
 /// Use with [`fff_grep_match_get_match_range`] to iterate the highlight spans.
 ///
 /// ## Safety
@@ -332,11 +339,8 @@ pub unsafe extern "C" fn fff_grep_match_get_match_ranges_count(m: *const FffGrep
     unsafe { (*m).match_ranges_count }
 }
 
-/// Returns a pointer to the `index`-th [`FffMatchRange`] highlight span.
-///
-/// Returns null if `m` is null, `index >= match_ranges_count`, or the
-/// ranges array is null. The returned pointer is valid until the owning
-/// `FffGrepResult` is freed; do not free it directly.
+/// Pointer to the `index`-th [`FffMatchRange`] highlight span; null if `m` is null,
+/// `index >= match_ranges_count`, or the ranges array is null. Valid until the owning `FffGrepResult` is freed; do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -355,9 +359,7 @@ pub unsafe extern "C" fn fff_grep_match_get_match_range(
     unsafe { m.match_ranges.add(index as usize) }
 }
 
-/// Returns the number of context lines captured before the match.
-/// Returns `0` if `m` is null.
-///
+/// Number of context lines captured before the match; `0` if `m` is null.
 /// Use with [`fff_grep_match_get_context_before`] to read each line.
 ///
 /// ## Safety
@@ -370,10 +372,8 @@ pub unsafe extern "C" fn fff_grep_match_get_context_before_count(m: *const FffGr
     unsafe { (*m).context_before_count }
 }
 
-/// Returns the `index`-th context line before the match.
-///
-/// Returns null if `m` is null, `index >= context_before_count`, or the
-/// context array is null. Do not free the returned pointer.
+/// The `index`-th context line before the match; null if `m` is null,
+/// `index >= context_before_count`, or the context array is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -392,9 +392,7 @@ pub unsafe extern "C" fn fff_grep_match_get_context_before(
     unsafe { *m.context_before.add(index as usize) }
 }
 
-/// Returns the number of context lines captured after the match.
-/// Returns `0` if `m` is null.
-///
+/// Number of context lines captured after the match; `0` if `m` is null.
 /// Use with [`fff_grep_match_get_context_after`] to read each line.
 ///
 /// ## Safety
@@ -407,10 +405,8 @@ pub unsafe extern "C" fn fff_grep_match_get_context_after_count(m: *const FffGre
     unsafe { (*m).context_after_count }
 }
 
-/// Returns the `index`-th context line after the match.
-///
-/// Returns null if `m` is null, `index >= context_after_count`, or the
-/// context array is null. Do not free the returned pointer.
+/// The `index`-th context line after the match; null if `m` is null,
+/// `index >= context_after_count`, or the context array is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -429,11 +425,8 @@ pub unsafe extern "C" fn fff_grep_match_get_context_after(
     unsafe { *m.context_after.add(index as usize) }
 }
 
-/// Returns the fuzzy match score. Returns `0` if `m` is null or no fuzzy
-/// score is present.
-///
-/// Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is
-/// ambiguous without that flag.
+/// Fuzzy match score; `0` if `m` is null or no fuzzy score is present.
+/// Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is ambiguous without that flag.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -445,8 +438,7 @@ pub unsafe extern "C" fn fff_grep_match_get_fuzzy_score(m: *const FffGrepMatch)
     unsafe { (*m).fuzzy_score }
 }
 
-/// Returns `true` if this match carries a valid fuzzy score.
-/// Returns `false` if `m` is null.
+/// `true` if this match carries a valid fuzzy score; `false` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -458,8 +450,7 @@ pub unsafe extern "C" fn fff_grep_match_get_has_fuzzy_score(m: *const FffGrepMat
     unsafe { (*m).has_fuzzy_score }
 }
 
-/// Returns `true` if the match was identified as a symbol definition.
-/// Returns `false` if `m` is null.
+/// `true` if the match was identified as a symbol definition; `false` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -471,8 +462,7 @@ pub unsafe extern "C" fn fff_grep_match_get_is_definition(m: *const FffGrepMatch
     unsafe { (*m).is_definition }
 }
 
-/// Returns `true` if the matched file was detected as binary.
-/// Returns `false` if `m` is null.
+/// `true` if the matched file was detected as binary; `false` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -486,7 +476,7 @@ pub unsafe extern "C" fn fff_grep_match_get_is_binary(m: *const FffGrepMatch) ->
 
 // ── FffSearchResult ──────────────────────────────────────────────────────────
 
-/// Returns the number of items in the result. Returns `0` if `r` is null.
+/// Number of items in the result; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffSearchResult` pointer or null.
@@ -498,8 +488,7 @@ pub unsafe extern "C" fn fff_search_result_get_count(r: *const FffSearchResult)
     unsafe { (*r).count }
 }
 
-/// Returns the total number of files that matched before the result was
-/// truncated to the page size. Returns `0` if `r` is null.
+/// Total number of files that matched before truncation to the page size; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffSearchResult` pointer or null.
@@ -511,8 +500,7 @@ pub unsafe extern "C" fn fff_search_result_get_total_matched(r: *const FffSearch
     unsafe { (*r).total_matched }
 }
 
-/// Returns the total number of indexed files considered during search.
-/// Returns `0` if `r` is null.
+/// Total number of indexed files considered during search; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffSearchResult` pointer or null.
@@ -526,7 +514,7 @@ pub unsafe extern "C" fn fff_search_result_get_total_files(r: *const FffSearchRe
 
 // ── FffGrepResult ─────────────────────────────────────────────────────────────
 
-/// Returns the number of matches in the result. Returns `0` if `r` is null.
+/// Number of matches in the result; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -538,8 +526,7 @@ pub unsafe extern "C" fn fff_grep_result_get_count(r: *const FffGrepResult) -> u
     unsafe { (*r).count }
 }
 
-/// Returns the total number of matches found across all pages.
-/// Returns `0` if `r` is null.
+/// Total number of matches found across all pages; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -551,8 +538,7 @@ pub unsafe extern "C" fn fff_grep_result_get_total_matched(r: *const FffGrepResu
     unsafe { (*r).total_matched }
 }
 
-/// Returns the number of files actually opened and searched in this call.
-/// Returns `0` if `r` is null.
+/// Number of files actually opened and searched in this call; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -564,8 +550,7 @@ pub unsafe extern "C" fn fff_grep_result_get_total_files_searched(r: *const FffG
     unsafe { (*r).total_files_searched }
 }
 
-/// Returns the total number of indexed files before any filtering.
-/// Returns `0` if `r` is null.
+/// Total number of indexed files before any filtering; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -577,8 +562,7 @@ pub unsafe extern "C" fn fff_grep_result_get_total_files(r: *const FffGrepResult
     unsafe { (*r).total_files }
 }
 
-/// Returns the number of files eligible for search after path/type filtering.
-/// Returns `0` if `r` is null.
+/// Number of files eligible for search after path/type filtering; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -590,9 +574,8 @@ pub unsafe extern "C" fn fff_grep_result_get_filtered_file_count(r: *const FffGr
     unsafe { (*r).filtered_file_count }
 }
 
-/// Returns the file offset for the next page, or `0` if all files have been
-/// searched or `r` is null. Pass this value as `file_offset` to a subsequent
-/// `fff_live_grep` or `fff_multi_grep` call to continue pagination.
+/// File offset for the next page; `0` if all files have been searched or `r` is null.
+/// Pass as `file_offset` to a subsequent `fff_live_grep`/`fff_multi_grep` call to continue pagination.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -604,10 +587,8 @@ pub unsafe extern "C" fn fff_grep_result_get_next_file_offset(r: *const FffGrepR
     unsafe { (*r).next_file_offset }
 }
 
-/// Returns the regex compilation error string if the engine fell back to
-/// literal matching, or null if there was no error or `r` is null.
-///
-/// Do not free the returned pointer.
+/// Regex compilation error string if the engine fell back to literal matching;
+/// null if there was no error or `r` is null. Do not free.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -801,8 +782,41 @@ mod tests {
         }
     }
 
+    #[test]
+    fn null_result_returns_zero_or_null() {
+        let null: *const FffResult = ptr::null();
+        unsafe {
+            assert!(!fff_result_get_success(null));
+            assert!(fff_result_get_error(null).is_null());
+            assert!(fff_result_get_handle(null).is_null());
+            assert_eq!(fff_result_get_int_value(null), 0);
+        }
+    }
+
     // ── data correctness tests ────────────────────────────────────────────────
 
+    #[test]
+    fn result_getters_return_correct_values() {
+        let error = CString::new("failed").unwrap();
+        let handle = 0x1234usize as *mut std::ffi::c_void;
+        let result = FffResult {
+            success: false,
+            error: error.as_ptr() as *mut std::ffi::c_char,
+            handle,
+            int_value: -7,
+        };
+        let p = &result as *const FffResult;
+        unsafe {
+            assert!(!fff_result_get_success(p));
+            assert_eq!(
+                std::ffi::CStr::from_ptr(fff_result_get_error(p)),
+                error.as_c_str()
+            );
+            assert_eq!(fff_result_get_handle(p), handle);
+            assert_eq!(fff_result_get_int_value(p), -7);
+        }
+    }
+
     #[test]
     fn file_item_getters_return_correct_values() {
         let mut item = make_file_item("src/main.rs", "main.rs");

crates/fff-c/src/ffi_types.rs

@@ -1,8 +1,5 @@
-//! FFI-compatible type definitions
-//!
-//! All result types use `#[repr(C)]` structs for direct memory access from any
-//! language with C FFI support. No JSON serialization is used for search or grep
-//! results — callers read struct fields directly.
+//! FFI-compatible type definitions: all result types are `#[repr(C)]` structs
+//! read directly from any language with C FFI — no JSON serialization.
 
 use std::ffi::{CString, c_char, c_void};
 use std::ptr;
@@ -14,6 +11,79 @@ use fff::{
     MixedSearchResult, Score, SearchResult,
 };
 
+/// Current used version of [`FffCreateOptions`].
+pub const FFF_CREATE_OPTIONS_VERSION: u32 = 2;
+
+/// Options for `fff_create_instance_with`.
+///
+/// Versioned struct: the layout is stable across releases, new fields are
+/// only appended.
+#[repr(C)]
+pub struct FffCreateOptions {
+    /// Set to [`FFF_CREATE_OPTIONS_VERSION`] when allocating; tells the
+    /// library which trailing fields are populated.
+    pub version: u32,
+    /// Directory to index (required, non-NULL).
+    pub base_path: *const c_char,
+    /// Frecency LMDB database path. NULL/empty to skip frecency tracking.
+    pub frecency_db_path: *const c_char,
+    /// Query history LMDB database path. NULL/empty to skip query tracking.
+    pub history_db_path: *const c_char,
+    /// Pre-populate mmap caches for top-frecency files after the initial scan.
+    pub enable_mmap_cache: bool,
+    /// Build content index after the initial scan for faster grep.
+    pub enable_content_indexing: bool,
+    /// Start a background file-system watcher for live updates.
+    pub watch: bool,
+    /// Enable AI-agent optimizations.
+    pub ai_mode: bool,
+    /// Tracing log file path. NULL/empty to skip log init.
+    pub log_file_path: *const c_char,
+    /// Log level: `"trace" | "debug" | "info" | "warn" | "error"`.
+    /// NULL/empty defaults to `"info"`. Ignored when `log_file_path` is unset.
+    pub log_level: *const c_char,
+    /// Content cache file-count cap. 0 = auto.
+    pub cache_budget_max_files: u64,
+    /// Content cache byte cap. 0 = auto.
+    pub cache_budget_max_bytes: u64,
+    /// Per-file byte cap inside the content cache. 0 = auto.
+    pub cache_budget_max_file_size: u64,
+    /// Allow indexing the filesystem root (`/`). Off by default: root is rarely
+    /// intended and floods the watcher with churn.
+    pub enable_fs_root_scanning: bool,
+    /// Allow indexing the user's home directory. Same trade-off as `enable_fs_root_scanning`.
+    pub enable_home_dir_scanning: bool,
+    // ----- v2 fields -----
+    /// Follow symlinks during scan and watcher walks. Off by default: without
+    /// external loop protection cyclic symlinks can wedge the watcher.
+    pub follow_symlinks: bool,
+    // ----- new version 3+ fields go here, ALWAYS appended -----
+}
+
+impl FffCreateOptions {
+    /// Default values for a v1 options struct.
+    pub fn defaults() -> Self {
+        Self {
+            version: FFF_CREATE_OPTIONS_VERSION,
+            base_path: ptr::null(),
+            frecency_db_path: ptr::null(),
+            history_db_path: ptr::null(),
+            enable_mmap_cache: true,
+            enable_content_indexing: true,
+            watch: true,
+            ai_mode: false,
+            log_file_path: ptr::null(),
+            log_level: ptr::null(),
+            cache_budget_max_files: 0,
+            cache_budget_max_bytes: 0,
+            cache_budget_max_file_size: 0,
+            enable_fs_root_scanning: false,
+            enable_home_dir_scanning: false,
+            follow_symlinks: false,
+        }
+    }
+}
+
 /// Allocate a heap CString from a `&str`, returning a raw pointer.
 fn cstring_new(s: &str) -> *mut c_char {
     CString::new(s).unwrap_or_default().into_raw()
@@ -58,10 +128,8 @@ unsafe fn free_cstring_array(arr: *mut *mut c_char, count: u32) {
     }
 }
 
-/// A file item returned by `fff_search`.
-///
-/// All string fields are heap-allocated and owned by the parent `FffSearchResult`.
-/// Free the entire result with `fff_free_search_result`.
+/// A file item returned by `fff_search`. Strings are owned by the parent
+/// `FffSearchResult`; free everything with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffFileItem {
     pub relative_path: *mut c_char,
@@ -155,13 +223,9 @@ impl FffScore {
     }
 }
 
-/// Location parsed from a query string (e.g. `"file.ts:42:10"`).
-///
-/// `tag` encodes the variant:
-///   0 = no location,
-///   1 = line only (`line` is set),
-///   2 = position (`line` + `col`),
-///   3 = range (`line`/`col` = start, `end_line`/`end_col` = end).
+/// Location parsed from a query string (e.g. `"file.ts:42:10"`). `tag`:
+/// 0 = none, 1 = line, 2 = position (`line` + `col`),
+/// 3 = range (`line`/`col` = start, `end_line`/`end_col` = end).
 #[repr(C)]
 pub struct FffLocation {
     pub tag: u8,
@@ -206,14 +270,12 @@ impl From<Option<&Location>> for FffLocation {
     }
 }
 
-/// Search result returned by `fff_search`.
-///
-/// The caller must free this with `fff_free_search_result`.
+/// Search result returned by `fff_search`; free with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffSearchResult {
-    /// Pointer to a heap-allocated array of `FffFileItem` (length = `count`).
+    /// Heap array of `FffFileItem` (length = `count`).
     pub items: *mut FffFileItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -261,10 +323,8 @@ pub struct FffMatchRange {
     pub end: u32,
 }
 
-/// A single grep match with file and line information.
-///
-/// All string fields and arrays are heap-allocated. Free the parent
-/// `FffGrepResult` with `fff_free_grep_result` to release everything.
+/// A single grep match with file and line information. Strings and arrays are
+/// owned by the parent `FffGrepResult`; free everything with `fff_free_grep_result`.
 #[repr(C)]
 pub struct FffGrepMatch {
     // -- pointers (8 bytes each) --
@@ -366,12 +426,11 @@ impl FffGrepMatch {
     }
 }
 
-/// Grep result returned by `fff_live_grep` and `fff_multi_grep`.
-///
-/// The caller must free this with `fff_free_grep_result`.
+/// Grep result returned by `fff_live_grep` and `fff_multi_grep`;
+/// free with `fff_free_grep_result`.
 #[repr(C)]
 pub struct FffGrepResult {
-    /// Pointer to a heap-allocated array of `FffGrepMatch` (length = `count`).
+    /// Heap array of `FffGrepMatch` (length = `count`).
     pub items: *mut FffGrepMatch,
     /// Number of matches in the `items` array.
     pub count: u32,
@@ -420,7 +479,9 @@ impl FffGrepResult {
 
 /// Result envelope returned by all `fff_*` functions.
 ///
-/// Heap-allocated — the caller must free it with `fff_free_result`.
+/// Heap-allocated. The caller must free it with `fff_free_result`. Calling `fff_free_result`
+/// **does not** deallocate the underlying `handle` pointer. It needs to be cleaned separately.
+/// see (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`, `fff_free_string`, etc.).
 ///
 /// Depending on the function, the payload is delivered through different fields:
 ///
@@ -440,18 +501,13 @@ impl FffGrepResult {
 /// | `fff_restart_index`        | (none)        | success flag only             |
 ///
 /// On failure, `success` is false and `error` contains the message.
-///
-/// **Important:** `fff_free_result` frees `error` but does **not** free `handle`.
-/// The caller must free the handle with the appropriate function
-/// (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`,
-///  `fff_free_string`, etc.).
 #[repr(C)]
 pub struct FffResult {
     /// Whether the operation succeeded.
     pub success: bool,
     /// Error message on failure. Null on success.
     pub error: *mut c_char,
-    /// Opaque pointer payload (instance handle, typed result struct, or string). May be null.
+    /// Opaque pointer payload. May be null.
     pub handle: *mut c_void,
     /// Integer payload for simple return values (bool as 0/1, counts, etc.).
     pub int_value: i64,
@@ -511,10 +567,8 @@ impl FffResult {
     }
 }
 
-/// A directory item returned by `fff_search_directories`.
-///
-/// All string fields are heap-allocated and owned by the parent `FffDirSearchResult`.
-/// Free the entire result with `fff_free_dir_search_result`.
+/// A directory item returned by `fff_search_directories`. Strings are owned by
+/// the parent `FffDirSearchResult`; free everything with `fff_free_dir_search_result`.
 #[repr(C)]
 pub struct FffDirItem {
     pub relative_path: *mut c_char,
@@ -545,14 +599,13 @@ impl FffDirItem {
     }
 }
 
-/// Directory search result returned by `fff_search_directories`.
-///
-/// The caller must free this with `fff_free_dir_search_result`.
+/// Directory search result returned by `fff_search_directories`;
+/// free with `fff_free_dir_search_result`.
 #[repr(C)]
 pub struct FffDirSearchResult {
-    /// Pointer to a heap-allocated array of `FffDirItem` (length = `count`).
+    /// Heap array of `FffDirItem` (length = `count`).
     pub items: *mut FffDirItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -587,9 +640,8 @@ impl FffDirSearchResult {
 }
 
 /// A single item in a mixed (files + directories) search result.
-///
-/// `item_type`: 0 = file, 1 = directory.
-/// All string fields are heap-allocated and owned by the parent `FffMixedSearchResult`.
+/// `item_type`: 0 = file, 1 = directory. Strings are owned by the parent
+/// `FffMixedSearchResult`.
 #[repr(C)]
 pub struct FffMixedItem {
     /// 0 = file, 1 = directory.
@@ -600,8 +652,7 @@ pub struct FffMixedItem {
     pub git_status: *mut c_char,
     pub size: u64,
     pub modified: u64,
-    /// The access frecency score for files, or max access frecency among all the immediate
-    /// children for directories.
+    /// Access frecency for files; max among immediate children for directories.
     pub access_frecency_score: i64,
     /// Always 0 for directories
     pub modification_frecency_score: i64,
@@ -658,14 +709,13 @@ impl FffMixedItem {
     }
 }
 
-/// Mixed search result returned by `fff_search_mixed`.
-///
-/// The caller must free this with `fff_free_mixed_search_result`.
+/// Mixed search result returned by `fff_search_mixed`
+/// free with `fff_free_mixed_search_result`.
 #[repr(C)]
 pub struct FffMixedSearchResult {
-    /// Pointer to a heap-allocated array of `FffMixedItem` (length = `count`).
+    /// Heap array of `FffMixedItem` (length = `count`).
     pub items: *mut FffMixedItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -725,3 +775,35 @@ impl From<fff::file_picker::ScanProgress> for FffScanProgress {
         }
     }
 }
+
+#[cfg(test)]
+mod options_layout_tests {
+    use super::FffCreateOptions;
+    use std::mem::{align_of, offset_of, size_of};
+
+    // THIS TEST HAVE TO BE NEVER UPDATED ONLY ADDED NEW FIELDS
+    // this is needed to ensure ABI backward compatibility
+    #[test]
+    #[cfg(target_pointer_width = "64")]
+    fn fff_create_options_layout_is_stable_64bit() {
+        assert_eq!(size_of::<FffCreateOptions>(), 88);
+        assert_eq!(align_of::<FffCreateOptions>(), 8);
+
+        assert_eq!(offset_of!(FffCreateOptions, version), 0);
+        assert_eq!(offset_of!(FffCreateOptions, base_path), 8);
+        assert_eq!(offset_of!(FffCreateOptions, frecency_db_path), 16);
+        assert_eq!(offset_of!(FffCreateOptions, history_db_path), 24);
+        assert_eq!(offset_of!(FffCreateOptions, enable_mmap_cache), 32);
+        assert_eq!(offset_of!(FffCreateOptions, enable_content_indexing), 33);
+        assert_eq!(offset_of!(FffCreateOptions, watch), 34);
+        assert_eq!(offset_of!(FffCreateOptions, ai_mode), 35);
+        assert_eq!(offset_of!(FffCreateOptions, log_file_path), 40);
+        assert_eq!(offset_of!(FffCreateOptions, log_level), 48);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_files), 56);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_bytes), 64);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_file_size), 72);
+        assert_eq!(offset_of!(FffCreateOptions, enable_fs_root_scanning), 80);
+        assert_eq!(offset_of!(FffCreateOptions, enable_home_dir_scanning), 81);
+        assert_eq!(offset_of!(FffCreateOptions, follow_symlinks), 82);
+    }
+}

crates/fff-c/src/lib.rs

@@ -1,26 +1,14 @@
-//! C FFI bindings for fff-core
+//! C FFI bindings for fff-core, usable from any language with C FFI
+//! (Bun, Node.js, Python, Ruby, etc.).
 //!
-//! This crate provides C-compatible FFI exports that can be used from any language
-//! with C FFI support (Bun, Node.js, Python, Ruby, etc.).
+//! All state is owned by an opaque instance handle: create with
+//! `fff_create_instance*`, pass to every call, free with `fff_destroy`.
+//! Multiple instances can coexist in one process.
 //!
-//! # Instance-based API
-//!
-//! All state is owned by an opaque `FffInstance` fff_handle. Callers create an instance
-//! with `fff_create_instance`, pass the fff_handle to every subsequent call, and free it with
-//! `fff_destroy`. Multiple independent instances can coexist in the same process.
-//!
-//! # Memory management
-//!
-//! * Every `fff_*` function that returns `*mut FffResult` requires the caller to
-//!   free the result with `fff_free_result`.
-//! * The instance itself must be freed with `fff_destroy`.
-//!
-//! # Parameter conventions
-//!
-//! * Optional `*const c_char` parameters: pass NULL or an empty string to omit.
-//! * Numeric parameters: 0 means "use default" unless documented otherwise.
-//! * Grep mode (`u8`): 0 = plain text, 1 = regex, 2 = fuzzy.
-//! * Multi-grep patterns are passed as a single newline-separated (`\n`) string.
+//! Conventions: every returned `*mut FffResult` is freed with
+//! `fff_free_result`; optional string params take NULL/empty; numeric 0 means
+//! "use default" unless documented otherwise; grep mode `u8` is 0 = plain
+//! text, 1 = regex, 2 = fuzzy; multi-grep patterns are `\n`-separated.
 
 use std::ffi::{CStr, CString, c_char, c_void};
 use std::path::PathBuf;
@@ -30,6 +18,7 @@ use fff::shared::SharedQueryTracker;
 
 mod accessors;
 mod ffi_types;
+mod watch;
 
 use fff::file_picker::FilePicker;
 use fff::frecency::FrecencyTracker;
@@ -37,24 +26,22 @@ use fff::query_tracker::QueryTracker;
 use fff::{DbHealthChecker, FFFMode, FuzzySearchOptions, PaginationArgs, QueryParser};
 use fff::{SharedFilePicker, SharedFrecency};
 use ffi_types::{
-    FffDirItem, FffDirSearchResult, FffFileItem, FffGrepMatch, FffGrepResult, FffMixedItem,
-    FffMixedSearchResult, FffResult, FffScanProgress, FffScore, FffSearchResult,
+    FFF_CREATE_OPTIONS_VERSION, FffCreateOptions, FffDirItem, FffDirSearchResult, FffFileItem,
+    FffGrepMatch, FffGrepResult, FffMixedItem, FffMixedSearchResult, FffResult, FffScanProgress,
+    FffScore, FffSearchResult,
 };
 
-/// Opaque fff_handle holding all per-instance state.
-///
-/// The caller receives this as `*mut c_void` and must pass it to every FFI call.
-/// The fff_handle is freed by `fff_destroy`.
+/// Opaque handle holding all per-instance state; freed by `fff_destroy`.
 struct FffInstance {
     picker: SharedFilePicker,
     frecency: SharedFrecency,
     query_tracker: SharedQueryTracker,
+    // we keep a single callback type
+    watch_callback: std::sync::Arc<watch::WatchCallbackSlot>,
 }
 
-/// Helper to convert C string to Rust &str.
-///
-/// Returns `None` if the pointer is null or the string is not valid UTF-8.
-unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> {
+/// Convert a C string to `&str`; `None` if null or invalid UTF-8.
+pub(crate) unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> {
     if s.is_null() {
         None
     } else {
@@ -62,17 +49,15 @@ unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> {
     }
 }
 
-/// Helper to convert an optional C string parameter.
-///
-/// Returns `None` if the pointer is null, empty, or not valid UTF-8.
+/// Optional C string param: `None` if null, empty, or invalid UTF-8.
 unsafe fn optional_cstr<'a>(s: *const c_char) -> Option<&'a str> {
     unsafe { cstr_to_str(s) }.filter(|s| !s.is_empty())
 }
 
-/// Recover a `&FffInstance` from the opaque pointer.
-///
-/// Returns an error `FffResult` if the pointer is null.
-unsafe fn instance_ref<'a>(fff_handle: *mut c_void) -> Result<&'a FffInstance, *mut FffResult> {
+/// Recover a `&FffInstance` from the opaque pointer; error `FffResult` if null.
+pub(crate) unsafe fn instance_ref<'a>(
+    fff_handle: *mut c_void,
+) -> Result<&'a FffInstance, *mut FffResult> {
     if fff_handle.is_null() {
         Err(FffResult::err(
             "Instance handle is null. Create one with fff_create_instance first.",
@@ -104,17 +89,17 @@ fn default_i32(val: i32, default: i32) -> i32 {
     if val == 0 { default } else { val }
 }
 
-/// Create a new file finder instance (legacy signature).
+/// Create a new file finder instance (legacy 8-arg positional signature).
 ///
-/// @deprecated prefer `fff_create_instance2`, which also exposes log file and
-/// cache-budget configuration. This function delegates to `fff_create_instance2`
-/// with NULL log paths and auto cache budget, so behaviour is unchanged.
-///
-/// The `use_unsafe_no_lock` parameter is deprecated and ignored; see
-/// [`fff_create_instance2`] for details.
+/// @deprecated Use [`fff_create_instance_with`] (or [`fff_create_instance_with_value`]
+/// for FFI bindings). The `use_unsafe_no_lock` parameter is ignored.
 ///
 /// ## Safety
-/// See `fff_create_instance2`.
+/// See `fff_create_instance_with`.
+#[deprecated(
+    since = "0.8.5",
+    note = "Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks."
+)]
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn fff_create_instance(
     base_path: *const c_char,
@@ -126,59 +111,28 @@ pub unsafe extern "C" fn fff_create_instance(
     watch: bool,
     ai_mode: bool,
 ) -> *mut FffResult {
-    unsafe {
-        fff_create_instance2(
-            base_path,
-            frecency_db_path,
-            history_db_path,
-            false,
-            enable_mmap_cache,
-            enable_content_indexing,
-            watch,
-            ai_mode,
-            std::ptr::null(),
-            std::ptr::null(),
-            0,
-            0,
-            0,
-        )
-    }
+    let mut opts = FffCreateOptions::defaults();
+    opts.base_path = base_path;
+    opts.frecency_db_path = frecency_db_path;
+    opts.history_db_path = history_db_path;
+    opts.enable_mmap_cache = enable_mmap_cache;
+    opts.enable_content_indexing = enable_content_indexing;
+    opts.watch = watch;
+    opts.ai_mode = ai_mode;
+    unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) }
 }
 
-/// Create a new file finder instance (v2, with full options).
+/// Create a new file finder instance (legacy 13-arg positional signature).
 ///
-/// Returns an opaque pointer that must be passed to all other `fff_*` calls
-/// and eventually freed with `fff_destroy`.
-///
-/// # Parameters
-///
-/// * `base_path`                   – directory to index (required)
-/// * `frecency_db_path`            – frecency LMDB database path (NULL/empty to skip)
-/// * `history_db_path`             – query history LMDB database path (NULL/empty to skip)
-/// * `use_unsafe_no_lock`          – **deprecated, ignored.** Previously enabled
-///   `MDB_NOLOCK|MDB_NOSYNC|MDB_NOMETASYNC` for LMDB; benchmarks showed no
-///   measurable win under realistic contention, so the flag is now a no-op.
-///   The parameter remains in the signature for ABI compatibility and will be
-///   removed in a future release.
-/// * `enable_mmap_cache`           – pre-populate mmap caches after the initial scan
-/// * `enable_content_indexing`     – build content index after the initial scan
-/// * `watch`                       – start a background file-system watcher for live updates
-/// * `ai_mode`                     – enable AI-agent optimizations
-/// * `log_file_path`               – tracing log file path (NULL/empty to skip).
-///   Only the first successful call in a process installs the subscriber;
-///   subsequent calls are no-ops at the log layer.
-/// * `log_level`                   – `"trace"`, `"debug"`, `"info"`, `"warn"`, `"error"`
-///   (NULL/empty defaults to `"info"`). Ignored when `log_file_path` is not set.
-/// * `cache_budget_max_files`      – content cache file-count cap (0 = auto)
-/// * `cache_budget_max_bytes`      – content cache byte cap (0 = auto)
-/// * `cache_budget_max_file_size`  – per-file byte cap (0 = auto)
-///
-/// When all three `cache_budget_*` values are 0 the budget is auto-computed
-/// from repo size after the initial scan. Otherwise an explicit budget is
-/// used: any field left at 0 falls back to its `unlimited()` default.
+/// @deprecated Use [`fff_create_instance_with`] (or [`fff_create_instance_with_value`]
+/// for FFI bindings). The `use_unsafe_no_lock` parameter is ignored.
 ///
 /// ## Safety
-/// String parameters must be valid null-terminated UTF-8 or NULL.
+/// See `fff_create_instance_with`.
+#[deprecated(
+    since = "0.8.5",
+    note = "Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks."
+)]
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn fff_create_instance2(
     base_path: *const c_char,
@@ -195,27 +149,68 @@ pub unsafe extern "C" fn fff_create_instance2(
     cache_budget_max_bytes: u64,
     cache_budget_max_file_size: u64,
 ) -> *mut FffResult {
-    let base_path_str = match unsafe { cstr_to_str(base_path) } {
+    let mut opts = FffCreateOptions::defaults();
+    opts.base_path = base_path;
+    opts.frecency_db_path = frecency_db_path;
+    opts.history_db_path = history_db_path;
+    opts.enable_mmap_cache = enable_mmap_cache;
+    opts.enable_content_indexing = enable_content_indexing;
+    opts.watch = watch;
+    opts.ai_mode = ai_mode;
+    opts.log_file_path = log_file_path;
+    opts.log_level = log_level;
+    opts.cache_budget_max_files = cache_budget_max_files;
+    opts.cache_budget_max_bytes = cache_budget_max_bytes;
+    opts.cache_budget_max_file_size = cache_budget_max_file_size;
+    unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) }
+}
+
+/// Create a new file finder instance from a versioned [`FffCreateOptions`] struct.
+///
+/// Populate the struct, set `version` to [`FFF_CREATE_OPTIONS_VERSION`], pass by
+/// pointer. New fields are only appended; older `version` values keep working.
+/// FFI bindings needing struct-by-value should use [`fff_create_instance_with_value`].
+///
+/// `opts.base_path` is required (non-NULL, non-empty). Zero `cache_budget_*`
+/// values are auto-computed from repo size after the initial scan.
+///
+/// ## Safety
+/// * `opts` must be a valid pointer to an `FffCreateOptions` whose `version`
+///   is in the range `1..=FFF_CREATE_OPTIONS_VERSION`.
+/// * All string pointers inside `opts` must be valid null-terminated UTF-8
+///   or NULL.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_create_instance_with(opts: *const FffCreateOptions) -> *mut FffResult {
+    if opts.is_null() {
+        return FffResult::err("opts is null");
+    }
+    let opts = unsafe { &*opts };
+    if opts.version == 0 || opts.version > FFF_CREATE_OPTIONS_VERSION {
+        return FffResult::err(&format!(
+            "Unsupported FffCreateOptions version {} (library understands up to {})",
+            opts.version, FFF_CREATE_OPTIONS_VERSION
+        ));
+    }
+
+    let base_path_str = match unsafe { cstr_to_str(opts.base_path) } {
         Some(s) if !s.is_empty() => s.to_string(),
-        _ => return FffResult::err("base_path is null or empty"),
+        _ => return FffResult::err("opts.base_path is null or empty"),
     };
 
-    if let Some(log_path) = unsafe { optional_cstr(log_file_path) } {
-        let level = unsafe { optional_cstr(log_level) };
-        if let Err(e) = fff::log::init_tracing(log_path, level) {
+    if let Some(log_path) = unsafe { optional_cstr(opts.log_file_path) } {
+        let level = unsafe { optional_cstr(opts.log_level) };
+        if let Err(e) = fff::log::init_tracing(log_path, level, None) {
             return FffResult::err(&format!("Failed to init tracing: {}", e));
         }
     }
 
-    let frecency_path = unsafe { optional_cstr(frecency_db_path) }.map(|s| s.to_string());
-    let history_path = unsafe { optional_cstr(history_db_path) }.map(|s| s.to_string());
+    let frecency_path = unsafe { optional_cstr(opts.frecency_db_path) }.map(|s| s.to_string());
+    let history_path = unsafe { optional_cstr(opts.history_db_path) }.map(|s| s.to_string());
 
-    // Create shared state that background threads will write into.
     let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
     let query_tracker = SharedQueryTracker::default();
 
-    // Initialize frecency tracker if path is provided
     if let Some(ref frecency_path) = frecency_path {
         if let Some(parent) = PathBuf::from(frecency_path).parent() {
             let _ = std::fs::create_dir_all(parent);
@@ -226,13 +221,11 @@ pub unsafe extern "C" fn fff_create_instance2(
                 if let Err(e) = shared_frecency.init(tracker) {
                     return FffResult::err(&format!("Failed to acquire frecency lock: {}", e));
                 }
-                let _ = shared_frecency.spawn_gc(frecency_path.clone());
             }
             Err(e) => return FffResult::err(&format!("Failed to init frecency db: {}", e)),
         }
     }
 
-    // Initialize query tracker if path is provided
     if let Some(ref history_path) = history_path {
         if let Some(parent) = PathBuf::from(history_path).parent() {
             let _ = std::fs::create_dir_all(parent);
@@ -248,29 +241,31 @@ pub unsafe extern "C" fn fff_create_instance2(
         }
     }
 
-    let mode = if ai_mode {
+    let mode = if opts.ai_mode {
         FFFMode::Ai
     } else {
         FFFMode::Neovim
     };
 
     let cache_budget = fff::ContentCacheBudget::from_overrides(
-        cache_budget_max_files as usize,
-        cache_budget_max_bytes,
-        cache_budget_max_file_size,
+        opts.cache_budget_max_files as usize,
+        opts.cache_budget_max_bytes,
+        opts.cache_budget_max_file_size,
     );
 
-    // Initialize file picker (writes directly into shared_picker)
     if let Err(e) = FilePicker::new_with_shared_state(
         shared_picker.clone(),
         shared_frecency.clone(),
         fff::FilePickerOptions {
             base_path: base_path_str,
-            enable_mmap_cache,
-            enable_content_indexing,
-            watch,
+            enable_mmap_cache: opts.enable_mmap_cache,
+            enable_content_indexing: opts.enable_content_indexing,
+            watch: opts.watch,
             mode,
             cache_budget,
+            follow_symlinks: opts.version >= 2 && opts.follow_symlinks,
+            enable_fs_root_scanning: opts.enable_fs_root_scanning,
+            enable_home_dir_scanning: opts.enable_home_dir_scanning,
         },
     ) {
         return FffResult::err(&format!("Failed to init file picker: {}", e));
@@ -280,12 +275,24 @@ pub unsafe extern "C" fn fff_create_instance2(
         picker: shared_picker,
         frecency: shared_frecency,
         query_tracker,
+        watch_callback: std::sync::Arc::new(watch::WatchCallbackSlot::default()),
     });
 
     let fff_handle = Box::into_raw(instance) as *mut c_void;
     FffResult::ok_handle(fff_handle)
 }
 
+/// [`fff_create_instance_with`] adapter taking [`FffCreateOptions`] **by value**,
+/// for FFI libraries that pass native structs by value (e.g. Node's `ffi-rs`).
+///
+/// ## Safety
+/// All `*const c_char` fields inside `opts` must be valid null-terminated
+/// UTF-8 or NULL. The struct itself is consumed by value.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_create_instance_with_value(opts: FffCreateOptions) -> *mut FffResult {
+    unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) }
+}
+
 /// Destroy a file finder instance and free all its resources.
 ///
 /// ## Safety
@@ -298,10 +305,14 @@ pub unsafe extern "C" fn fff_destroy(fff_handle: *mut c_void) {
 
     let instance = unsafe { Box::from_raw(fff_handle as *mut FffInstance) };
 
+    // The C callback and user_data may be freed as soon as this returns.
+    instance.picker.shutdown_watches_and_wait();
+    instance.watch_callback.clear();
+
     if let Ok(mut guard) = instance.picker.write()
-        && let Some(mut picker) = guard.take()
+        && let Some(picker) = guard.take()
     {
-        picker.stop_background_monitor();
+        drop(picker);
     }
 
     if let Ok(mut guard) = instance.frecency.write() {
@@ -314,16 +325,9 @@ pub unsafe extern "C" fn fff_destroy(fff_handle: *mut c_void) {
 
 /// Perform fuzzy search on indexed files.
 ///
-/// # Parameters
-///
-/// * `fff_handle`              – instance from `fff_create_instance`
-/// * `query`                   – search query string
-/// * `current_file`            – path of the currently open file for deprioritization (NULL/empty to skip)
-/// * `max_threads`             – maximum worker threads (0 = auto-detect)
-/// * `page_index`              – pagination offset (0 = first page)
-/// * `page_size`               – results per page (0 = default 100)
-/// * `combo_boost_multiplier`  – score multiplier for combo matches (0 = default 100)
-/// * `min_combo_count`         – minimum combo count before boost applies (0 = default 3)
+/// `current_file` deprioritizes the currently open file (NULL/empty to skip).
+/// Zero picks the default: `max_threads` auto, `page_size` 100,
+/// `combo_boost_multiplier` 100, `min_combo_count` 3.
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -396,16 +400,72 @@ pub unsafe extern "C" fn fff_search(
     FffResult::ok_handle(search_result as *mut c_void)
 }
 
+/// Glob-only search: filter indexed files by a single glob pattern (passed
+/// through verbatim, no query parsing), rank by frecency, and paginate.
+///
+/// `current_file` deprioritizes the currently open file (NULL/empty to skip).
+/// Zero picks the default: `max_threads` auto, `page_size` 100.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` and `current_file` must be valid null-terminated UTF-8 strings or NULL.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_glob(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    current_file: *const c_char,
+    max_threads: u32,
+    page_index: u32,
+    page_size: u32,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+
+    let pattern_str = match unsafe { cstr_to_str(pattern) } {
+        Some(s) if !s.is_empty() => s,
+        _ => return FffResult::err("Pattern is null, empty, or invalid UTF-8"),
+    };
+
+    let current_file_str = unsafe { optional_cstr(current_file) };
+    let page_size = default_u32(page_size, 100) as usize;
+
+    let picker_guard = match inst.picker.read() {
+        Ok(g) => g,
+        Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
+    };
+
+    let picker = match picker_guard.as_ref() {
+        Some(p) => p,
+        None => {
+            return FffResult::err("File picker not initialized. Call fff_create_instance first.");
+        }
+    };
+
+    let results = picker.glob(
+        pattern_str,
+        FuzzySearchOptions {
+            max_threads: max_threads as usize,
+            current_file: current_file_str,
+            project_path: Some(picker.base_path()),
+            combo_boost_score_multiplier: 0,
+            min_combo_count: 0,
+            pagination: PaginationArgs {
+                offset: page_index as usize,
+                limit: page_size,
+            },
+        },
+    );
+
+    let search_result = FffSearchResult::from_core(&results, picker);
+    FffResult::ok_handle(search_result as *mut c_void)
+}
+
 /// Perform fuzzy search on indexed directories.
 ///
-/// # Parameters
-///
-/// * `fff_handle`   – instance from `fff_create_instance`
-/// * `query`        – search query string
-/// * `current_file` – path of the currently open file for distance scoring (NULL/empty to skip)
-/// * `max_threads`  – maximum worker threads (0 = auto-detect)
-/// * `page_index`   – pagination offset (0 = first page)
-/// * `page_size`    – results per page (0 = default 100)
+/// `current_file` is used for distance scoring (NULL/empty to skip).
+/// Zero picks the default: `max_threads` auto, `page_size` 100.
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -468,20 +528,8 @@ pub unsafe extern "C" fn fff_search_directories(
 
 /// Perform a mixed fuzzy search across both files and directories.
 ///
-/// Returns a single flat list where files and directories are interleaved
-/// by total score in descending order. Each item has an `item_type` field
-/// (0 = file, 1 = directory).
-///
-/// # Parameters
-///
-/// * `fff_handle`              – instance from `fff_create_instance`
-/// * `query`                   – search query string
-/// * `current_file`            – path of the currently open file (NULL/empty to skip)
-/// * `max_threads`             – maximum worker threads (0 = auto-detect)
-/// * `page_index`              – pagination offset (0 = first page)
-/// * `page_size`               – results per page (0 = default 100)
-/// * `combo_boost_multiplier`  – score multiplier for combo matches (0 = default 100)
-/// * `min_combo_count`         – minimum combo count before boost applies (0 = default 3)
+/// Returns one flat list interleaved by descending total score; each item's
+/// `item_type` is 0 = file, 1 = directory. Parameters as in [`fff_search`].
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -555,20 +603,11 @@ pub unsafe extern "C" fn fff_search_mixed(
 
 /// Perform content search (grep) across indexed files.
 ///
-/// # Parameters
-///
-/// * `fff_handle`            – instance from `fff_create_instance`
-/// * `query`                 – search query (supports constraint syntax like `*.rs pattern`)
-/// * `mode`                  – 0 = plain text (SIMD), 1 = regex, 2 = fuzzy
-/// * `max_file_size`         – skip files larger than this in bytes (0 = default 10 MB)
-/// * `max_matches_per_file`  – max matches per file (0 = unlimited)
-/// * `smart_case`            – case-insensitive when query is all lowercase
-/// * `file_offset`           – file-based pagination offset (0 = start)
-/// * `page_limit`            – max matches to return (0 = default 50)
-/// * `time_budget_ms`        – wall-clock budget in ms (0 = unlimited)
-/// * `before_context`        – context lines before each match
-/// * `after_context`         – context lines after each match
-/// * `classify_definitions`  – tag matches that are code definitions
+/// `query` supports constraint syntax like `*.rs pattern`; `mode` is
+/// 0 = plain text (SIMD), 1 = regex, 2 = fuzzy. Zero picks the default:
+/// `max_file_size` 10 MB, `page_limit` 50, `max_matches_per_file` and
+/// `time_budget_ms` unlimited. `smart_case` is case-insensitive for
+/// all-lowercase queries; `classify_definitions` tags code definitions.
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -637,25 +676,11 @@ pub unsafe extern "C" fn fff_live_grep(
     FffResult::ok_handle(grep_result as *mut c_void)
 }
 
-/// Perform multi-pattern OR search (Aho-Corasick) across indexed files.
+/// Multi-pattern OR search (SIMD Aho-Corasick): lines matching ANY pattern.
 ///
-/// Searches for lines matching ANY of the provided patterns using
-/// SIMD-accelerated multi-needle matching.
-///
-/// # Parameters
-///
-/// * `fff_handle`              – instance from `fff_create_instance`
-/// * `patterns_joined`         – patterns separated by `\n` (e.g. `"foo\nbar\nbaz"`)
-/// * `constraints`             – file filter like `"*.rs"` or `"/src/"` (NULL/empty to skip)
-/// * `max_file_size`           – skip files larger than this in bytes (0 = default 10 MB)
-/// * `max_matches_per_file`    – max matches per file (0 = unlimited)
-/// * `smart_case`              – case-insensitive when all patterns are lowercase
-/// * `file_offset`             – file-based pagination offset (0 = start)
-/// * `page_limit`              – max matches to return (0 = default 50)
-/// * `time_budget_ms`          – wall-clock budget in ms (0 = unlimited)
-/// * `before_context`          – context lines before each match
-/// * `after_context`           – context lines after each match
-/// * `classify_definitions`    – tag matches that are code definitions
+/// `patterns_joined` is `\n`-separated (e.g. `"foo\nbar"`); `constraints` is an
+/// optional file filter like `"*.rs"` or `"/src/"` (NULL/empty to skip).
+/// Remaining parameters as in [`fff_live_grep`].
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -704,19 +729,12 @@ pub unsafe extern "C" fn fff_multi_grep(
         }
     };
 
-    let is_ai = picker.mode().is_ai();
-
     // Parse constraints from the optional string (e.g. "*.rs /src/")
-    let parsed_constraints = constraints_str.map(|c| {
-        if is_ai {
-            fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse(c)
-        } else {
-            fff::grep::parse_grep_query(c)
-        }
-    });
+    let parsed_constraints = constraints_str
+        .map(|c| fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse_constraints(c));
 
     let constraint_refs: &[fff::Constraint<'_>] = match &parsed_constraints {
-        Some(q) => &q.constraints,
+        Some(constraints) => constraints,
         None => &[],
     };
 
@@ -777,10 +795,8 @@ pub unsafe extern "C" fn fff_is_scanning(fff_handle: *mut c_void) -> bool {
         .unwrap_or(false)
 }
 
-/// Get the base path of the file picker.
-///
-/// Returns an `FffResult` with a heap-allocated C string in the `handle`
-/// field. Free the string with `fff_free_string` after reading it.
+/// Get the picker's base path as a heap C string in `handle`;
+/// free it with `fff_free_string`.
 ///
 /// ## Safety
 /// `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -897,24 +913,25 @@ pub unsafe extern "C" fn fff_restart_index(
         Err(e) => return FffResult::err(&format!("Failed to canonicalize path: {}", e)),
     };
 
-    let mut guard = match inst.picker.write() {
+    let guard = match inst.picker.write() {
         Ok(g) => g,
         Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
     };
 
-    let (warmup_caches, content_indexing, watch, mode) = if let Some(mut picker) = guard.take() {
-        let warmup = picker.has_mmap_cache();
-        let enable_content_indexing = picker.has_content_indexing();
-        let watch = picker.has_watcher();
-        let mode = picker.mode();
-
-        picker.stop_background_monitor();
-
-        (warmup, enable_content_indexing, watch, mode)
-    } else {
-        // this is error state anyway
-        (false, true, true, FFFMode::default())
-    };
+    let (warmup_caches, content_indexing, watch, mode, fs_root, home_dir, follow_symlinks) =
+        if let Some(ref picker) = *guard {
+            (
+                picker.has_mmap_cache(),
+                picker.has_content_indexing(),
+                picker.has_watcher(),
+                picker.mode(),
+                picker.fs_root_scanning_enabled(),
+                picker.home_dir_scanning_enabled(),
+                picker.follows_symlinks(),
+            )
+        } else {
+            (false, true, true, FFFMode::default(), false, false, false)
+        };
 
     drop(guard);
 
@@ -928,6 +945,9 @@ pub unsafe extern "C" fn fff_restart_index(
             watch,
             mode,
             cache_budget: None,
+            follow_symlinks,
+            enable_fs_root_scanning: fs_root,
+            enable_home_dir_scanning: home_dir,
         },
     ) {
         Ok(()) => FffResult::ok_empty(),
@@ -1238,10 +1258,8 @@ pub unsafe extern "C" fn fff_health_check(
     }
 }
 
-/// Free a search result returned by `fff_search`.
-///
-/// This frees the `FffSearchResult` struct, its `items` and `scores` arrays,
-/// and all heap-allocated strings within each item and score.
+/// Free a search result returned by `fff_search`: the struct, its `items`
+/// and `scores` arrays, and every string within.
 ///
 /// ## Safety
 /// `result` must be a valid pointer previously returned via `FffResult.handle`
@@ -1271,10 +1289,8 @@ pub unsafe extern "C" fn fff_free_search_result(result: *mut FffSearchResult) {
     }
 }
 
-/// Get a pointer to the `index`-th `FffFileItem` in a search result.
-///
-/// Returns null if `result` is null or `index >= result->count`.
-/// The returned pointer is valid until the search result is freed.
+/// Pointer to the `index`-th `FffFileItem`; null if `result` is null or
+/// `index >= count`. Valid until the search result is freed.
 ///
 /// ## Safety
 /// `result` must be a valid `FffSearchResult` pointer from `fff_search`.
@@ -1293,10 +1309,8 @@ pub unsafe extern "C" fn fff_search_result_get_item(
     unsafe { result.items.add(index as usize) }
 }
 
-/// Get a pointer to the `index`-th `FffScore` in a search result.
-///
-/// Returns null if `result` is null or `index >= result->count`.
-/// The returned pointer is valid until the search result is freed.
+/// Pointer to the `index`-th `FffScore`; null if `result` is null or
+/// `index >= count`. Valid until the search result is freed.
 ///
 /// ## Safety
 /// `result` must be a valid `FffSearchResult` pointer from `fff_search`.
@@ -1315,10 +1329,8 @@ pub unsafe extern "C" fn fff_search_result_get_score(
     unsafe { result.scores.add(index as usize) }
 }
 
-/// Free a grep result returned by `fff_live_grep` or `fff_multi_grep`.
-///
-/// This frees the `FffGrepResult` struct, its `items` array, and all
-/// heap-allocated strings, match ranges, and context arrays within each match.
+/// Free a grep result returned by `fff_live_grep` or `fff_multi_grep`:
+/// the struct, its `items` array, and all strings/ranges/context within.
 ///
 /// ## Safety
 /// `result` must be a valid pointer previously returned via `FffResult.handle`
@@ -1345,10 +1357,8 @@ pub unsafe extern "C" fn fff_free_grep_result(result: *mut FffGrepResult) {
     }
 }
 
-/// Get a pointer to the `index`-th `FffGrepMatch` in a grep result.
-///
-/// Returns null if `result` is null or `index >= result->count`.
-/// The returned pointer is valid until the grep result is freed.
+/// Pointer to the `index`-th `FffGrepMatch`; null if `result` is null or
+/// `index >= count`. Valid until the grep result is freed.
 ///
 /// ## Safety
 /// `result` must be a valid `FffGrepResult` pointer from `fff_live_grep` or `fff_multi_grep`.
@@ -1379,10 +1389,8 @@ pub unsafe extern "C" fn fff_free_scan_progress(result: *mut FffScanProgress) {
     }
 }
 
-/// Offset a pointer by `byte_offset` bytes.
-///
-/// General-purpose utility for FFI consumers that need pointer arithmetic
-/// (e.g. iterating over arrays). Returns null if `base` is null.
+/// Offset a pointer by `byte_offset` bytes (FFI array iteration helper).
+/// Returns null if `base` is null.
 ///
 /// ## Safety
 /// The resulting pointer must be within the bounds of the original allocation.
@@ -1394,7 +1402,9 @@ pub unsafe extern "C" fn fff_ptr_offset(base: *const c_void, byte_offset: usize)
     unsafe { (base as *const u8).add(byte_offset) as *const c_void }
 }
 
-/// Free a result returned by any `fff_*` function.
+/// Free a result envelope returned by any `fff_*` function.
+/// **IMPORTANT:** the `handle` payload is NOT freed release it separately
+/// using handle specific cleaning methods (`fff_destroy`, `fff_free_search_result`, etc.).
 ///
 /// ## Safety
 /// `result_ptr` must be a valid pointer returned by a `fff_*` function.
@@ -1409,9 +1419,8 @@ pub unsafe extern "C" fn fff_free_result(result_ptr: *mut FffResult) {
         if !result.error.is_null() {
             drop(CString::from_raw(result.error));
         }
-        // Note: `handle` is NOT freed here — the caller must free it
-        // with the appropriate function (fff_destroy, fff_free_search_result,
-        // fff_free_grep_result, fff_free_string, fff_free_scan_progress, etc.).
+
+        // note: handle is not freed by design
     }
 }
 

crates/fff-c/src/watch.rs

@@ -0,0 +1,351 @@
+use std::ffi::{CString, c_char, c_void};
+use std::ptr;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use fff::{WatchEvent, WatchId, WatchOptions};
+
+use crate::ffi_types::FffResult;
+use crate::instance_ref;
+
+/// Current version of [`FffWatchOptions`].
+pub const FFF_WATCH_OPTIONS_VERSION: u32 = 1;
+
+/// Options for `fff_watch`. Versioned: new fields are only appended.
+#[repr(C)]
+pub struct FffWatchOptions {
+    /// Set to [`FFF_WATCH_OPTIONS_VERSION`] when allocating.
+    pub version: u32,
+    /// Per-subscription excludes (parcel-watcher style): entries with wildcards
+    /// are base-relative globs, entries without are path prefixes. NULL when
+    /// `ignore_count` is 0.
+    pub ignore: *const *const c_char,
+    pub ignore_count: u32,
+    // ----- new version 2+ fields go here, ALWAYS appended -----
+}
+
+/// A single watch event. `kind`: 0 = created, 1 = modified, 2 = removed,
+/// 3 = rescan (events were lost; re-stat what you care about).
+#[repr(C)]
+pub struct FffWatchEvent {
+    /// Absolute path (heap C string owned by the parent batch).
+    pub path: *mut c_char,
+    pub kind: u8,
+}
+
+/// A batch of watch events. Free with `fff_free_watch_events`.
+#[repr(C)]
+pub struct FffWatchEventBatch {
+    pub events: *mut FffWatchEvent,
+    pub count: u32,
+}
+
+/// Instance-wide callback invoked with `(watch_id, batch)` for every `fff_watch`
+/// subscription. The callee owns and frees `batch` via `fff_free_watch_events`.
+pub type FffWatchCallback =
+    unsafe extern "C" fn(watch_id: u64, batch: *mut FffWatchEventBatch, user_data: *mut c_void);
+
+fn batch_into_raw(events: &[WatchEvent]) -> *mut FffWatchEventBatch {
+    let items: Vec<FffWatchEvent> = events
+        .iter()
+        .map(|ev| FffWatchEvent {
+            path: CString::new(ev.path.to_string_lossy().as_bytes())
+                .unwrap_or_default()
+                .into_raw(),
+            kind: ev.kind as u8,
+        })
+        .collect();
+
+    let count = items.len() as u32;
+    let events_ptr = if items.is_empty() {
+        ptr::null_mut()
+    } else {
+        let mut boxed = items.into_boxed_slice();
+        let p = boxed.as_mut_ptr();
+        std::mem::forget(boxed);
+        p
+    };
+
+    Box::into_raw(Box::new(FffWatchEventBatch {
+        events: events_ptr,
+        count,
+    }))
+}
+
+unsafe fn watch_options_from_ffi(
+    opts: *const FffWatchOptions,
+) -> Result<WatchOptions, *mut FffResult> {
+    if opts.is_null() {
+        return Ok(WatchOptions::default());
+    }
+    let opts = unsafe { &*opts };
+    if opts.version == 0 || opts.version > FFF_WATCH_OPTIONS_VERSION {
+        return Err(FffResult::err(&format!(
+            "Unsupported FffWatchOptions version {} (library understands up to {})",
+            opts.version, FFF_WATCH_OPTIONS_VERSION
+        )));
+    }
+
+    let mut ignore = Vec::with_capacity(opts.ignore_count as usize);
+    if opts.ignore_count > 0 {
+        if opts.ignore.is_null() {
+            return Err(FffResult::err("ignore_count > 0 but ignore is NULL"));
+        }
+        for i in 0..opts.ignore_count as usize {
+            let entry = unsafe { *opts.ignore.add(i) };
+            match unsafe { crate::cstr_to_str(entry) } {
+                Some(s) if !s.is_empty() => ignore.push(s.to_string()),
+                Some(_) => {}
+                None => return Err(FffResult::err("ignore entry is NULL or invalid UTF-8")),
+            }
+        }
+    }
+
+    Ok(WatchOptions { ignore })
+}
+
+// The caller guarantees user_data is safe on the callback thread.
+struct UserData(*mut c_void);
+unsafe impl Send for UserData {}
+unsafe impl Sync for UserData {}
+
+// Shared so a closure surviving an unwatch race never dangles.
+#[derive(Default)]
+pub(crate) struct WatchCallbackSlot(Mutex<Option<(FffWatchCallback, UserData)>>);
+
+impl WatchCallbackSlot {
+    fn get(&self) -> Option<(FffWatchCallback, *mut c_void)> {
+        self.0
+            .lock()
+            .ok()
+            .and_then(|guard| guard.as_ref().map(|(cb, ud)| (*cb, ud.0)))
+    }
+
+    fn set(&self, callback: FffWatchCallback, user_data: *mut c_void) {
+        if let Ok(mut guard) = self.0.lock() {
+            *guard = Some((callback, UserData(user_data)));
+        }
+    }
+
+    pub(crate) fn clear(&self) {
+        if let Ok(mut guard) = self.0.lock() {
+            *guard = None;
+        }
+    }
+}
+
+/// Register the instance-wide watch callback used by all `fff_watch`
+/// subscriptions; call before the first `fff_watch`, calling again replaces it.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `callback` must remain callable until fff_unwatch called
+///   `fff_destroy(fff_handle)` returns.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_set_watch_callback(
+    fff_handle: *mut c_void,
+    callback: FffWatchCallback,
+    user_data: *mut c_void,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    inst.watch_callback.set(callback, user_data);
+    FffResult::ok_empty()
+}
+
+/// Subscribe to filesystem changes, delivered through the instance callback
+/// registered by `fff_set_watch_callback`.
+///
+/// Returns the watch id, pass it to `fff_unwatch` to stop.
+///
+/// `pattern` if non `NULL` can be wildcard pattern, absolute, or relative path
+/// that will be used to filter the events triggering exact subscription.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` must be NULL or valid null-terminated UTF-8.
+/// * `opts` must be NULL or a valid `FffWatchOptions` pointer.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    opts: *const FffWatchOptions,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    // NULL pattern = watch the entire indexed tree ("" in core).
+    let pattern_str = if pattern.is_null() {
+        ""
+    } else {
+        match unsafe { crate::cstr_to_str(pattern) } {
+            Some(s) => s,
+            None => return FffResult::err("Pattern is not valid UTF-8"),
+        }
+    };
+    let options = match unsafe { watch_options_from_ffi(opts) } {
+        Ok(o) => o,
+        Err(e) => return e,
+    };
+    if inst.watch_callback.get().is_none() {
+        return FffResult::err("No watch callback registered. Call fff_set_watch_callback first.");
+    }
+
+    let slot = Arc::clone(&inst.watch_callback);
+    let result = inst.picker.watch(pattern_str, options, move |id, events| {
+        if let Some((cb, user_data)) = slot.get() {
+            let batch = batch_into_raw(events);
+            unsafe { cb(id.0, batch, user_data) };
+        }
+    });
+
+    match result {
+        Ok(id) => FffResult::ok_int(id.0 as i64),
+        Err(e) => FffResult::err(&format!("Failed to subscribe: {}", e)),
+    }
+}
+
+/// [`fff_watch`] adapter with flattened options, for FFI libraries that cannot
+/// marshal pointer arrays inside structs (e.g. Node's `ffi-rs`).
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` must be NULL (watch everything) or valid null-terminated UTF-8.
+/// * `ignore` must be NULL or point to `ignore_count` valid C strings.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_args(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    ignore: *const *const c_char,
+    ignore_count: u32,
+) -> *mut FffResult {
+    let opts = FffWatchOptions {
+        version: FFF_WATCH_OPTIONS_VERSION,
+        ignore,
+        ignore_count,
+    };
+    unsafe { fff_watch(fff_handle, pattern, &opts) }
+}
+
+/// Remove a watch subscription. `int_value` = 1 if the id existed, 0 otherwise.
+///
+/// ## Safety
+/// `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_unwatch(fff_handle: *mut c_void, watch_id: u64) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    FffResult::ok_int(inst.picker.unwatch(WatchId(watch_id)) as i64)
+}
+
+/// Number of events in a batch, 0 if `batch` is null.
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_count(batch: *const FffWatchEventBatch) -> u32 {
+    if batch.is_null() {
+        return 0;
+    }
+    unsafe { (*batch).count }
+}
+
+/// Absolute path of event `index`, will be null when out of bounds
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_get_path(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> *const c_char {
+    match unsafe { watch_event_at(batch, index) } {
+        Some(ev) => ev.path,
+        None => ptr::null(),
+    }
+}
+
+/// Kind of event `index` (0 = created, 1 = modified, 2 = removed, 3 = rescan)
+/// 3 (rescan aka "re-stat something" kind) returned when OS based buffer
+/// has been overflown and some events might be loss. Paths will contain a list of
+/// directories that needs to be rescanned to ensure consistency.
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_get_kind(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> u8 {
+    match unsafe { watch_event_at(batch, index) } {
+        Some(ev) => ev.kind,
+        None => 3,
+    }
+}
+
+unsafe fn watch_event_at<'a>(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> Option<&'a FffWatchEvent> {
+    if batch.is_null() {
+        return None;
+    }
+    let batch = unsafe { &*batch };
+    if batch.events.is_null() || index >= batch.count {
+        return None;
+    }
+    Some(unsafe { &*batch.events.add(index as usize) })
+}
+
+/// Free a watch event batch delivered to the instance callback.
+///
+/// ## Safety
+/// `batch` must be a pointer produced by this library, or null (no-op).
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_free_watch_events(batch: *mut FffWatchEventBatch) {
+    if batch.is_null() {
+        return;
+    }
+    unsafe {
+        let batch = Box::from_raw(batch);
+        if !batch.events.is_null() {
+            let events =
+                Vec::from_raw_parts(batch.events, batch.count as usize, batch.count as usize);
+            for ev in events {
+                if !ev.path.is_null() {
+                    drop(CString::from_raw(ev.path));
+                }
+            }
+        }
+    }
+}
+
+// THESE TESTS MUST NEVER BE UPDATED, ONLY EXTENDED WITH NEW FIELDS —
+// bindings hardcode these offsets (ABI stability).
+#[cfg(test)]
+mod layout_tests {
+    use super::*;
+    use std::mem::{offset_of, size_of};
+
+    #[test]
+    #[cfg(target_pointer_width = "64")]
+    fn watch_ffi_layouts_are_stable_64bit() {
+        assert_eq!(size_of::<FffWatchOptions>(), 24);
+        assert_eq!(offset_of!(FffWatchOptions, version), 0);
+        assert_eq!(offset_of!(FffWatchOptions, ignore), 8);
+        assert_eq!(offset_of!(FffWatchOptions, ignore_count), 16);
+
+        assert_eq!(size_of::<FffWatchEvent>(), 16);
+        assert_eq!(offset_of!(FffWatchEvent, path), 0);
+        assert_eq!(offset_of!(FffWatchEvent, kind), 8);
+
+        assert_eq!(size_of::<FffWatchEventBatch>(), 16);
+        assert_eq!(offset_of!(FffWatchEventBatch, events), 0);
+        assert_eq!(offset_of!(FffWatchEventBatch, count), 8);
+    }
+}

crates/fff-c/tests/smoke.c

@@ -0,0 +1,275 @@
+/*
+ * Smoke test for libfff_c — the smallest possible end-to-end exercise of
+ * the public C API. We:
+ *
+ *   1. Create a picker with an `FffCreateOptions` populated via C99
+ *      designated initializers (the recommended idiom for direct C use).
+ *   2. Wait for the initial scan to complete.
+ *   3. Search for "smoke.c".
+ *   4. Fail unless this very file appears in the results.
+ *
+ * Build + run via `make test-c-smoke`. Override $(CC) to test other
+ * compilers.
+ */
+
+/* expose mkdtemp/usleep under -std=c99 on glibc; harmless on musl/darwin */
+#define _DEFAULT_SOURCE
+#define _BSD_SOURCE
+
+#include <fff.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+// simple mock function to make sure that both globbing patterns and dir based pattern work
+static int watch_glob_hits = 0;
+static int watch_dir_hits = 0;
+static int watch_all_hits = 0;
+static int watch_ignored_leaks = 0;
+static uint64_t watch_glob_id = 0;
+static uint64_t watch_dir_id = 0;
+static uint64_t watch_all_id = 0;
+
+static void on_watch_batch(uint64_t watch_id, struct FffWatchEventBatch *batch, void *user_data) {
+    (void)user_data;
+    /* route by id like real SDKs do; unknown ids are benign no-ops */
+    for (uint32_t i = 0; i < batch->count; i++) {
+        const char *path = batch->events[i].path;
+        if (!path) continue;
+        if (watch_id == watch_glob_id && strstr(path, "hello.txt")) {
+            watch_glob_hits++;
+        }
+        if (watch_id == watch_dir_id) {
+            if (strstr(path, "hello.txt")) watch_dir_hits++;
+            if (strstr(path, "noise.log")) watch_ignored_leaks++;
+        }
+        if (watch_id == watch_all_id && strstr(path, "hello.txt")) {
+            watch_all_hits++;
+        }
+    }
+
+    fff_free_watch_events(batch); // need to clean dynamic array of events
+}
+
+static int watch_smoke(void) {
+    char tmpl[] = "/tmp/fff-c-watch-XXXXXX";
+    char *dir = mkdtemp(tmpl);
+    if (!dir) {
+        fprintf(stderr, "watch_smoke: mkdtemp failed\n");
+        return 1;
+    }
+
+    struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){
+        .version = FFF_CREATE_OPTIONS_VERSION,
+        .base_path = dir,
+        .enable_mmap_cache = false,
+        .enable_content_indexing = false,
+        .watch = true,
+    });
+    if (!create_result->success) {
+        fprintf(stderr, "watch_smoke: create failed: %s\n",
+                create_result->error ? create_result->error : "?");
+        fff_free_result(create_result);
+        return 1;
+    }
+    void *picker = create_result->handle;
+    fff_free_result(create_result);
+
+    struct FffResult *r = fff_wait_for_scan(picker, 10000);
+    fff_free_result(r);
+    r = fff_wait_for_watcher(picker, 10000);
+    fff_free_result(r);
+    usleep(300 * 1000); /* let the FSEvents stream settle */
+
+    /* instance-wide callback, then two subscriptions routed by id */
+    r = fff_set_watch_callback(picker, on_watch_batch, NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: fff_set_watch_callback failed: %s\n",
+                r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    fff_free_result(r);
+
+    r = fff_watch(picker, "**/*.txt", NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: fff_watch failed: %s\n", r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_glob_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    /* whole-tree dir subscription with an ignore glob */
+    const char *ignores[] = {"*.log"};
+    r = fff_watch(picker, dir,
+                  &(struct FffWatchOptions){.version = FFF_WATCH_OPTIONS_VERSION,
+                                            .ignore = ignores,
+                                            .ignore_count = 1});
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: dir fff_watch failed: %s\n", r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_dir_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    /* NULL pattern subscribes to the entire indexed tree */
+    r = fff_watch(picker, NULL, NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: NULL-pattern fff_watch failed: %s\n",
+                r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_all_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    char file_path[512];
+    snprintf(file_path, sizeof(file_path), "%s/hello.txt", dir);
+    FILE *f = fopen(file_path, "w");
+    if (!f) {
+        fprintf(stderr, "watch_smoke: fopen failed\n");
+        fff_destroy(picker);
+        return 1;
+    }
+    fputs("hello watch\n", f);
+    fclose(f);
+
+    /* must be filtered out by the dir subscription's ignore glob */
+    char log_path[512];
+    snprintf(log_path, sizeof(log_path), "%s/noise.log", dir);
+    FILE *lf = fopen(log_path, "w");
+    if (lf) {
+        fputs("noise\n", lf);
+        fclose(lf);
+    }
+
+    for (int attempt = 0;
+         attempt < 100 && (watch_glob_hits == 0 || watch_dir_hits == 0 || watch_all_hits == 0);
+         attempt++) {
+        usleep(100 * 1000);
+    }
+
+    r = fff_unwatch(picker, watch_glob_id);
+    fff_free_result(r);
+    r = fff_unwatch(picker, watch_dir_id);
+    fff_free_result(r);
+    r = fff_unwatch(picker, watch_all_id);
+    fff_free_result(r);
+    /* unwatch of an unknown id reports 0, not an error */
+    r = fff_unwatch(picker, watch_dir_id);
+    int unwatch_idempotent = r->success && r->int_value == 0;
+    fff_free_result(r);
+
+    /* fff_destroy is the quiescence barrier: after it returns the callback
+     * will never run again and could be freed (ours is static). */
+    fff_destroy(picker);
+
+    if (watch_glob_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: glob subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_dir_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: dir subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_all_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: NULL-pattern subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_ignored_leaks > 0) {
+        fprintf(stderr, "watch_smoke FAIL: ignore glob leaked %d events\n", watch_ignored_leaks);
+        return 1;
+    }
+    if (!unwatch_idempotent) {
+        fprintf(stderr, "watch_smoke FAIL: repeated unwatch was not a no-op\n");
+        return 1;
+    }
+
+    fprintf(stderr, "watch_smoke PASS (glob=%d dir=%d all=%d)\n", watch_glob_hits, watch_dir_hits,
+            watch_all_hits);
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    const char *base_path = argc > 1 ? argv[1] : ".";
+
+    // make sure that FFF C api is designed more for FFI rather than for direct C usage (I'm sorry)
+    struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){
+        .version = FFF_CREATE_OPTIONS_VERSION,
+        .base_path = base_path,
+        .enable_mmap_cache = false,
+        .enable_content_indexing = false,
+        .watch = false,
+    });
+
+    if (!create_result->success) {
+        fprintf(stderr, "fff couldn't create instance: %s\n",
+                create_result->error ? create_result->error : "?");
+        fff_free_result(create_result);
+        return 1;
+    }
+
+    void *file_picker = create_result->handle;
+    fff_free_result(create_result); // safe to drop now: handle outlives the envelope
+
+    struct FffResult *scan_result = fff_wait_for_scan(file_picker, 5000);
+    if (!scan_result->success) {
+        fprintf(stderr, "wait_for_scan failed: %s\n",
+                scan_result->error ? scan_result->error : "?");
+        fff_free_result(scan_result);
+        fff_destroy(file_picker);
+        return 1;
+    }
+    // int_value: 1 = scan completed in time, 0 = timed out.
+    if (scan_result->int_value == 0) {
+        fprintf(stderr, "wait_for_scan: timed out before initial scan finished\n");
+        fff_free_result(scan_result);
+        fff_destroy(file_picker);
+        return 1;
+    }
+    fff_free_result(scan_result);
+
+    struct FffResult *res = fff_search(file_picker, "smkoe.c", "", 0, 0, 50, 0, 0);
+    if (!res->success) {
+        fprintf(stderr, "search failed: %s\n", res->error ? res->error : "?");
+        fff_free_result(res);
+        fff_destroy(file_picker);
+        return 1;
+    }
+
+    struct FffSearchResult *sr = (struct FffSearchResult *)res->handle;
+    uint32_t total = sr->count;
+    int found = 0;
+    for (uint32_t i = 0; i < sr->count; i++) {
+        const char *path = sr->items[i].relative_path;
+        if (path && strstr(path, "smoke.c")) {
+            found = 1;
+            fprintf(stderr, "found self: %s\n", path);
+            break;
+        }
+    }
+
+    fff_free_search_result(sr);
+    fff_free_result(res);
+    fff_destroy(file_picker);
+
+    if (!found) {
+        fprintf(stderr, "FAIL: smoke.c not in search results (count=%u)\n", total);
+        return 1;
+    }
+
+    if (watch_smoke() != 0) {
+        fprintf(stderr, "FAIL: watch test failed\n");
+        return 1;
+    }
+
+    fprintf(stderr, "PASS\n");
+    return 0;
+}

crates/fff-core/Cargo.toml

@@ -1,14 +1,17 @@
 [package]
 name = "fff-search"
-version = "0.7.1"
+version = "0.10.5"
 edition = "2024"
 license = "MIT"
 authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>"]
 description = "Faboulous & Fast File Finder - a fast and extremely correct file finder SDK with typo resistance, SIMD, prefiltering, and more"
 
+[lints]
+workspace = true
+
 [lib]
 path = "src/lib.rs"
-crate-type = ["rlib", "staticlib", "cdylib"]
+crate-type = ["rlib"]
 
 [[bench]]
 name = "parse_bench"
@@ -22,10 +25,29 @@ harness = false
 name = "memmem_bench"
 harness = false
 
+[[bench]]
+name = "glob_bench"
+harness = false
+required-features = ["zlob"]
+
+[[bench]]
+name = "grep_bench"
+harness = false
+
 [features]
-default = []
+# `ripgrep` is the pure-Rust walker/glob backend and is on by default so
+# consumers build without a Zig toolchain. CI/release opt into zlob via
+# `--no-default-features --features zlob`.
+default = ["ripgrep"]
 # Enable C FFI exports
 ffi = []
+# Count full rescans and their causes. Always on in debug builds; enable this
+# to keep the accounting in a release build (used by the rescan_probe binary).
+rescan-stats = []
+# Enables POC definition classification for grep result matched lines
+definitions = []
+# Pure-Rust filesystem walker + glob matcher (ignore + globset crates).
+ripgrep = ["dep:ignore", "dep:globset", "fff-query-parser/ripgrep"]
 # Call mi_collect(true) after large allocator churn (bigram build).
 # Requires mimalloc to be the global allocator (linked by fff-nvim).
 mimalloc-collect = ["dep:libmimalloc-sys"]
@@ -39,35 +61,27 @@ rayon = { workspace = true }
 smallvec = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
-fff-query-parser = { workspace = true }
-
-# External dependencies
-bindet = { workspace = true }
+fff-query-parser = { workspace = true , version = "0.9.0" }
 blake3 = { workspace = true }
-chrono = { workspace = true }
 dirs = { workspace = true }
 libc = "0.2"
 git2 = { workspace = true }
 glidesort = { workspace = true }
-globset = { workspace = true }
-fff-grep = { workspace = true }
+globset = { workspace = true, optional = true }
+fff-grep = { workspace = true , version = "0.9.0" }
 aho-corasick = "1"
 memchr = "2"
 heed = { workspace = true }
-ignore = { workspace = true }
+ignore = { workspace = true, optional = true }
 memmap2 = { workspace = true }
 neo_frizbee = { workspace = true }
 notify = { workspace = true }
 notify-debouncer-full = { workspace = true }
-once_cell = { workspace = true }
 parking_lot = { workspace = true }
 pathdiff = { workspace = true }
 regex = { workspace = true }
 regex-syntax = "0.8"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-smartstring = { version = "1.0.1", features = ["serde"] }
-toml = "0.8"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zlob = { workspace = true, optional = true }
@@ -77,9 +91,15 @@ mimalloc = { version = "0.1", optional = true, features = ["local_dynamic_tls"]
 [target.'cfg(windows)'.dependencies]
 dunce = { workspace = true }
 
+# signal-hook only compiles on unix; we wrap the SIGSEGV handler behind cfg(unix)
+[target.'cfg(unix)'.dependencies]
+signal-hook-registry = { workspace = true }
+
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
 ctor = "0.2"
 proptest = { version = "1", default-features = false, features = ["std", "fork"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tempfile = "3.8"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+

crates/fff-core/README.md

@@ -2,6 +2,9 @@
 
 fff is a file search toolkit. It is faster than ripgrep and fzf and designed for a long running applications like file editors, ai agents, or file exploerers.
 
+> [!Important performance information]  
+> For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine.
+
 ## Features
 
 - Fuzzy file name search 

crates/fff-core/benches/glob_bench.rs

@@ -0,0 +1,380 @@
+//! Compare three glob-matching strategies for `match_glob_pattern` in constraints.rs
+//! need to make sure that my zlob changes are not affecting fff's internal globbing
+use ahash::AHashSet;
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+use zlob::{ZlobFlags, ZlobPattern, zlob_match_paths, zlob_match_paths_indices};
+
+fn make_paths(n: usize) -> Vec<String> {
+    let exts = ["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"];
+    let dirs = [
+        "src/core",
+        "src/ui",
+        "crates/fff-core/src",
+        "lua/fff",
+        "tests/integration",
+        "vendor/lib",
+        "node_modules/foo/bar",
+        "docs/internal",
+    ];
+    let mut out = Vec::with_capacity(n);
+    for i in 0..n {
+        let dir = dirs[i % dirs.len()];
+        let ext = exts[i % exts.len()];
+        out.push(format!("{dir}/file_{i}.{ext}"));
+    }
+    out
+}
+
+fn current_impl(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(Some(matches)) = zlob_match_paths(pattern, paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    let matched_set: AHashSet<usize> = matches.iter().map(|s| s.as_ptr() as usize).collect();
+    paths
+        .iter()
+        .enumerate()
+        .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
+        .map(|(i, _)| i)
+        .collect()
+}
+
+fn indices_free_fn(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(hits) = zlob_match_paths_indices(pattern, paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    hits.to_iter().collect()
+}
+
+fn compiled_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(p) = ZlobPattern::compile(pattern, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    let Ok(hits) = p.match_indices(paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    hits.to_iter().collect()
+}
+
+fn bench_glob_strategies(c: &mut Criterion) {
+    let path_counts = [1_000usize, 10_000, 100_000];
+    let patterns: &[(&str, &str)] = &[
+        ("ext_rs", "**/*.rs"),
+        ("dir_glob", "src/**/*.{ts,lua}"),
+        ("literal_seg", "**/node_modules/**"),
+        ("brace_multi", "**/*.{rs,ts,lua,md}"),
+    ];
+
+    for &count in &path_counts {
+        let owned = make_paths(count);
+        let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+
+        let mut group = c.benchmark_group(format!("glob_{count}"));
+        group.sample_size(50);
+
+        for &(name, pat) in patterns {
+            let id_curr = BenchmarkId::new("current_ptr_trick", name);
+            group.bench_with_input(id_curr, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = current_impl(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+
+            let id_idx = BenchmarkId::new("match_indices_fn", name);
+            group.bench_with_input(id_idx, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = indices_free_fn(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+
+            let id_comp = BenchmarkId::new("compiled_pattern", name);
+            group.bench_with_input(id_comp, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = compiled_pattern(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+        }
+
+        group.finish();
+    }
+}
+
+/// Hot-loop: pattern compiled ONCE, matched many times against fresh path slices.
+/// Models a hypothetical change where we cache compiled patterns across calls.
+fn bench_compiled_reuse(c: &mut Criterion) {
+    let owned = make_paths(10_000);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let pat = "**/*.{rs,ts,lua,md}";
+
+    let mut group = c.benchmark_group("glob_reuse_10k");
+    group.sample_size(100);
+
+    group.bench_function("recompile_each_time", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let hits = p
+                .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED)
+                .unwrap();
+            black_box(hits.len());
+        });
+    });
+
+    let compiled = ZlobPattern::compile(pat, ZlobFlags::RECOMMENDED).unwrap();
+    group.bench_function("reuse_compiled", |b| {
+        b.iter(|| {
+            let hits = compiled
+                .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED)
+                .unwrap();
+            black_box(hits.len());
+        });
+    });
+
+    group.finish();
+}
+
+/// End-to-end: build the lookup AND iterate items checking membership, modeling the
+/// real call shape in `apply_constraints` (filter loop reads the result for every item).
+fn bench_full_pipeline(c: &mut Criterion) {
+    bench_full_pipeline_size(c, 100_000);
+    bench_full_pipeline_size(c, 500_000);
+}
+
+fn bench_full_pipeline_size(c: &mut Criterion, count: usize) {
+    let owned = make_paths(count);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let pat = "**/*.{rs,ts,lua,md}";
+
+    let mut group = c.benchmark_group(format!("glob_full_pipeline_{count}"));
+    group.sample_size(50);
+
+    // (A) current: indices -> AHashSet -> per-item set.contains
+    group.bench_function("indices_to_ahashset_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let set: AHashSet<usize> = hits.to_iter().collect();
+            let count = (0..paths.len()).filter(|i| set.contains(i)).count();
+            black_box(count);
+        });
+    });
+
+    // (B) indices -> Vec<bool> bitmap -> per-item array lookup
+    group.bench_function("indices_to_bitmap_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for i in hits.to_iter() {
+                mask[i] = true;
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    // (C) compiled pattern + per-item matches() inside the filter loop. No batch.
+    group.bench_function("compiled_per_item_matches", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let count = paths.iter().filter(|path| p.matches_default(path)).count();
+            black_box(count);
+        });
+    });
+
+    // (D) compiled pattern + chunked batch -> Vec<bool> bitmap. Best of both:
+    //     SIMD batch wins inside chunks, no global allocation pressure, O(1) lookup.
+    group.bench_function("compiled_chunked_to_bitmap", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for (chunk_idx, chunk) in paths.chunks(512).enumerate() {
+                let base = chunk_idx * 512;
+                let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap();
+                for i in hits.to_iter() {
+                    mask[base + i] = true;
+                }
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    // (E') indices -> bit-packed Vec<u64> -> per-item bit test
+    group.bench_function("indices_to_bitset_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let words = paths.len().div_ceil(64);
+            let mut bits = vec![0u64; words];
+            for i in hits.to_iter() {
+                bits[i >> 6] |= 1u64 << (i & 63);
+            }
+            let count = (0..paths.len())
+                .filter(|&i| (bits[i >> 6] >> (i & 63)) & 1 == 1)
+                .count();
+            black_box(count);
+        });
+    });
+
+    // (E) (D) but larger chunk
+    group.bench_function("compiled_chunked_4096_to_bitmap", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for (chunk_idx, chunk) in paths.chunks(4096).enumerate() {
+                let base = chunk_idx * 4096;
+                let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap();
+                for i in hits.to_iter() {
+                    mask[base + i] = true;
+                }
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+/// Mixed-constraint pipeline: glob + ext. Compare pre-pass batch (current) vs
+/// inline `ZlobPattern::matches` after the cheap ext check rejects items.
+///
+/// Variables: ext rejection rate. Extreme cases reveal where each strategy wins.
+fn bench_mixed_pipeline(c: &mut Criterion) {
+    let count = 100_000;
+    let owned = make_paths(count);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let glob_pat = "**/*.{rs,ts,lua,md}";
+
+    // 4 ext sets: from very selective (1/10 paths kept) to permissive (kept all).
+    let scenarios: &[(&str, &[&str])] = &[
+        ("ext_1of10", &["rs"]),
+        ("ext_4of10", &["rs", "ts", "lua", "md"]),
+        (
+            "ext_8of10",
+            &["rs", "ts", "lua", "md", "toml", "go", "py", "c"],
+        ),
+        (
+            "ext_all",
+            &["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"],
+        ),
+    ];
+
+    fn ext_match(name: &str, exts: &[&str]) -> bool {
+        exts.iter().any(|e| {
+            let bytes = name.as_bytes();
+            let elen = e.len();
+            bytes.len() > elen + 1
+                && bytes[bytes.len() - elen - 1] == b'.'
+                && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes())
+        })
+    }
+
+    let mut group = c.benchmark_group("glob_mixed_100k");
+    group.sample_size(50);
+
+    for &(name, exts) in scenarios {
+        // (A) PRE-PASS: build bitmap for ALL paths, then per-item ext-then-bitmap.
+        let id_pre = BenchmarkId::new("prepass_bitmap", name);
+        group.bench_with_input(id_pre, &exts, |b, &exts| {
+            b.iter(|| {
+                let hits =
+                    zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED)
+                        .unwrap();
+                let mut mask = vec![false; paths.len()];
+                for i in hits.to_iter() {
+                    mask[i] = true;
+                }
+                let count = paths
+                    .iter()
+                    .enumerate()
+                    .filter(|&(_, p)| ext_match(p, exts))
+                    .filter(|&(i, _)| mask[i])
+                    .count();
+                black_box(count);
+            });
+        });
+
+        // (B) INLINE: compile once, per-item ext check first, then matches() only on survivors.
+        let id_inline = BenchmarkId::new("inline_compiled", name);
+        group.bench_with_input(id_inline, &exts, |b, &exts| {
+            b.iter(|| {
+                let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap();
+                let count = paths
+                    .iter()
+                    .filter(|path| ext_match(path, exts))
+                    .filter(|path| p.matches_default(path))
+                    .count();
+                black_box(count);
+            });
+        });
+    }
+
+    group.finish();
+}
+
+/// Compare hand-rolled `file_has_extension` byte compare vs compiling extensions
+/// into a single brace glob `**/*.{rs,ts,lua,md}` and dispatching through zlob.
+/// Both share the same per-item "filter then count" shape.
+fn bench_extensions_vs_glob(c: &mut Criterion) {
+    let owned = make_paths(100_000);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let exts = ["rs", "ts", "lua", "md"];
+    let glob_pat = "**/*.{rs,ts,lua,md}";
+
+    fn ext_match(name: &str, exts: &[&str]) -> bool {
+        let bytes = name.as_bytes();
+        exts.iter().any(|e| {
+            let elen = e.len();
+            bytes.len() > elen + 1
+                && bytes[bytes.len() - elen - 1] == b'.'
+                && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes())
+        })
+    }
+
+    let mut group = c.benchmark_group("ext_vs_glob_100k");
+    group.sample_size(50);
+
+    group.bench_function("file_has_extension_loop", |b| {
+        b.iter(|| {
+            let count = paths.iter().filter(|p| ext_match(p, &exts)).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("compiled_brace_glob_inline", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap();
+            let count = paths.iter().filter(|path| p.matches_default(path)).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("brace_glob_prepass_bitmap", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED)
+                    .unwrap();
+            let mut mask = vec![false; paths.len()];
+            for i in hits.to_iter() {
+                mask[i] = true;
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_glob_strategies,
+    bench_compiled_reuse,
+    bench_full_pipeline,
+    bench_mixed_pipeline,
+    bench_extensions_vs_glob
+);
+criterion_main!(benches);

crates/fff-core/benches/grep_bench.rs

@@ -0,0 +1,105 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use fff_search::file_picker::{FilePicker, FilePickerOptions};
+use fff_search::{GrepMode, GrepSearchOptions, parse_grep_query};
+use std::io::Write;
+
+/// Synthetic repo: half the files contain the needle on every line (stresses
+/// the per-match find/highlight path), half are pure noise (stresses the
+/// whole-file prefilter path).
+fn setup_repo(dir: &std::path::Path) {
+    for i in 0..400 {
+        let mut f = std::fs::File::create(dir.join(format!("match_{i}.rs"))).unwrap();
+        for j in 0..100 {
+            writeln!(
+                f,
+                "fn handle_{j}() {{ let controller = Controller::new({j}); controller.run(); }}"
+            )
+            .unwrap();
+        }
+    }
+    for i in 0..400 {
+        let mut f = std::fs::File::create(dir.join(format!("noise_{i}.rs"))).unwrap();
+        for j in 0..100 {
+            writeln!(
+                f,
+                "fn compute_{j}() {{ let value = {j} * 42; process(value); }}"
+            )
+            .unwrap();
+        }
+    }
+}
+
+fn options(mode: GrepMode) -> GrepSearchOptions {
+    GrepSearchOptions {
+        // Force a full scan of every file so we measure matcher/sink work,
+        // not pagination early-exit.
+        page_limit: usize::MAX,
+        max_matches_per_file: 0,
+        mode,
+        ..Default::default()
+    }
+}
+
+fn bench_grep(c: &mut Criterion) {
+    let dir = tempfile::tempdir().unwrap();
+    setup_repo(dir.path());
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: dir.path().to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+    assert_eq!(picker.get_files().len(), 800);
+
+    let mut group = c.benchmark_group("grep_e2e");
+    group.sample_size(30);
+
+    // Case-sensitive, 40k matched lines: hottest find_at/highlight path
+    let query = parse_grep_query("Controller");
+    let opts = options(GrepMode::PlainText);
+    group.bench_function("plain_case_sensitive_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    // Case-insensitive (SIMD folding path), 120k matched spans
+    let query = parse_grep_query("controller");
+    group.bench_function("plain_case_insensitive_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    // No matches anywhere: whole-file prefilter dominates
+    let query = parse_grep_query("Qqzyx");
+    group.bench_function("plain_no_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 0);
+            std::hint::black_box(r.total_files_searched)
+        });
+    });
+
+    // Regex mode: must be unaffected by NeedleFinder changes
+    let query = parse_grep_query("Contr[a-z]+ller");
+    let regex_opts = options(GrepMode::Regex);
+    group.bench_function("regex_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &regex_opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_grep);
+criterion_main!(benches);

crates/fff-core/benches/memmem_bench.rs

@@ -1,5 +1,5 @@
 use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
-use fff_search::case_insensitive_memmem;
+use fff_search::simd_string_utils::memmem;
 use std::path::Path;
 
 /// Load real source files from the repository as benchmark haystacks.
@@ -41,7 +41,7 @@ fn load_real_files() -> Vec<(&'static str, Vec<u8>)> {
 }
 
 fn bench_memmem(c: &mut Criterion) {
-    let mut group = c.benchmark_group("case_insensitive_memmem");
+    let mut group = c.benchmark_group("simd_string_utils_memmem");
 
     let files = load_real_files();
     assert!(!files.is_empty(), "No source files found for benchmarking");
@@ -69,26 +69,10 @@ fn bench_memmem(c: &mut Criterion) {
             let id = format!("{file_label}/{needle_label}");
 
             group.bench_with_input(
-                BenchmarkId::new("packed_pair", &id),
+                BenchmarkId::new("find", &id),
                 &(haystack, &needle_lower),
                 |b, &(h, n)| {
-                    b.iter(|| black_box(case_insensitive_memmem::search_packed_pair(h, n)));
-                },
-            );
-
-            group.bench_with_input(
-                BenchmarkId::new("memchr2_search", &id),
-                &(haystack, &needle_lower),
-                |b, &(h, n)| {
-                    b.iter(|| black_box(case_insensitive_memmem::search(h, n)));
-                },
-            );
-
-            group.bench_with_input(
-                BenchmarkId::new("scalar_baseline", &id),
-                &(haystack, &needle_lower),
-                |b, &(h, n)| {
-                    b.iter(|| black_box(case_insensitive_memmem::search_scalar(h, n)));
+                    b.iter(|| black_box(memmem::find(h, n)));
                 },
             );
         }

crates/fff-core/build.rs

@@ -3,6 +3,15 @@ fn main() {
     // used by tests/fuzz_git_watcher_stress.rs
     println!("cargo::rustc-check-cfg=cfg(stress)");
 
+    // Full-rescan accounting. Debug builds get it for free; a release build has
+    // to opt in with `--features rescan-stats` (what the rescan_probe needs).
+    println!("cargo::rustc-check-cfg=cfg(rescan_stats)");
+    if std::env::var("DEBUG").is_ok_and(|debug| debug != "false")
+        || std::env::var("CARGO_FEATURE_RESCAN_STATS").is_ok()
+    {
+        println!("cargo::rustc-cfg=rescan_stats");
+    }
+
     // When the `zlob` feature is enabled (Zig-compiled C library):
     // On Windows MSVC, explicitly link the C runtime libraries.
     // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the
@@ -41,6 +50,5 @@ fn zig_available() -> bool {
         .stdout(std::process::Stdio::null())
         .stderr(std::process::Stdio::null())
         .status()
-        .map(|s| s.success())
-        .unwrap_or(false)
+        .is_ok()
 }

crates/fff-core/src/background_watcher.rs

@@ -1,886 +0,0 @@
-use crate::error::Error;
-use crate::file_picker::{FFFMode, FilePicker};
-use crate::git::GitStatusCache;
-use crate::shared::{SharedFilePicker, SharedFrecency};
-use crate::sort_buffer::sort_with_buffer;
-use git2::Repository;
-use notify::event::{AccessKind, AccessMode};
-use notify::{Config, EventKind, EventKindMask, RecursiveMode};
-use notify_debouncer_full::{DebounceEventResult, DebouncedEvent, NoCache, new_debouncer_opt};
-use parking_lot::Mutex;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::sync::mpsc;
-use std::time::Duration;
-use tracing::{Level, debug, error, info, warn};
-
-type Debouncer = notify_debouncer_full::Debouncer<notify::RecommendedWatcher, NoCache>;
-
-/// Owns the file-system watcher and guarantees that all background threads
-/// are fully joined before `stop()` / `Drop` returns.
-pub struct BackgroundWatcher {
-    debouncer: Arc<Mutex<Option<Debouncer>>>,
-    watch_tx: Option<mpsc::Sender<PathBuf>>,
-    owner_thread: Option<std::thread::JoinHandle<()>>,
-}
-
-const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(50);
-const MAX_PATHS_THRESHOLD: usize = 1024;
-/// On macOS, each `watch()` call creates a separate FSEventStream. When the
-/// number of directories exceeds this threshold we fall back to a single
-/// recursive watch to avoid exhausting the per-process stream limit.
-const MAX_MACOS_NONRECURSIVE_WATCHES: usize = 4096;
-/// Minimum seconds between frecency tracks of the same file in AI mode.
-/// Prevents score inflation from rapid burst edits by AI agents.
-const AI_MODE_COOLDOWN_SECS: u64 = 5 * 60;
-const MAX_OVERFLOW_FILES: usize = 1024;
-
-impl BackgroundWatcher {
-    pub fn new(
-        base_path: PathBuf,
-        git_workdir: Option<PathBuf>,
-        shared_picker: SharedFilePicker,
-        shared_frecency: SharedFrecency,
-        mode: FFFMode,
-    ) -> Result<Self, Error> {
-        info!(
-            "Initializing background watcher for path: {}, mode: {:?}",
-            base_path.display(),
-            mode,
-        );
-
-        // Refuse to watch the filesystem root or the user's home directory.
-        // These are prone to high-volume event churn (editor temp files,
-        // browser caches, log rotations) which inflates the overflow arena
-        // and, on macOS, can exhaust the per-process FSEvents stream limit.
-        if base_path.parent().is_none()
-            || Some(base_path.as_os_str()) == dirs::home_dir().as_ref().map(|p| p.as_os_str())
-        {
-            return Err(Error::FilesystemRoot(base_path));
-        }
-
-        // macOS: always use a single recursive FSEvent stream.
-        //
-        // Per-dir NonRecursive watches create one FSEvent stream per dir.
-        // The per-process FSEvent cap is lower than expected in practice
-        // (4096 per process, but FFF usually is running within code editors),
-        // and each failed `watch()` after the cap blocks ~40 ms on kernel retry.
-        // Yes we pay for filtering events on handler phase but it is usable
-        //
-        // macOS and Windows use a single recursive watch. FSEvents and
-        // ReadDirectoryChangesW both support true kernel-level recursion
-        // on one handle — per-dir NonRecursive watches burn streams/handles
-        // for no benefit and, on Windows, have been observed to silently
-        // drop Modify events for nested paths.
-        //
-        // Linux keeps the per-dir NonRecursive strategy: inotify has no
-        // kernel-level recursion, so Recursive here would still register
-        // one watch per subdir but without the ignored-dir filtering we
-        // get by iterating `picker.for_each_dir` ourselves.
-        let use_recursive = cfg!(any(target_os = "macos", target_os = "windows"));
-
-        let (watch_tx, watch_rx) = mpsc::channel::<PathBuf>();
-        let watch_tx_for_debouncer = watch_tx.clone();
-
-        let owner_weak_picker = shared_picker.weaken();
-        let owner_frecency = shared_frecency.clone();
-        let owner_git_workdir = git_workdir.clone();
-
-        let debouncer = Self::create_debouncer(
-            base_path,
-            git_workdir,
-            shared_picker,
-            shared_frecency,
-            mode,
-            use_recursive,
-            watch_tx_for_debouncer,
-        )?;
-
-        info!("Background file watcher initialized successfully");
-
-        // debouncer is shared with the owner thread, once it's dropped the thread is closed
-        let debouncer = Arc::new(Mutex::new(Some(debouncer)));
-        // Only the Linux per-dir-watch branch needs this clone; on other
-        // platforms the owner thread never touches the debouncer.
-        #[cfg(target_os = "linux")]
-        let owner_debouncer = Arc::clone(&debouncer);
-
-        let owner_thread = std::thread::Builder::new()
-            .name("fff-watcher-own".into())
-            .spawn(move || {
-                while let Ok(dir) = watch_rx.recv() {
-                    // if the picker is dropped we do need to exit the loop
-                    let Some(strong_picker) = owner_weak_picker.upgrade() else {
-                        break;
-                    };
-
-                    // Only inotify (Linux) has no kernel-level recursion, so
-                    // it's the only platform that needs a per-subdir watch to
-                    // be registered at runtime. macOS FSEvents and Windows
-                    // ReadDirectoryChangesW are already watching recursively
-                    // from the base path (see `create_debouncer`), and
-                    // registering a second overlapping stream there produces
-                    // duplicate/out-of-order events.
-                    #[cfg(target_os = "linux")]
-                    {
-                        // Register the new directory with the debouncer, then
-                        // drop the mutex BEFORE doing picker-side work — see
-                        // the comment on `BackgroundWatcher::stop` for the
-                        // lock-ordering rationale.
-                        let mut guard = owner_debouncer.lock();
-                        let Some(debouncer) = guard.as_mut() else {
-                            break;
-                        };
-
-                        if let Err(e) = debouncer.watch(&dir, RecursiveMode::NonRecursive) {
-                            warn!(
-                                ?e,
-                                dir = %dir.display(),
-                                "Failed to init watcher for new directory"
-                            );
-                        }
-                    }
-
-                    track_files_from_new_directories(
-                        &dir,
-                        &strong_picker,
-                        &owner_frecency,
-                        &owner_git_workdir,
-                    );
-
-                    // Transient strong ref drops here, back
-                    // to weak-only before the next `recv()`.
-                }
-
-                tracing::info!("Background watcher is stopped");
-            })
-            .expect("failed to spawn fff-watcher-owner thread");
-
-        Ok(Self {
-            debouncer,
-            watch_tx: Some(watch_tx),
-            owner_thread: Some(owner_thread),
-        })
-    }
-
-    fn create_debouncer(
-        base_path: PathBuf,
-        git_workdir: Option<PathBuf>,
-        shared_picker: SharedFilePicker,
-        shared_frecency: SharedFrecency,
-        mode: FFFMode,
-        use_recursive: bool,
-        watch_tx: mpsc::Sender<PathBuf>,
-    ) -> Result<Debouncer, Error> {
-        let config = Config::default()
-            // do not follow symlinks as then notifiers spawns a bunch of events for symlinked
-            // files that could be git ignored, we have to property differentiate those and if
-            // the file was edited through a
-            .with_follow_symlinks(false)
-            // only the actual modification events, ignore the open syscals that we can generate by
-            // our own grep calls and preview window rendering
-            .with_event_kinds(EventKindMask::CORE);
-
-        // `use_recursive` was decided by the caller from a cheap size hint,
-        // so the event-handler closure can capture it directly.
-        //
-        // The closure lives on the debouncer's internal event thread
-        // for as long as the debouncer exists — i.e. the full
-        // lifetime of `BackgroundWatcher`. Capturing a strong
-        // `SharedFilePicker` here would re-introduce the Arc cycle
-        // we just broke with `owner_picker`'s `downgrade()` above.
-        // Capture a weak handle instead and upgrade per-batch.
-        let git_workdir_for_handler = git_workdir.clone();
-        let shared_picker_for_watching = shared_picker.clone();
-        let event_picker = shared_picker.weaken();
-        let mut debouncer = new_debouncer_opt(
-            DEBOUNCE_TIMEOUT,
-            Some(DEBOUNCE_TIMEOUT / 2), // tick rate for the event span
-            {
-                move |result: DebounceEventResult| match result {
-                    Ok(events) => {
-                        // Upgrade just long enough to drive one
-                        // debounced batch. Failure means every
-                        // external `SharedFilePicker` has already
-                        // dropped and teardown is already underway.
-                        let Some(strong_picker) = event_picker.upgrade() else {
-                            return;
-                        };
-
-                        let new_dirs = handle_debounced_events(
-                            events,
-                            &git_workdir_for_handler,
-                            &strong_picker,
-                            &shared_frecency,
-                            mode,
-                        );
-
-                        // every new directory creates had to be reflected in the picker state
-                        for dir in new_dirs {
-                            if let Err(e) = watch_tx.send(dir) {
-                                warn!(?e, "Failed to send directory update error");
-                            }
-                        }
-                    }
-                    Err(errors) => {
-                        error!("File watcher errors: {:?}", errors);
-                    }
-                }
-            },
-            // There is an issue with recommended cache implementation on macos
-            // it keeps track of all the files added to the watcher which is not a problem
-            // for us because any rename to the file will anyway require the removing from the
-            // ordedred index and adding it back with the new name
-            NoCache::new(),
-            config,
-        )?;
-
-        // Watching strategy:
-        //
-        // For small-to-medium repos we watch each indexed directory individually
-        // (NonRecursive). This avoids receiving events for gitignored paths like
-        // node_modules/ and keeps the event volume low.
-        //
-        // On macOS, each `watch()` call creates a separate FSEventStream. Large
-        // repos (e.g. Chromium with 487K+ files) can have tens of thousands of
-        // directories, which exhausts the per-process FSEvents stream limit and
-        // causes "unable to start FSEvent stream" errors. When the directory
-        // count exceeds the threshold we fall back to a single Recursive watch
-        // on the base path. FSEvents handles this efficiently with one kernel
-        // stream for the entire subtree. Gitignored paths are already filtered
-        // in the event handler via `should_include_file()`.
-        //
-        // On Linux (inotify), RecursiveMode::Recursive creates one kernel watch
-        // per subdirectory *including* gitignored ones, wasting file descriptors.
-        // The per-directory NonRecursive approach is always used on Linux.
-        //
-        // New directories created at runtime are detected via Create events on
-        // the parent and dynamically added by the owner thread via watch_tx.
-
-        if use_recursive {
-            debouncer.watch(base_path.as_path(), RecursiveMode::Recursive)?;
-            info!(
-                "File watcher initialized with single recursive watch on {} \
-                 (exceeded threshold of {})",
-                base_path.display(),
-                MAX_MACOS_NONRECURSIVE_WATCHES,
-            );
-        } else {
-            debouncer.watch(base_path.as_path(), RecursiveMode::NonRecursive)?;
-
-            // Stream watch-dir registration directly under the picker
-            // read lock. Only Linux (inotify) reaches this branch —
-            // macOS always takes the recursive path above. `inotify`'s
-            // `inotify_add_watch()` is fast-fail: on ENOSPC it returns
-            // immediately, no kernel retry loop, so holding the read
-            // lock across the stream is O(ms) even for large repos.
-            //
-            // Abort the loop after a run of failures. Once ENOSPC hits,
-            // further calls won't succeed until the user raises
-            // `fs.inotify.max_user_watches`, so there's no value in
-            // continuing.
-            const MAX_CONSECUTIVE_WATCH_FAILURES: usize = 16;
-
-            let mut watched = 0usize;
-            let mut consecutive_failures = 0usize;
-            let mut aborted_early = false;
-
-            if let Some(guard) = shared_picker_for_watching.read().ok()
-                && let Some(picker) = guard.as_ref()
-            {
-                use std::ops::ControlFlow;
-                picker.for_each_dir(|dir| {
-                    match debouncer.watch(dir, RecursiveMode::NonRecursive) {
-                        Ok(()) => {
-                            watched += 1;
-                            consecutive_failures = 0;
-                            ControlFlow::Continue(())
-                        }
-                        Err(e) => {
-                            consecutive_failures += 1;
-                            if consecutive_failures <= 4 {
-                                warn!("Failed to watch directory {}: {}", dir.display(), e);
-                            }
-
-                            if consecutive_failures >= MAX_CONSECUTIVE_WATCH_FAILURES {
-                                warn!(
-                                    consecutive_failures,
-                                    watched,
-                                    "Aborting NonRecursive watch loop — per-process \
-                                 watch cap exhausted, further dirs would just burn \
-                                 kernel time for no coverage"
-                                );
-                                aborted_early = true;
-                                ControlFlow::Break(())
-                            } else {
-                                ControlFlow::Continue(())
-                            }
-                        }
-                    }
-                });
-            }
-
-            info!(
-                "File watcher initialized for {} directories (NonRecursive) under {} (aborted_early={})",
-                watched,
-                base_path.display(),
-                aborted_early,
-            );
-        }
-
-        // The .git directory is excluded from the file list but we still need
-        // to observe changes that affect git status (staging, unstaging,
-        // committing, branch switches, merges, etc.).
-        // When using recursive mode the base watch already covers .git/,
-        // but these targeted watches are cheap (at most 3 extra streams)
-        // and ensure we catch status changes even if the recursive backend
-        // coalesces or delays .git events.
-        watch_git_status_paths(&mut debouncer, git_workdir.as_ref());
-
-        Ok(debouncer)
-    }
-
-    /// Signal the watcher to shut down without blocking on its worker
-    /// threads. Safe to call from any context, including while holding
-    /// the [`SharedFilePicker`] write lock.
-    ///
-    /// Both the debouncer's internal event loop and our owner thread
-    /// may call `SharedFilePicker::write()` inside their handlers. A
-    /// blocking join here would deadlock against a caller that already
-    /// holds that lock (e.g. `stop_background_monitor` under a
-    /// `shared_picker.write()` guard). Instead we:
-    ///
-    ///   * drop the `watch_tx` Sender — the owner thread's
-    ///     `watch_rx.recv()` returns `Err` and the thread exits at
-    ///     its next `recv`.
-    ///   * call `debouncer.stop_nonblocking()` — signals the debouncer
-    ///     event loop to exit on its next tick and drops the watcher,
-    ///     closing the FSEvent / inotify / ReadDirectoryChangesW stream.
-    ///   * detach both `JoinHandle`s.
-    ///
-    /// In-flight handler invocations finish on their own (at most one
-    /// more batch) once the caller releases any locks they hold.
-    pub fn stop(&mut self) {
-        self.watch_tx.take();
-        if let Some(debouncer) = self.debouncer.lock().take() {
-            debouncer.stop_nonblocking();
-        }
-
-        self.owner_thread.take();
-
-        info!("Background file watcher stop signaled");
-    }
-
-    /// Queue a non-recursive watch registration on `dir`.
-    ///
-    /// The owner thread is always blocked on `watch_rx.recv()`, so
-    /// the `send()` here wakes it immediately via the channel's
-    /// condvar — no external unpark needed.
-    ///
-    /// Returns `false` once `stop()` has dropped our `Sender` — any
-    /// further request is silently discarded.
-    pub(crate) fn request_watch_dir(&self, dir: PathBuf) -> bool {
-        match self.watch_tx.as_ref() {
-            Some(tx) => tx.send(dir).is_ok(),
-            None => false,
-        }
-    }
-}
-
-impl Drop for BackgroundWatcher {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}
-
-#[tracing::instrument(name = "fs_events", skip(events, shared_picker, shared_frecency), level = Level::DEBUG)]
-fn handle_debounced_events(
-    events: Vec<DebouncedEvent>,
-    git_workdir: &Option<PathBuf>,
-    shared_picker: &SharedFilePicker,
-    shared_frecency: &SharedFrecency,
-    mode: FFFMode,
-) -> Vec<PathBuf> {
-    // this will be called very often, we have to minimiy the lock time for file picker
-    let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok());
-    let mut need_full_rescan = false;
-    let mut need_full_git_rescan = false;
-    let mut paths_to_remove = Vec::new();
-    let mut dirs_to_remove: Vec<PathBuf> = Vec::new();
-    let mut paths_to_add_or_modify = Vec::new();
-    let mut new_dirs_to_watch = Vec::new();
-    let mut affected_paths_count = 0usize;
-
-    for debounced_event in &events {
-        // It is very important to not react to the access errors because we inevitably
-        // gonna trigger the sync by our own preview or other unnecessary noise
-        if matches!(
-            debounced_event.event.kind,
-            EventKind::Access(
-                AccessKind::Read
-                    | AccessKind::Open(_)
-                    | AccessKind::Close(AccessMode::Read | AccessMode::Execute)
-            )
-        ) {
-            continue;
-        }
-
-        // When macOS FSEvents (or other backends) overflow their event buffer, the kernel
-        // drops individual events and emits a Rescan flag telling us to re-scan the subtree.
-        // Without handling this, modified source files can be silently missed.
-        if debounced_event.event.need_rescan() {
-            warn!(
-                "Received rescan event for paths {:?}, triggering full rescan",
-                debounced_event.event.paths
-            );
-            need_full_rescan = true;
-            break;
-        }
-
-        tracing::debug!(event = ?debounced_event.event, "Processing FS event");
-        for path in &debounced_event.event.paths {
-            if is_ignore_definition_path(path) {
-                info!(
-                    "Detected change in ignore definition file: {}",
-                    path.display()
-                );
-                need_full_rescan = true;
-                break;
-            }
-
-            if is_dotgit_change_affecting_status(path, &repo) {
-                need_full_git_rescan = true;
-            }
-
-            if is_git_file(path) {
-                continue;
-            }
-
-            // Use a combination of event kind and filesystem state to decide
-            // whether a path is an addition/modification or a removal.
-            //
-            // We cannot rely on `path.exists()` alone because:
-            //   - A freshly created file might not be visible yet (race).
-            //   - macOS FSEvents uses Modify(Name(Any)) for both rename-in
-            //     and rename-out, so we must stat the path to disambiguate.
-            //
-            // We cannot rely on event kind alone because:
-            //   - Remove events are not always emitted (macOS often sends
-            //     Modify(Name(Any)) instead of Remove).
-            let is_removal = matches!(debounced_event.event.kind, EventKind::Remove(_));
-            // Directory-level remove: macOS FSEvents delivers a single
-            // `Remove(Folder)` event for a whole directory tree (e.g.
-            // after `git reset --hard` wipes a dir full of staged-but-
-            // uncommitted files). Individual per-file Remove events for
-            // the children do *not* arrive. Treat the folder removal as
-            // "evict every indexed descendant".
-            let is_folder_removal = matches!(
-                debounced_event.event.kind,
-                EventKind::Remove(notify::event::RemoveKind::Folder)
-            );
-
-            if is_folder_removal {
-                dirs_to_remove.push(path.to_path_buf());
-            } else if is_removal || !path.exists() {
-                paths_to_remove.push(path.as_path());
-            } else if path.is_dir() {
-                // New directory — collect it so the caller can register a
-                // watcher. No filesystem scanning: files that arrive later
-                // will be handled by the newly registered watch.
-                if !is_path_ignored(path, &repo) {
-                    new_dirs_to_watch.push(path.to_path_buf());
-                }
-            } else {
-                // For additions/modifications, still filter gitignored files.
-                if should_include_file(path, &repo) {
-                    paths_to_add_or_modify.push(path.as_path());
-                }
-            }
-        }
-
-        affected_paths_count += debounced_event.event.paths.len();
-        if affected_paths_count > MAX_PATHS_THRESHOLD {
-            warn!(
-                "Too many affected paths ({}) in a single batch, triggering full rescan",
-                affected_paths_count
-            );
-
-            need_full_rescan = true;
-            break;
-        }
-
-        if need_full_rescan {
-            break;
-        }
-    }
-
-    if need_full_rescan {
-        info!(?affected_paths_count, "Triggering full rescan");
-        if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) {
-            error!("Failed to trigger full rescan: {:?}", e);
-        }
-        return Vec::new();
-    }
-
-    // It's important to get the allocated sort
-    sort_with_buffer(paths_to_add_or_modify.as_mut_slice(), |a, b| {
-        a.as_os_str().cmp(b.as_os_str())
-    });
-    paths_to_add_or_modify.dedup_by(|a, b| a.as_os_str().eq(b.as_os_str()));
-
-    info!(
-        "Event processing summary: {} to remove, {} dirs to remove, {} to add/modify, {} new dirs",
-        paths_to_remove.len(),
-        dirs_to_remove.len(),
-        paths_to_add_or_modify.len(),
-        new_dirs_to_watch.len()
-    );
-
-    // Apply file index updates (add/remove) unconditionally — these must
-    // happen even when there is no git repository.
-    let (files_to_update_git_status, overflow_count) = if !paths_to_remove.is_empty()
-        || !dirs_to_remove.is_empty()
-        || !paths_to_add_or_modify.is_empty()
-    {
-        debug!(
-            "Applying file index changes: {} to remove, {} dirs to remove, {} to add/modify",
-            paths_to_remove.len(),
-            dirs_to_remove.len(),
-            paths_to_add_or_modify.len(),
-        );
-
-        let apply_changes = |picker: &mut FilePicker| -> (Vec<PathBuf>, usize) {
-            // Remove whole directories first so any subsequent single-file
-            // remove event for a path that lived under them becomes a cheap
-            // no-op rather than a failed lookup.
-            for dir in &dirs_to_remove {
-                let count = picker.remove_all_files_in_dir(dir);
-                debug!("remove_all_files_in_dir({:?}) -> {} files", dir, count);
-            }
-
-            for path in &paths_to_remove {
-                let removed = picker.remove_file_by_path(path);
-                debug!("remove_file_by_path({:?}) -> {}", path, removed);
-            }
-
-            let mut files_to_update = Vec::with_capacity(paths_to_add_or_modify.len());
-            for path in &paths_to_add_or_modify {
-                let added = picker.on_create_or_modify(path).is_some();
-                if added {
-                    debug!("on_create_or_modify({:?}) -> Some", path);
-                    files_to_update.push(path.to_path_buf());
-                } else {
-                    error!("on_create_or_modify({:?}) -> None (file not added!)", path);
-                }
-            }
-            let overflow_count = picker.get_overflow_files().len();
-            info!(
-                "apply_changes complete: {} files to update git status, overflow={}",
-                files_to_update.len(),
-                overflow_count,
-            );
-            (files_to_update, overflow_count)
-        };
-
-        let Ok(mut guard) = shared_picker.write() else {
-            error!("Failed to acquire file picker write lock");
-            return new_dirs_to_watch;
-        };
-        let Some(ref mut picker) = *guard else {
-            error!("File picker not initialized");
-            return new_dirs_to_watch;
-        };
-        apply_changes(picker)
-    } else {
-        debug!("No file index changes to apply");
-        (Vec::new(), 0)
-    };
-
-    // The overflow arena grows monotonically as new files are created — a
-    // file's chunks are added on creation but never reclaimed on removal.
-    // On directories with high churn (e.g. `$HOME` with editor temp files,
-    // browser caches) this inflates RSS unboundedly. Once overflow exceeds
-    // the threshold, fall back to a full rescan: that replaces `sync_data`
-    // and drops the builder arena, which is the only path that reclaims it.
-    if overflow_count > MAX_OVERFLOW_FILES {
-        warn!(
-            ?overflow_count,
-            "Overflow count exceeded the threshold, triggering full rescan.",
-        );
-        if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) {
-            error!("Failed to trigger full rescan: {:?}", e);
-        }
-        return new_dirs_to_watch;
-    }
-
-    // AI mode: auto-track frecency for all modified/created files.
-    // Uses a 5-minute cooldown per file to prevent score inflation from rapid
-    // burst edits (AI agents often edit the same file many times in minutes).
-    // This runs after apply_changes so the picker write lock is released.
-    if mode.is_ai() && !paths_to_add_or_modify.is_empty() {
-        let mut tracked_count = 0usize;
-        if let Ok(frecency_guard) = shared_frecency.read()
-            && let Some(ref frecency) = *frecency_guard
-        {
-            for path in &paths_to_add_or_modify {
-                // Skip if this file was tracked less than 5 minutes ago
-                let should_track = match frecency.seconds_since_last_access(path) {
-                    Ok(Some(secs)) => secs >= AI_MODE_COOLDOWN_SECS,
-                    Ok(None) => true, // Never tracked before
-                    Err(_) => true,   // DB error, track anyway
-                };
-                if !should_track {
-                    continue;
-                }
-
-                if let Err(e) = frecency.track_access(path) {
-                    error!("Failed to track frecency for {:?}: {:?}", path, e);
-                } else {
-                    tracked_count += 1;
-                }
-            }
-            if tracked_count > 0 {
-                info!("AI mode: tracked frecency for {} files", tracked_count);
-            }
-        }
-
-        // Update in-memory frecency scores for tracked files
-        if tracked_count > 0
-            && let Ok(mut picker_guard) = shared_picker.write()
-            && let Some(ref mut picker) = *picker_guard
-            && let Ok(frecency_guard) = shared_frecency.read()
-            && let Some(ref frecency) = *frecency_guard
-        {
-            for path in &paths_to_add_or_modify {
-                let _ = picker.update_single_file_frecency(path, frecency);
-            }
-        }
-    }
-
-    // Git status updates require a repository.
-    let Some(repo) = repo.as_ref() else {
-        debug!("No git repo available, skipping git status updates");
-        return new_dirs_to_watch;
-    };
-
-    if need_full_git_rescan {
-        info!("Triggering full git rescan");
-
-        if let Err(e) = shared_picker.refresh_git_status(shared_frecency) {
-            error!("Failed to refresh git status: {:?}", e);
-        }
-        // IMPORTANT: do NOT return here. When a batch contains both
-        // `.git/index` events (e.g. from `git add`) AND worktree-file
-        // Modify events (e.g. a subsequent edit to the same file),
-        // `refresh_git_status` might run while libgit2 sees an
-        // intermediate state — lock-wait mitigates this but can't fully
-        // eliminate it, and refresh doesn't always observe the final
-        // worktree contents if the edit event landed just before the
-        // batch flushed. Re-running the per-path query for explicitly
-        // changed files overrides any stale bits from refresh with an
-        // authoritative per-file status read.
-    }
-
-    if !files_to_update_git_status.is_empty() {
-        info!(
-            "Fetching git status for {} files",
-            files_to_update_git_status.len()
-        );
-
-        let status = match GitStatusCache::git_status_for_paths(repo, &files_to_update_git_status) {
-            Ok(status) => status,
-            Err(e) => {
-                tracing::error!(?e, "Failed to query git status");
-                return new_dirs_to_watch;
-            }
-        };
-
-        if let Ok(mut guard) = shared_picker.write()
-            && let Some(ref mut picker) = *guard
-        {
-            if let Err(e) = picker.update_git_statuses(status, shared_frecency) {
-                error!("Failed to update git statuses: {:?}", e);
-            } else {
-                info!("Successfully updated git statuses in picker");
-            }
-        } else {
-            error!("Failed to acquire picker lock for git status update");
-        }
-    }
-
-    new_dirs_to_watch
-}
-
-/// After registering a watch on a newly created directory, list its
-/// immediate children and add any files to the picker.
-fn track_files_from_new_directories(
-    dir: &Path,
-    shared_picker: &SharedFilePicker,
-    shared_frecency: &SharedFrecency,
-    git_workdir: &Option<PathBuf>,
-) {
-    let Ok(entries) = std::fs::read_dir(dir) else {
-        return;
-    };
-
-    let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok());
-    let mut files_to_add = Vec::new();
-
-    for entry in entries.flatten() {
-        if entry.file_type().is_ok_and(|ft| ft.is_file()) {
-            let path = entry.path();
-            if should_include_file(&path, &repo) {
-                files_to_add.push(path);
-            }
-        }
-    }
-
-    if files_to_add.is_empty() {
-        return;
-    }
-
-    // brief read lock
-    {
-        let Ok(mut guard) = shared_picker.write() else {
-            return;
-        };
-
-        let Some(ref mut picker) = *guard else {
-            return;
-        };
-
-        for path in &files_to_add {
-            picker.on_create_or_modify(path);
-        }
-    }
-
-    if let Some(repo) = repo.as_ref() {
-        let status = match GitStatusCache::git_status_for_paths(repo, &files_to_add) {
-            Ok(status) => status,
-            Err(e) => {
-                tracing::error!(?e, "inject_existing_files: git status query failed");
-                return;
-            }
-        };
-
-        if let Ok(mut guard) = shared_picker.write()
-            && let Some(ref mut picker) = *guard
-            && let Err(e) = picker.update_git_statuses(status, shared_frecency)
-        {
-            error!("inject_existing_files: failed to update git statuses: {e:?}");
-        }
-    }
-
-    debug!(
-        "Injected {} existing files from new directory {}",
-        files_to_add.len(),
-        dir.display(),
-    );
-}
-
-fn should_include_file(path: &Path, repo: &Option<Repository>) -> bool {
-    // Directories are not indexed — only regular files (and symlinks to files).
-    if path.is_dir() {
-        return false;
-    }
-
-    match repo.as_ref() {
-        Some(repo) => repo.is_path_ignored(path) != Ok(true),
-        None => {
-            // No git repo — apply basic sanity filters.
-            // Hidden directories are skipped by the watcher setup (hidden(true)),
-            // but events can still arrive for files in known non-code directories.
-            !is_non_code_directory(path)
-        }
-    }
-}
-
-fn is_non_code_directory(path: &Path) -> bool {
-    crate::ignore::is_non_code_directory(path)
-}
-
-#[inline]
-fn is_path_ignored(path: &Path, repo: &Option<Repository>) -> bool {
-    match repo.as_ref() {
-        Some(repo) => repo.is_path_ignored(path) == Ok(true),
-        None => is_non_code_directory(path),
-    }
-}
-
-#[inline]
-pub(crate) fn is_git_file(path: &Path) -> bool {
-    path.components()
-        .any(|component| component.as_os_str() == ".git")
-}
-
-fn is_dotgit_change_affecting_status(changed: &Path, repo: &Option<Repository>) -> bool {
-    let Some(repo) = repo.as_ref() else {
-        return false;
-    };
-
-    let git_dir = repo.path();
-
-    if let Ok(path_in_git_dir) = changed.strip_prefix(git_dir) {
-        // Only react to changes that rewrite the worktree state: commits,
-        // staging, checkouts, merges, conflict resolution. Ref-only updates
-        // under refs/ (fetch, push, tag writes, pack-refs) do not change
-        // which files are modified/untracked, so we deliberately skip them —
-        // watching refs/ recursively would cost one inotify watch per ref
-        // namespace on repos with many branches/remotes.
-        if path_in_git_dir == Path::new("index") || path_in_git_dir == Path::new("index.lock") {
-            return true;
-        }
-        if path_in_git_dir == Path::new("HEAD") {
-            return true;
-        }
-        if path_in_git_dir == Path::new("info/exclude")
-            || path_in_git_dir == Path::new("info/sparse-checkout")
-        {
-            return true;
-        }
-
-        if let Some(fname) = path_in_git_dir.file_name().and_then(|f| f.to_str())
-            && matches!(fname, "MERGE_HEAD" | "CHERRY_PICK_HEAD" | "REVERT_HEAD")
-        {
-            return true;
-        }
-    }
-
-    false
-}
-
-fn is_ignore_definition_path(path: &Path) -> bool {
-    matches!(
-        path.file_name().and_then(|f| f.to_str()),
-        Some(".ignore") | Some(".gitignore")
-    )
-}
-
-fn watch_git_status_paths(debouncer: &mut Debouncer, git_workdir: Option<&PathBuf>) {
-    let Some(workdir) = git_workdir else {
-        return;
-    };
-
-    let git_dir = workdir.join(".git");
-    if !git_dir.is_dir() {
-        return;
-    }
-
-    // Watch .git/ non-recursively to catch top-level files:
-    // index, index.lock, HEAD, MERGE_HEAD, CHERRY_PICK_HEAD, REVERT_HEAD.
-    // We intentionally do NOT watch refs/ — individual ref updates don't
-    // affect worktree status, and a recursive watch there blows up inotify
-    // watch counts on repos with many branches/remotes/tags.
-    if let Err(e) = debouncer.watch(&git_dir, RecursiveMode::NonRecursive) {
-        warn!("Failed to watch .git directory: {}", e);
-        return;
-    }
-
-    // Watch info/ non-recursively for exclude and sparse-checkout
-    let info_dir = git_dir.join("info");
-    if info_dir.is_dir()
-        && let Err(e) = debouncer.watch(&info_dir, RecursiveMode::NonRecursive)
-    {
-        warn!("Failed to watch .git/info: {}", e);
-    }
-}

crates/fff-core/src/case_insensitive_memmem.rs

@@ -1,662 +0,0 @@
-//! SIMD-accelerated case-insensitive substring search.
-//!
-//! Implementations (fastest → simplest):
-//! - `search_packed_pair`: AVX2 packed-pair scan (two rare bytes at known offsets)
-//! - `search`:             memchr2 first-byte scan + verify
-//!
-//! The packed-pair approach mirrors what `memchr::memmem` does internally for
-//! case-sensitive search — pick two rare bytes from the needle, SIMD-scan for
-//! both simultaneously, verify candidates.  This gives quadratic selectivity
-//! over the single-byte memchr2 approach.
-
-// this is stolen from the memchr2 crate
-const BYTE_FREQUENCIES: [u8; 256] = [
-    55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00
-    42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10
-    255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20
-    208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30
-    120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185,
-    167, // 0x40 A-O
-    186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114,
-    223, // 0x50 P-_
-    151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246,
-    244, // 0x60 a-o
-    231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127,
-    27, // 0x70 p-DEL
-    212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80
-    207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90
-    118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0
-    166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0
-];
-
-#[inline]
-fn ascii_fold_byte(b: u8) -> u8 {
-    if b.is_ascii_uppercase() { b | 0x20 } else { b }
-}
-
-/// Toggle ASCII letter case by flipping bit 5.
-/// `'n' → 'N'`, `'N' → 'n'`.
-#[inline]
-fn ascii_swap_case(b: u8) -> u8 {
-    b ^ 0x20
-}
-
-/// Effective frequency rank for a case-insensitive byte position.
-/// Takes the max of lower/upper ranks because we must scan for both.
-#[inline]
-fn case_insensitive_rank(lower: u8) -> u8 {
-    if lower.is_ascii_lowercase() {
-        let upper = ascii_swap_case(lower);
-        BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[upper as usize])
-    } else {
-        BYTE_FREQUENCIES[lower as usize]
-    }
-}
-
-/// Pick two needle positions with the rarest bytes (case-insensitive).
-/// Returns (index1, index2) where index1 <= index2.
-fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) {
-    debug_assert!(needle_lower.len() >= 2);
-
-    let mut best1 = (u8::MAX, 0usize); // (rank, position)
-    let mut best2 = (u8::MAX, 1usize);
-
-    for (i, &b) in needle_lower.iter().enumerate() {
-        let r = case_insensitive_rank(b);
-        if r < best1.0 {
-            best2 = best1;
-            best1 = (r, i);
-        } else if r < best2.0 && i != best1.1 {
-            best2 = (r, i);
-        }
-    }
-
-    let i1 = best1.1.min(best2.1);
-    let i2 = best1.1.max(best2.1);
-    (i1, i2)
-}
-
-#[inline]
-fn verify_scalar(h: *const u8, needle_lower: &[u8]) -> bool {
-    for (i, _) in needle_lower.iter().enumerate() {
-        if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] {
-            return false;
-        }
-    }
-    true
-}
-
-/// AVX2 case-insensitive verify: checks whether `needle_lower` matches
-/// the haystack bytes starting at `h`, treating ASCII uppercase as lowercase.
-///
-/// Processes 32 bytes at a time using a SIMD trick: AVX2 only has a
-/// **signed** byte compare (`cmpgt`), but we need an **unsigned** range
-/// check (`'A' <= byte <= 'Z'`).  The trick is to XOR every byte with
-/// `0x80`, which maps the unsigned range `[0, 255]` into the signed range
-/// `[-128, 127]` while preserving order.  After the flip, signed `cmpgt`
-/// gives correct unsigned comparisons.
-///
-/// Once we know which bytes are uppercase, we set bit 5 (`0x20`) on them
-/// — this converts `'A'..'Z'` to `'a'..'z'` — then compare against the
-/// pre-lowered needle.
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn verify_avx2(h: *const u8, needle_lower: &[u8]) -> bool {
-    use core::arch::x86_64::*;
-
-    let len = needle_lower.len();
-    let mut i = 0usize;
-
-    // Broadcast constants used every iteration:
-    //
-    //   flip      = 0x80 in every lane — XOR converts unsigned→signed domain
-    //   a_minus_1 = ('A' - 1) ^ 0x80  — lower bound for the range check (signed)
-    //   z_plus_1  = ('Z' + 1) ^ 0x80  — upper bound for the range check (signed)
-    //   bit20     = 0x20 in every lane — OR this onto uppercase bytes to lowercase them
-    let flip = _mm256_set1_epi8(0x80u8 as i8);
-    let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8);
-    let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8);
-    let bit20 = _mm256_set1_epi8(0x20u8 as i8);
-
-    while i + 32 <= len {
-        // Load 32 bytes from the haystack candidate position.
-        let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) };
-        // Load 32 bytes from the pre-lowercased needle.
-        let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) };
-
-        // Flip into signed domain: x = hv ^ 0x80.
-        // After this, unsigned ordering is preserved under signed compare.
-        let x = _mm256_xor_si256(hv, flip);
-
-        // ge_a[lane] = 0xFF if x[lane] > a_minus_1, i.e. hv[lane] >= 'A' (unsigned).
-        let ge_a = _mm256_cmpgt_epi8(x, a_minus_1);
-        // le_z[lane] = 0xFF if z_plus_1 > x[lane], i.e. hv[lane] <= 'Z' (unsigned).
-        let le_z = _mm256_cmpgt_epi8(z_plus_1, x);
-        // upper[lane] = 0xFF only for bytes in the range 'A'..='Z'.
-        let upper = _mm256_and_si256(ge_a, le_z);
-
-        // Case-fold: set bit 5 on uppercase bytes → converts 'A'..'Z' to 'a'..'z'.
-        // Non-letter bytes are untouched because their `upper` lane is 0x00.
-        let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20));
-
-        // Compare the folded haystack against the lowercase needle.
-        let eq = _mm256_cmpeq_epi8(folded, nv);
-        // movemask extracts the high bit of each lane into a 32-bit mask.
-        // All-equal → all high bits set → mask == 0xFFFFFFFF == -1i32.
-        if _mm256_movemask_epi8(eq) != -1i32 {
-            return false;
-        }
-
-        i += 32;
-    }
-
-    // Scalar tail: handle remaining bytes that don't fill a full 32-byte vector.
-    while i < len {
-        if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] {
-            return false;
-        }
-        i += 1;
-    }
-    true
-}
-
-// ======== NEON + dotprod (aarch64) ===========================================
-
-/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF).
-/// Bit *i* of the result corresponds to byte *i* of the input vector.
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-#[inline]
-unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 {
-    use core::arch::aarch64::*;
-
-    // AND each byte with its bit-position mask, then horizontally sum each half.
-    // Max possible sum per half = 1+2+4+8+16+32+64+128 = 255, fits in u8.
-    static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
-    let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) };
-    let masked = vandq_u8(v, bit_mask);
-    let lo = vaddv_u8(vget_low_u8(masked));
-    let hi = vaddv_u8(vget_high_u8(masked));
-    (lo as u16) | ((hi as u16) << 8)
-}
-
-/// NEON + dotprod case-insensitive verify.
-///
-/// Uses unsigned range checks (NEON has `vcge`/`vcle` for unsigned bytes
-/// no XOR-0x80 trick needed unlike AVX2) to detect uppercase ASCII, folds
-/// to lowercase, then checks equality via UDOT: XOR the folded haystack
-/// with the pre-lowered needle and dot-product the difference with itself.
-/// Any non-zero byte produces a non-zero u32 lane.
-///
-/// The UDOT instruction is emitted via inline asm because the `vdotq_u32`
-/// intrinsic is still behind an unstable feature gate on stable Rust.
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon,dotprod")]
-unsafe fn verify_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool {
-    use core::arch::aarch64::*;
-
-    let len = needle_lower.len();
-    let mut i = 0usize;
-
-    let a_val = vdupq_n_u8(b'A');
-    let z_val = vdupq_n_u8(b'Z');
-    let bit20 = vdupq_n_u8(0x20);
-
-    while i + 16 <= len {
-        let hv = unsafe { vld1q_u8(h.add(i)) };
-        let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) };
-
-        // Unsigned range check: 'A' <= byte <= 'Z'
-        let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val));
-        // Case-fold: set bit 5 on uppercase bytes → 'A'..'Z' → 'a'..'z'
-        let folded = vorrq_u8(hv, vandq_u8(upper, bit20));
-
-        // XOR with needle — all-zero iff every byte matches.
-        let xored = veorq_u8(folded, nv);
-
-        // UDOT: dot(xored, xored) sums squares of 4 consecutive byte
-        // differences into each of the 4 u32 lanes (accumulates into zero).
-        // Any non-zero byte produces a positive u32 contribution.
-        let dots: uint32x4_t;
-        let zero = vdupq_n_u32(0);
-        unsafe {
-            core::arch::asm!(
-                "udot {d:v}.4s, {a:v}.16b, {b:v}.16b",
-                d = inlateout(vreg) zero => dots,
-                a = in(vreg) xored,
-                b = in(vreg) xored,
-            );
-        }
-
-        if vmaxvq_u32(dots) != 0 {
-            return false;
-        }
-
-        i += 16;
-    }
-
-    // Scalar tail
-    while i < len {
-        if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] {
-            return false;
-        }
-        i += 1;
-    }
-    true
-}
-
-/// NEON packed-pair kernel: scan 16 haystack positions per iteration,
-/// checking two rare bytes (case-insensitive) simultaneously.
-/// Same algorithm as the AVX2 version but with 128-bit vectors.
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-unsafe fn search_packed_pair_neon(
-    haystack: &[u8],
-    needle_lower: &[u8],
-    i1: usize,
-    i2: usize,
-) -> bool {
-    use core::arch::aarch64::*;
-
-    let n = needle_lower.len();
-    let hlen = haystack.len();
-    let ptr = haystack.as_ptr();
-    let last_start = hlen - n;
-
-    let b1 = needle_lower[i1];
-    let b1_alt = if b1.is_ascii_lowercase() {
-        ascii_swap_case(b1)
-    } else {
-        b1
-    };
-    let b2 = needle_lower[i2];
-    let b2_alt = if b2.is_ascii_lowercase() {
-        ascii_swap_case(b2)
-    } else {
-        b2
-    };
-
-    let v1_lo = vdupq_n_u8(b1);
-    let v1_hi = vdupq_n_u8(b1_alt);
-    let v2_lo = vdupq_n_u8(b2);
-    let v2_hi = vdupq_n_u8(b2_alt);
-
-    let max_idx = i1.max(i2);
-    let max_offset = hlen.saturating_sub(max_idx + 16);
-    let mut offset = 0usize;
-
-    while offset <= max_offset {
-        let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) };
-        let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) };
-
-        // Case-insensitive match: OR both case variants, then AND the two positions.
-        let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi));
-        let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi));
-
-        let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) };
-
-        while mask != 0 {
-            let bit = mask.trailing_zeros() as usize;
-            let candidate = offset + bit;
-            if candidate > last_start {
-                return false;
-            }
-            if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                return true;
-            }
-            mask &= mask - 1;
-        }
-
-        offset += 16;
-    }
-
-    // Tail: remaining positions that couldn't fill a full vector.
-    if offset <= last_start {
-        let rare_pos =
-            if case_insensitive_rank(needle_lower[i1]) <= case_insensitive_rank(needle_lower[i2]) {
-                i1
-            } else {
-                i2
-            };
-        let rare_byte = needle_lower[rare_pos];
-        let tail_start = offset + rare_pos;
-        let tail_end = last_start + rare_pos + 1;
-        if tail_start < tail_end {
-            let tail_space = &haystack[tail_start..tail_end];
-            if rare_byte.is_ascii_lowercase() {
-                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            } else {
-                for pos in memchr::memchr_iter(rare_byte, tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            }
-        }
-    }
-
-    false
-}
-
-#[inline]
-unsafe fn verify_dispatch(h: *const u8, needle_lower: &[u8]) -> bool {
-    #[cfg(target_arch = "x86_64")]
-    {
-        if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") {
-            return unsafe { verify_avx2(h, needle_lower) };
-        }
-    }
-    #[cfg(target_arch = "aarch64")]
-    {
-        if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") {
-            return unsafe { verify_neon_dotprod(h, needle_lower) };
-        }
-    }
-
-    verify_scalar(h, needle_lower)
-}
-
-// ── Packed-pair search (AVX2) ───────────────────────────────────────────
-
-/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration,
-/// checking two rare bytes (case-insensitive) simultaneously.
-/// 4 cmpeq + 2 or + 1 and + 1 movemask per 32 bytes — same memory
-/// bandwidth as memchr2 but quadratic selectivity.
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn search_packed_pair_avx2(
-    haystack: &[u8],
-    needle_lower: &[u8],
-    i1: usize,
-    i2: usize,
-) -> bool {
-    use core::arch::x86_64::*;
-
-    let n = needle_lower.len();
-    let hlen = haystack.len();
-    let ptr = haystack.as_ptr();
-    let last_start = hlen - n; // last valid match-start position
-
-    let b1 = needle_lower[i1];
-    let b1_alt = if b1.is_ascii_lowercase() {
-        ascii_swap_case(b1)
-    } else {
-        b1
-    };
-    let b2 = needle_lower[i2];
-    let b2_alt = if b2.is_ascii_lowercase() {
-        ascii_swap_case(b2)
-    } else {
-        b2
-    };
-
-    let v1_lo = _mm256_set1_epi8(b1 as i8);
-    let v1_hi = _mm256_set1_epi8(b1_alt as i8);
-    let v2_lo = _mm256_set1_epi8(b2 as i8);
-    let v2_hi = _mm256_set1_epi8(b2_alt as i8);
-
-    // Main loop: process 32 candidate positions per iteration.
-    // We load from ptr+offset+i1 and ptr+offset+i2, so we need
-    // offset + max(i1,i2) + 31 < hlen.
-    let max_idx = i1.max(i2);
-    let max_offset = hlen.saturating_sub(max_idx + 32);
-    let mut offset = 0usize;
-
-    while offset <= max_offset {
-        let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) };
-        let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) };
-
-        // Case-insensitive match: OR both case variants, then AND the two positions.
-        let eq1 = _mm256_or_si256(
-            _mm256_cmpeq_epi8(chunk1, v1_lo),
-            _mm256_cmpeq_epi8(chunk1, v1_hi),
-        );
-        let eq2 = _mm256_or_si256(
-            _mm256_cmpeq_epi8(chunk2, v2_lo),
-            _mm256_cmpeq_epi8(chunk2, v2_hi),
-        );
-
-        let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32;
-
-        while mask != 0 {
-            let bit = mask.trailing_zeros() as usize;
-            let candidate = offset + bit;
-            if candidate > last_start {
-                // Past the end — no more valid positions in this or future chunks.
-                return false;
-            }
-            if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                return true;
-            }
-            mask &= mask - 1;
-        }
-
-        offset += 32;
-    }
-
-    // Tail: remaining positions that couldn't fill a full vector.
-    // Use memchr2 on the rarest byte for these last few positions.
-    if offset <= last_start {
-        let rare_pos =
-            if case_insensitive_rank(needle_lower[i1]) <= case_insensitive_rank(needle_lower[i2]) {
-                i1
-            } else {
-                i2
-            };
-        let rare_byte = needle_lower[rare_pos];
-        let tail_start = offset + rare_pos;
-        let tail_end = last_start + rare_pos + 1;
-        if tail_start < tail_end {
-            let tail_space = &haystack[tail_start..tail_end];
-            if rare_byte.is_ascii_lowercase() {
-                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            } else {
-                for pos in memchr::memchr_iter(rare_byte, tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            }
-        }
-    }
-
-    false
-}
-
-/// Packed-pair case-insensitive substring search.
-///
-/// Selects the two rarest bytes from the needle (using the memchr byte
-/// frequency heuristic), then SIMD-scans for both at their known offsets
-/// simultaneously.  Falls back to `search` for needles shorter than 2 bytes.
-pub fn search_packed_pair(haystack: &[u8], needle_lower: &[u8]) -> bool {
-    let n = needle_lower.len();
-    if n == 0 {
-        return true;
-    }
-    if n < 2 {
-        return search(haystack, needle_lower);
-    }
-    if n > haystack.len() {
-        return false;
-    }
-
-    let (i1, i2) = select_rare_pair(needle_lower);
-
-    #[cfg(target_arch = "x86_64")]
-    {
-        if std::is_x86_feature_detected!("avx2") {
-            // Need enough haystack for at least one vector load.
-            let max_idx = i1.max(i2);
-            if haystack.len() >= max_idx + 32 {
-                return unsafe { search_packed_pair_avx2(haystack, needle_lower, i1, i2) };
-            }
-        }
-    }
-
-    #[cfg(target_arch = "aarch64")]
-    {
-        // The NEON packed-pair scan checks 16 bytes/iteration with ~7 ops,
-        // while memchr's optimized loop processes more bytes with fewer ops.
-        // Packed-pair wins when the first byte is common (lots of false
-        // positives for memchr2 that we avoid). But when the first byte is
-        // rare (z, q, x, ...) memchr2 has no false positives and its raw
-        // throughput dominates. Threshold 200 on the frequency table splits
-        // common letters (s=243, e=253, f=227) from rare ones (z=152, q=139).
-        let first_byte_rank = case_insensitive_rank(needle_lower[0]);
-        let max_idx = i1.max(i2);
-        if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 {
-            return unsafe { search_packed_pair_neon(haystack, needle_lower, i1, i2) };
-        }
-    }
-
-    // Fallback for short haystacks or non-SIMD platforms.
-    search(haystack, needle_lower)
-}
-
-// ── Original memchr2 first-byte search ──────────────────────────────────
-
-/// Case-insensitive search using memchr2 on the first byte.
-pub fn search(haystack: &[u8], needle_lower: &[u8]) -> bool {
-    let n = needle_lower.len();
-    if n == 0 {
-        return true;
-    }
-    if n > haystack.len() {
-        return false;
-    }
-
-    let search_space = &haystack[..=haystack.len() - n];
-    let first = needle_lower[0];
-
-    if first.is_ascii_lowercase() {
-        let alt = ascii_swap_case(first);
-        for pos in memchr::memchr2_iter(first, alt, search_space) {
-            if unsafe { verify_dispatch(haystack.as_ptr().add(pos), needle_lower) } {
-                return true;
-            }
-        }
-    } else {
-        for pos in memchr::memchr_iter(first, search_space) {
-            if unsafe { verify_dispatch(haystack.as_ptr().add(pos), needle_lower) } {
-                return true;
-            }
-        }
-    }
-    false
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn basic_case_insensitive() {
-        assert!(search_packed_pair(b"Hello World", b"hello"));
-        assert!(search_packed_pair(b"Hello World", b"world"));
-        assert!(search_packed_pair(b"NOMORE bugs", b"nomore"));
-        assert!(!search_packed_pair(b"Hello World", b"xyz"));
-    }
-
-    #[test]
-    fn edge_cases() {
-        assert!(search_packed_pair(b"ab", b"ab"));
-        assert!(search_packed_pair(b"AB", b"ab"));
-        assert!(!search_packed_pair(b"a", b"ab"));
-        assert!(search_packed_pair(b"anything", b""));
-        assert!(!search_packed_pair(b"", b"x"));
-    }
-
-    #[test]
-    fn packed_pair_matches_search() {
-        let haystacks: &[&[u8]] = &[
-            b"The quick brown fox jumps over the lazy dog",
-            b"int mutex_lock(struct mutex *lock) { return 0; }",
-            b"#define NOMORE_RETRIES 5\nif (nomore) return;",
-            b"abcdefghijklmnopqrstuvwxyz",
-            b"short",
-        ];
-        let needles: &[&[u8]] = &[b"fox", b"mutex", b"nomore", b"xyz", b"the", b"short", b"qr"];
-        for h in haystacks {
-            for n in needles {
-                let lower: Vec<u8> = n.iter().map(|b| b.to_ascii_lowercase()).collect();
-                assert_eq!(
-                    search_packed_pair(h, &lower),
-                    search(h, &lower),
-                    "mismatch for haystack={:?} needle={:?}",
-                    std::str::from_utf8(h),
-                    std::str::from_utf8(n),
-                );
-            }
-        }
-    }
-
-    #[test]
-    fn long_haystack_neon_path() {
-        // Haystack > 16 bytes exercises NEON packed-pair search loop
-        let haystack =
-            b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb";
-        assert!(search_packed_pair(haystack, b"this_is_a_long_needle_test"));
-        assert!(!search_packed_pair(
-            haystack,
-            b"this_is_a_long_needle_testz"
-        ));
-
-        // Needle >= 16 bytes exercises NEON dotprod verify
-        let long_needle = b"struct mutex *lock";
-        let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }";
-        assert!(search_packed_pair(haystack2, long_needle));
-
-        // All uppercase haystack, lowercase needle
-        let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-        assert!(search_packed_pair(upper_hay, b"qrstuvwxyz0123456789a"));
-        assert!(!search_packed_pair(upper_hay, b"qrstuvwxyz01234567899"));
-
-        // Needle at very end
-        let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me";
-        assert!(search_packed_pair(end_hay, b"find_me"));
-
-        // Needle at very start
-        assert!(search_packed_pair(end_hay, b"xx"));
-
-        // 1KB haystack with needle near the end
-        let mut big = vec![b'z'; 1024];
-        big[1000..1010].copy_from_slice(b"hElLo_WoRl");
-        assert!(search_packed_pair(&big, b"hello_wo"));
-        assert!(!search_packed_pair(&big, b"hello_world"));
-    }
-
-    #[test]
-    fn rare_pair_selection() {
-        // For "nomore": n=246, o=244, m=233, o=244, r=245, e=253
-        // Rarest positions should include 'm' (pos 2, rank 233)
-        let (i1, i2) = select_rare_pair(b"nomore");
-        let ranks: Vec<u8> = b"nomore"
-            .iter()
-            .map(|&b| case_insensitive_rank(b))
-            .collect();
-        let r1 = ranks[i1];
-        let r2 = ranks[i2];
-        // Both selected ranks should be <= all other ranks
-        for (i, &r) in ranks.iter().enumerate() {
-            if i != i1 && i != i2 {
-                assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal");
-            }
-        }
-    }
-}

crates/fff-core/src/constants.rs

@@ -0,0 +1,48 @@
+/// Largest file whose full content fff will touch: the default grep read cap
+/// (`GrepSearchOptions::max_file_size`) and the content-cache mmap cap
+/// (`ContentCacheBudget::max_file_size`). Binary detection also streams up to
+/// this far so nothing grep would read is left unclassified.
+pub const MAX_FFFILE_SIZE: u64 = 10 * 1024 * 1024;
+
+/// Upper bound on a file the bigram builder will build, if the file is very large there is a
+/// big probability it will only bloat the available bigrams and will anyway pop ut from the prefilter
+pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
+
+/// Total bytes the persistent content mmap cache may hold for a small repo.
+pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
+
+/// Files below one page waste the remainder when mmapped, so the cache skips
+/// them and falls back to chunked reads. Unused on Windows (no content cache)
+#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
+pub const MMAP_THRESHOLD: u64 = 16 * 1024;
+#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
+pub const MMAP_THRESHOLD: u64 = 4 * 1024;
+
+/// Watcher overflow capacity reserved after the initial scan
+pub const MAX_OVERFLOW_FILES: usize = 1024;
+
+/// Minimum delay between watcher-initiated rescans.
+pub const RESCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// Rescan delay for large indexes.
+pub const RESCAN_MIN_INTERVAL_LARGE_INDEX: std::time::Duration =
+    std::time::Duration::from_secs(5 * 60);
+
+/// Live-file count at which [`RESCAN_MIN_INTERVAL_LARGE_INDEX`] takes over.
+pub const LARGE_INDEX_FILE_COUNT: usize = 1_000_000;
+
+/// Fresh-mmap threshold: files at or above this size get mmapped directly on
+/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
+/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this
+/// constant is gated to non-Windows targets to keep `-D unused-imports` happy.
+#[cfg(target_os = "macos")]
+pub const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024;
+#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
+pub const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024;
+
+// we do not support 32kb path limit on windows
+#[cfg(target_os = "windows")]
+pub const PATH_BUF_SIZE: usize = 4096;
+
+#[cfg(not(target_os = "windows"))]
+pub const PATH_BUF_SIZE: usize = libc::PATH_MAX as usize;

crates/fff-core/src/constraints.rs

@@ -1,780 +0,0 @@
-//! Constraint-based prefiltering for search queries.
-
-use ahash::AHashSet;
-use fff_query_parser::{Constraint, GitStatusFilter};
-use smallvec::SmallVec;
-
-use crate::git::is_modified_status;
-use crate::simd_path::ArenaPtr;
-
-/// `needle` must already be lowercase.
-#[inline]
-fn contains_ascii_ci(haystack: &str, needle: &str) -> bool {
-    let h = haystack.as_bytes();
-    let n = needle.as_bytes();
-    if n.len() > h.len() {
-        return false;
-    }
-    if n.is_empty() {
-        return true;
-    }
-    let first = n[0];
-    for i in 0..=(h.len() - n.len()) {
-        if h[i].to_ascii_lowercase() == first
-            && h[i..i + n.len()]
-                .iter()
-                .zip(n)
-                .all(|(a, b)| a.to_ascii_lowercase() == *b)
-        {
-            return true;
-        }
-    }
-    false
-}
-
-const PAR_THRESHOLD: usize = 10_000;
-
-pub(crate) trait Constrainable {
-    fn write_file_name(&self, arena: ArenaPtr, out: &mut String);
-    fn git_status(&self) -> Option<git2::Status>;
-    fn write_relative_path(&self, arena: ArenaPtr, out: &mut String);
-}
-
-/// Windows stores paths with `\\`; `/` comes from user queries.
-#[inline]
-fn is_path_sep(b: u8) -> bool {
-    #[cfg(windows)]
-    {
-        b == b'/' || b == b'\\'
-    }
-    #[cfg(not(windows))]
-    {
-        b == b'/'
-    }
-}
-
-#[inline]
-fn path_slice_eq(a: &[u8], b: &[u8]) -> bool {
-    if a.len() != b.len() {
-        return false;
-    }
-    a.iter().zip(b).all(|(x, y)| {
-        if is_path_sep(*x) && is_path_sep(*y) {
-            true
-        } else {
-            x.eq_ignore_ascii_case(y)
-        }
-    })
-}
-
-/// Path ends with suffix at a path-separator boundary (case-insensitive).
-#[inline]
-pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool {
-    let path_bytes = path.as_bytes();
-    let suffix_bytes = suffix.as_bytes();
-    if path_bytes.len() < suffix_bytes.len() {
-        return false;
-    }
-
-    let start = path.len() - suffix.len();
-
-    // Multi-byte UTF-8 may put `start` inside a char.
-    if !path.is_char_boundary(start) {
-        return false;
-    }
-
-    if !path_slice_eq(&path_bytes[start..], suffix_bytes) {
-        return false;
-    }
-
-    // Exact or preceded by a separator. Scan backward past any multi-byte
-    // continuation bytes to find the preceding ASCII byte.
-    if start == 0 {
-        return true;
-    }
-    let mut i = start;
-    while i > 0 {
-        i -= 1;
-        if path_bytes[i] < 128 {
-            return is_path_sep(path_bytes[i]);
-        }
-    }
-    false
-}
-
-#[inline]
-pub fn file_has_extension(file_name: &str, ext: &str) -> bool {
-    let name_bytes = file_name.as_bytes();
-    let ext_bytes = ext.as_bytes();
-    if name_bytes.len() <= ext_bytes.len() + 1 {
-        return false;
-    }
-    let start = name_bytes.len() - ext_bytes.len() - 1;
-    if start > 0 && !file_name.is_char_boundary(start) {
-        return false;
-    }
-    name_bytes.get(start) == Some(&b'.') && name_bytes[start + 1..].eq_ignore_ascii_case(ext_bytes)
-}
-
-/// Matches multi-segment queries like `libswscale/aarch64`.
-#[inline]
-pub fn path_contains_segment(path: &str, segment: &str) -> bool {
-    let path_bytes = path.as_bytes();
-    let segment_bytes = segment.as_bytes();
-    let segment_len = segment_bytes.len();
-
-    if path_bytes.len() > segment_len
-        && is_path_sep(path_bytes[segment_len])
-        && path.is_char_boundary(segment_len)
-        && path_slice_eq(&path_bytes[..segment_len], segment_bytes)
-    {
-        return true;
-    }
-
-    if path_bytes.len() < segment_len + 2 {
-        return false;
-    }
-
-    for i in 0..path_bytes.len().saturating_sub(segment_len + 1) {
-        if is_path_sep(path_bytes[i]) {
-            let start = i + 1;
-            let end = start + segment_len;
-            if end < path_bytes.len()
-                && is_path_sep(path_bytes[end])
-                && path.is_char_boundary(start)
-                && path.is_char_boundary(end)
-                && path_slice_eq(&path_bytes[start..end], segment_bytes)
-            {
-                return true;
-            }
-        }
-    }
-    false
-}
-
-#[inline]
-#[allow(clippy::too_many_arguments)]
-fn item_matches_constraint_at_index<T: Constrainable>(
-    item: &T,
-    item_index: usize,
-    constraint: &Constraint<'_>,
-    glob_results: &[(bool, AHashSet<usize>)],
-    glob_idx: &mut usize,
-    negate: bool,
-    arena: ArenaPtr,
-    fname_buf: &mut String,
-    path_buf: &mut String,
-) -> bool {
-    let matches = match constraint {
-        Constraint::Extension(ext) => {
-            item.write_file_name(arena, fname_buf);
-            file_has_extension(fname_buf, ext)
-        }
-        Constraint::Glob(_) => {
-            let result = glob_results
-                .get(*glob_idx)
-                .map(|(is_neg, set)| {
-                    let matched = set.contains(&item_index);
-
-                    if *is_neg { !matched } else { matched }
-                })
-                .unwrap_or(true);
-            *glob_idx += 1;
-            return if negate { !result } else { result };
-        }
-        Constraint::PathSegment(segment) => {
-            item.write_relative_path(arena, path_buf);
-            path_contains_segment(path_buf, segment)
-        }
-        Constraint::FilePath(suffix) => {
-            item.write_relative_path(arena, path_buf);
-            path_ends_with_suffix(path_buf, suffix)
-        }
-        Constraint::GitStatus(status_filter) => match (item.git_status(), status_filter) {
-            (Some(status), GitStatusFilter::Modified) => is_modified_status(status),
-            (Some(status), GitStatusFilter::Untracked) => status.contains(git2::Status::WT_NEW),
-            (Some(status), GitStatusFilter::Staged) => status.intersects(
-                git2::Status::INDEX_NEW
-                    | git2::Status::INDEX_MODIFIED
-                    | git2::Status::INDEX_DELETED
-                    | git2::Status::INDEX_RENAMED
-                    | git2::Status::INDEX_TYPECHANGE,
-            ),
-            (Some(status), GitStatusFilter::Unmodified) => status.is_empty(),
-            (None, GitStatusFilter::Unmodified) => true,
-            (None, _) => false,
-        },
-        Constraint::Not(inner) => {
-            return item_matches_constraint_at_index(
-                item,
-                item_index,
-                inner,
-                glob_results,
-                glob_idx,
-                !negate,
-                arena,
-                fname_buf,
-                path_buf,
-            );
-        }
-
-        // only works with negation
-        Constraint::Text(text) => {
-            item.write_relative_path(arena, path_buf);
-            contains_ascii_ci(path_buf, text)
-        }
-
-        // Parts and Exclude are handled at a higher level
-        Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true,
-    };
-
-    if negate { !matches } else { matches }
-}
-
-/// Returns `None` if no constraints are present, `Some(filtered)` otherwise.
-/// Extension constraints use OR logic; all others use AND.
-pub(crate) fn apply_constraints<'a, T: Constrainable + Sync>(
-    items: &'a [T],
-    constraints: &[Constraint<'_>],
-    arena: ArenaPtr,
-) -> Option<Vec<&'a T>> {
-    if constraints.is_empty() {
-        return None;
-    }
-
-    // Separate extension constraints from other constraints — they use OR logic
-    let mut extensions: SmallVec<[&str; 8]> = SmallVec::new();
-    let mut other_constraints: SmallVec<[&Constraint<'_>; 8]> = SmallVec::new();
-
-    for constraint in constraints {
-        match constraint {
-            Constraint::Extension(ext) => extensions.push(ext),
-            _ => other_constraints.push(constraint),
-        }
-    }
-
-    // Only collect paths if we have glob constraints (expensive)
-    let has_globs = other_constraints
-        .iter()
-        .any(|c| matches!(c, Constraint::Glob(_) | Constraint::Not(_)));
-
-    let glob_results = if has_globs {
-        // Build a single contiguous buffer of all relative paths + offset table.
-        // One allocation for the buffer, one for offsets — NOT one String per file.
-        // On Windows we fold `\\` into `/` while copying so globset/zlob see a
-        // canonical separator. The rewrite is in place on bytes we just wrote.
-        let mut path_buf = Vec::<u8>::new();
-        let mut offsets = Vec::<(usize, usize)>::with_capacity(items.len());
-        let mut tmp = String::with_capacity(64);
-        for item in items.iter() {
-            let start = path_buf.len();
-            item.write_relative_path(arena, &mut tmp);
-            path_buf.extend_from_slice(tmp.as_bytes());
-            #[cfg(windows)]
-            for b in &mut path_buf[start..] {
-                if *b == b'\\' {
-                    *b = b'/';
-                }
-            }
-            offsets.push((start, path_buf.len() - start));
-        }
-        let path_refs: Vec<&str> = offsets
-            .iter()
-            .map(|&(off, len)| unsafe { std::str::from_utf8_unchecked(&path_buf[off..off + len]) })
-            .collect();
-        precompute_glob_matches(&other_constraints, &path_refs)
-    } else {
-        Vec::new()
-    };
-
-    let filtered: Vec<&T> = if items.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        items
-            .par_iter()
-            .enumerate()
-            .map_init(
-                || (String::with_capacity(64), String::with_capacity(64)),
-                |(fname_buf, path_buf), (i, item)| {
-                    if !extensions.is_empty() {
-                        item.write_file_name(arena, fname_buf);
-                        if !extensions
-                            .iter()
-                            .any(|ext| file_has_extension(fname_buf, ext))
-                        {
-                            return None;
-                        }
-                    }
-
-                    let mut glob_idx = 0;
-                    if other_constraints.iter().all(|constraint| {
-                        item_matches_constraint_at_index(
-                            item,
-                            i,
-                            constraint,
-                            &glob_results,
-                            &mut glob_idx,
-                            false,
-                            arena,
-                            fname_buf,
-                            path_buf,
-                        )
-                    }) {
-                        Some(item)
-                    } else {
-                        None
-                    }
-                },
-            )
-            .flatten()
-            .collect()
-    } else {
-        let mut fname_buf = String::with_capacity(64);
-        let mut path_buf = String::with_capacity(64);
-
-        items
-            .iter()
-            .enumerate()
-            .filter(|&(i, item)| {
-                if !extensions.is_empty() {
-                    item.write_file_name(arena, &mut fname_buf);
-                    if !extensions
-                        .iter()
-                        .any(|ext| file_has_extension(&fname_buf, ext))
-                    {
-                        return false;
-                    }
-                }
-
-                let mut glob_idx = 0;
-                other_constraints.iter().all(|constraint| {
-                    item_matches_constraint_at_index(
-                        item,
-                        i,
-                        constraint,
-                        &glob_results,
-                        &mut glob_idx,
-                        false,
-                        arena,
-                        &mut fname_buf,
-                        &mut path_buf,
-                    )
-                })
-            })
-            .map(|(_, item)| item)
-            .collect()
-    };
-
-    Some(filtered)
-}
-
-fn precompute_glob_matches<'a>(
-    constraints: &[&Constraint<'a>],
-    paths: &[&str],
-) -> Vec<(bool, AHashSet<usize>)> {
-    let mut results = Vec::new();
-    for constraint in constraints {
-        collect_glob_indices(constraint, paths, &mut results, false);
-    }
-    results
-}
-
-fn collect_glob_indices<'a>(
-    constraint: &Constraint<'a>,
-    paths: &[&str],
-    results: &mut Vec<(bool, AHashSet<usize>)>,
-    _is_negated: bool,
-) {
-    match constraint {
-        Constraint::Glob(pattern) => {
-            let indices = match_glob_pattern(pattern, paths);
-            // Negation is handled by the `negate` parameter in
-            // `item_matches_constraint_at_index`, NOT here.  Storing
-            // `is_negated=true` caused a double-negation bug when the
-            // Glob arm also applied `negate`.
-            results.push((false, indices));
-        }
-        Constraint::Not(inner) => {
-            collect_glob_indices(inner, paths, results, true);
-        }
-        _ => {}
-    }
-}
-
-/// Match a glob pattern against a list of paths, returning the set of matching indices.
-///
-/// When the `zlob` feature is enabled, delegates to `zlob::zlob_match_paths` (Zig-compiled
-/// C library, fastest). Otherwise falls back to `globset::Glob` (pure Rust).
-#[cfg(feature = "zlob")]
-fn match_glob_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
-    let Ok(Some(matches)) = zlob::zlob_match_paths(pattern, paths, zlob::ZlobFlags::RECOMMENDED)
-    else {
-        return AHashSet::new();
-    };
-
-    let matched_set: AHashSet<usize> = matches.iter().map(|s| s.as_ptr() as usize).collect();
-
-    if paths.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        paths
-            .par_iter()
-            .enumerate()
-            .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
-            .map(|(i, _)| i)
-            .collect::<Vec<_>>()
-            .into_iter()
-            .collect()
-    } else {
-        paths
-            .iter()
-            .enumerate()
-            .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
-            .map(|(i, _)| i)
-            .collect()
-    }
-}
-
-#[cfg(not(feature = "zlob"))]
-fn match_glob_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
-    let Ok(glob) = globset::Glob::new(pattern) else {
-        return AHashSet::new();
-    };
-    let matcher = glob.compile_matcher();
-
-    if paths.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        paths
-            .par_iter()
-            .enumerate()
-            .filter(|(_, p)| matcher.is_match(p))
-            .map(|(i, _)| i)
-            .collect::<Vec<_>>()
-            .into_iter()
-            .collect()
-    } else {
-        paths
-            .iter()
-            .enumerate()
-            .filter(|(_, p)| matcher.is_match(p))
-            .map(|(i, _)| i)
-            .collect()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[derive(Clone)]
-    struct TestItem {
-        relative_path: &'static str,
-        file_name: &'static str,
-    }
-
-    impl Constrainable for TestItem {
-        fn write_file_name(&self, _arena: ArenaPtr, out: &mut String) {
-            out.clear();
-            out.push_str(self.file_name);
-        }
-
-        fn write_relative_path(&self, _arena: ArenaPtr, out: &mut String) {
-            out.clear();
-            out.push_str(self.relative_path);
-        }
-
-        fn git_status(&self) -> Option<git2::Status> {
-            None
-        }
-    }
-
-    #[test]
-    fn test_file_has_extension() {
-        assert!(file_has_extension("file.rs", "rs"));
-        assert!(file_has_extension("file.RS", "rs")); // case-insensitive
-        assert!(file_has_extension("file.test.rs", "rs"));
-        assert!(file_has_extension("a.rs", "rs"));
-
-        assert!(!file_has_extension("file.tsx", "rs"));
-        assert!(!file_has_extension("rs", "rs")); // too short
-        assert!(!file_has_extension(".rs", "rs")); // just extension
-        assert!(!file_has_extension("file.rsx", "rs")); // different extension
-        assert!(!file_has_extension("filers", "rs")); // no dot
-    }
-
-    #[test]
-    fn test_path_contains_segment() {
-        // Segment at start
-        assert!(path_contains_segment("src/lib.rs", "src"));
-        assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive
-
-        // Segment in middle
-        assert!(path_contains_segment("app/src/lib.rs", "src"));
-        assert!(path_contains_segment("app/SRC/lib.rs", "src"));
-
-        // Multiple levels
-        assert!(path_contains_segment("core/workflow/src/main.rs", "src"));
-        assert!(path_contains_segment(
-            "core/workflow/src/main.rs",
-            "workflow"
-        ));
-        assert!(path_contains_segment("core/workflow/src/main.rs", "core"));
-
-        // Should not match partial segments
-        assert!(!path_contains_segment("source/lib.rs", "src"));
-        assert!(!path_contains_segment("mysrc/lib.rs", "src"));
-
-        // Should not match filename
-        assert!(!path_contains_segment("lib/src", "src"));
-
-        // Multi-segment constraints
-        assert!(path_contains_segment(
-            "libswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "foo/libswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "foo/LibSwscale/AArch64/input.S",
-            "libswscale/aarch64"
-        )); // case-insensitive
-        assert!(!path_contains_segment(
-            "xlibswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        )); // partial match at start
-        assert!(!path_contains_segment(
-            "foo/libswscale/aarch64x/input.S",
-            "libswscale/aarch64"
-        )); // partial match at end
-        assert!(path_contains_segment(
-            "crates/fff-core/src/grep.rs",
-            "fff-core/src"
-        ));
-
-        // Edge cases
-        assert!(!path_contains_segment("", "src"));
-        assert!(!path_contains_segment("src", "src")); // no trailing slash
-    }
-
-    #[cfg(windows)]
-    #[test]
-    fn test_path_contains_segment_accepts_backslash() {
-        assert!(path_contains_segment("src\\lib.rs", "src"));
-        assert!(path_contains_segment(
-            "app\\modules\\src\\services\\x.lua",
-            "src"
-        ));
-        assert!(path_contains_segment("app\\SRC\\x.lua", "src"));
-
-        assert!(path_contains_segment(
-            "foo\\libswscale\\aarch64\\input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "crates\\fff-core\\src\\grep.rs",
-            "fff-core/src"
-        ));
-
-        assert!(!path_contains_segment("mysrc\\lib.rs", "src"));
-        assert!(!path_contains_segment(
-            "xlibswscale\\aarch64\\in.S",
-            "libswscale/aarch64"
-        ));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix() {
-        // Exact match
-        assert!(path_ends_with_suffix(
-            "libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Suffix match at / boundary
-        assert!(path_ends_with_suffix(
-            "foo/libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Deep nesting
-        assert!(path_ends_with_suffix(
-            "a/b/c/libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // No boundary — partial directory name
-        assert!(!path_ends_with_suffix(
-            "xlibswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Case insensitive
-        assert!(path_ends_with_suffix(
-            "foo/LibSwscale/Input.C",
-            "libswscale/input.c"
-        ));
-
-        // Single file name
-        assert!(path_ends_with_suffix("input.c", "input.c"));
-        assert!(!path_ends_with_suffix("xinput.c", "input.c"));
-
-        // Suffix longer than path
-        assert!(!path_ends_with_suffix("input.c", "foo/input.c"));
-
-        // Simple path
-        assert!(path_ends_with_suffix("src/main.rs", "src/main.rs"));
-        assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs"));
-    }
-
-    #[cfg(windows)]
-    #[test]
-    fn test_path_ends_with_suffix_accepts_backslash() {
-        assert!(path_ends_with_suffix(
-            "app\\modules\\src\\services\\handler.lua",
-            "services/handler.lua"
-        ));
-        assert!(path_ends_with_suffix(
-            "foo\\libswscale\\input.c",
-            "libswscale/input.c"
-        ));
-        assert!(!path_ends_with_suffix(
-            "xlibswscale\\input.c",
-            "libswscale/input.c"
-        ));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix_does_not_panic_on_unicode_suffix() {
-        assert!(!path_ends_with_suffix("유니코드_파일_테스트.csv", "트.c"));
-        assert!(path_ends_with_suffix(
-            "data/유니코드_파일_테스트.csv",
-            "유니코드_파일_테스트.csv"
-        ));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix_unicode_apostrophe_mismatch() {
-        assert!(!path_ends_with_suffix(
-            "dir/\u{2019}bar/file.txt",
-            "'bar/file.txt"
-        ));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix_unicode_space_mismatch() {
-        assert!(!path_ends_with_suffix(
-            "dir/\u{202f}am/file.txt",
-            " am/file.txt"
-        ));
-    }
-
-    #[test]
-    fn test_path_contains_segment_does_not_panic_on_unicode_segment() {
-        assert!(!path_contains_segment("문서/notes.txt", "문x"));
-        assert!(path_contains_segment("프로젝트/문서/notes.txt", "문서"));
-    }
-
-    #[test]
-    fn test_path_contains_segment_unicode_no_panic() {
-        assert!(!path_contains_segment(
-            "Library/Cloud/Project\u{2019}s Folder/books.ttl",
-            "Project's Folder"
-        ));
-    }
-
-    #[test]
-    fn test_file_has_extension_unicode_no_panic() {
-        assert!(!file_has_extension("cat\u{00e9}.rs", "s"));
-    }
-
-    #[test]
-    fn test_file_has_extension_unicode_filename() {
-        assert!(file_has_extension("운영-가이드.md", "md"));
-        assert!(file_has_extension("테스트.csv", "csv"));
-        assert!(!file_has_extension("테스트.csv", "md"));
-    }
-
-    #[test]
-    fn test_apply_constraints_file_path_with_unicode_suffix() {
-        let arena_ptr = ArenaPtr(std::ptr::null());
-
-        let item = TestItem {
-            relative_path: "data/유니코드_파일_테스트.csv",
-            file_name: "유니코드_파일_테스트.csv",
-        };
-
-        let exact = [Constraint::FilePath("유니코드_파일_테스트.csv")];
-        let mismatch = [Constraint::FilePath("트.c")];
-
-        let exact_items = [item.clone()];
-        let exact_matches =
-            apply_constraints(&exact_items, &exact, arena_ptr).expect("constraints applied");
-        assert_eq!(exact_matches.len(), 1);
-
-        let mismatch_items = [item];
-        let mismatch_matches =
-            apply_constraints(&mismatch_items, &mismatch, arena_ptr).expect("constraints applied");
-        assert!(mismatch_matches.is_empty());
-    }
-
-    #[test]
-    fn test_unicode_path_no_panic_real_korean_cases() {
-        // Real Korean paths that caused panics
-        let path1 = "Downloads/(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv";
-        let path2 = "hermes-agent-lecture-materials/세부_커리큘럼_최종.csv";
-        let path3 = "projects/fastcampus-hermes-agent-curriculum/chapters/part-02-Hermes-설치-및-기본-사용/section-02-doctor로-설치-상태-검증/research/03-fix가-자동-수정하는-것과-못하는-것.md";
-
-        // These must not panic regardless of segment/suffix used
-        assert!(!path_contains_segment(path1, "작성"));
-        assert!(!path_ends_with_suffix(path1, "작성.csv"));
-        assert!(!path_contains_segment(path2, "최종"));
-        assert!(!path_ends_with_suffix(path2, "최종.csv"));
-        assert!(!path_contains_segment(path3, "수정"));
-        assert!(!path_ends_with_suffix(path3, "것.md"));
-
-        // Positive cases should still work
-        assert!(path_contains_segment(
-            path2,
-            "hermes-agent-lecture-materials"
-        ));
-        assert!(path_ends_with_suffix(
-            path1,
-            "(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv"
-        ));
-        assert!(path_ends_with_suffix(path2, "세부_커리큘럼_최종.csv"));
-    }
-
-    #[test]
-    fn test_negated_glob_excludes_matching_files() {
-        let arena_ptr = ArenaPtr(std::ptr::null());
-
-        let items = vec![
-            TestItem {
-                relative_path: "src/main.rs",
-                file_name: "main.rs",
-            },
-            TestItem {
-                relative_path: "src/lib.ts",
-                file_name: "lib.ts",
-            },
-            TestItem {
-                relative_path: "include/fff.h",
-                file_name: "fff.h",
-            },
-        ];
-
-        // Not(Glob("**/*.rs")) should exclude .rs files
-        let constraints = vec![Constraint::Not(Box::new(Constraint::Glob("**/*.rs")))];
-        let result = apply_constraints(&items, &constraints, arena_ptr).unwrap();
-        let paths: Vec<&str> = result.iter().map(|i| i.relative_path).collect();
-        assert!(
-            !paths.contains(&"src/main.rs"),
-            "rs file should be excluded"
-        );
-        assert!(paths.contains(&"src/lib.ts"), "ts file should be included");
-        assert!(
-            paths.contains(&"include/fff.h"),
-            "h file should be included"
-        );
-    }
-}

crates/fff-core/src/dbs/db_healthcheck.rs

@@ -9,16 +9,23 @@ pub struct DbHealth {
     pub disk_size: u64,
     /// Entry counts by table name
     pub entry_counts: Vec<(&'static str, u64)>,
+    /// Set to `false` if can not acquire the write lock
+    pub healthy: bool,
 }
 
 pub trait DbHealthChecker {
-    fn get_env(&self) -> &heed::Env;
+    fn get_env(&self) -> &heed::Env<heed::WithoutTls>;
+    fn is_healthy(&self) -> bool;
+    /// Entries per database, each group has a static string label
     fn count_entries(&self) -> Result<Vec<(&'static str, u64)>>;
 
+    /// Health summary of the database, returns summary struct
     fn get_health(&self) -> Result<DbHealth> {
         let env = self.get_env();
 
-        let size = env.real_disk_size().map_err(crate::error::Error::EnvOpen)?;
+        let size = env
+            .real_disk_size()
+            .map_err(crate::error::Error::GenericDbError)?;
         let path = env.path().to_string_lossy().to_string();
         let entry_counts = self.count_entries()?;
 
@@ -26,6 +33,7 @@ pub trait DbHealthChecker {
             path,
             disk_size: size,
             entry_counts,
+            healthy: self.is_healthy(),
         })
     }
 }

crates/fff-core/src/dbs/env_pool.rs

@@ -0,0 +1,284 @@
+use heed::{Env, EnvOpenOptions, WithoutTls};
+use std::collections::HashMap;
+use std::fs;
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, LazyLock, Mutex, MutexGuard, PoisonError, Weak};
+use std::thread;
+use std::time::Duration;
+
+use crate::error::{Error, Result};
+use crate::lmdb::DbHealth;
+
+pub(crate) struct EnvSpec {
+    pub label: &'static str,
+    pub map_size: usize,
+    pub max_dbs: u32,
+    pub size_cap_bytes: u64,
+}
+
+pub(crate) struct PooledEnv {
+    env: Env<WithoutTls>,
+    key: PathBuf,
+    /// lmdb's env spec label
+    label: &'static str,
+    map_size: usize,
+    max_dbs: u32,
+    health: DbHealth,
+    gc_started: AtomicBool,
+    dbi_lock: Mutex<()>,
+}
+
+impl Drop for PooledEnv {
+    fn drop(&mut self) {
+        let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+        // Only remove a dead entry: begin_exclusive_destroy may have removed ours.
+        if pool.get(&self.key).is_some_and(|w| w.strong_count() == 0) {
+            pool.remove(&self.key);
+        }
+        // heed closes the env right after this body; a concurrent reopen of the
+        // same path rides out that gap via env_closing_event in get_or_open.
+    }
+}
+
+// Cloneable handle to a process-shared LMDB env, derefs to `heed::Env`.
+#[derive(Clone)]
+pub(crate) struct SharedEnv(Arc<PooledEnv>);
+
+impl Deref for SharedEnv {
+    type Target = Env<WithoutTls>;
+    fn deref(&self) -> &Env<WithoutTls> {
+        &self.0.env
+    }
+}
+
+impl std::fmt::Debug for SharedEnv {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("SharedEnv").field(&self.0.env).finish()
+    }
+}
+
+impl SharedEnv {
+    pub(crate) fn get_or_open(db_path: &Path, spec: &EnvSpec) -> Result<Self> {
+        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+        let path = fs::canonicalize(db_path).map_err(|e| Error::EnvOpen {
+            db: spec.label,
+            source: heed::Error::Io(e),
+        })?;
+
+        let mut close_waits = 0u32;
+        let mut transient_retries = 0u32;
+
+        loop {
+            let mut open_failed = false;
+
+            {
+                let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+                if let Some(existing) = pool.get(&path).and_then(Weak::upgrade) {
+                    drop(pool);
+                    if existing.label != spec.label
+                        || existing.map_size != spec.map_size
+                        || existing.max_dbs != spec.max_dbs
+                    {
+                        return Err(Error::EnvSpecMismatch {
+                            path,
+                            open_as: existing.label,
+                            requested_as: spec.label,
+                        });
+                    }
+                    return Ok(Self(existing));
+                }
+
+                erase_if_oversized(&path, spec);
+                let result = unsafe {
+                    // MDB_NOTLS: reader slots are tied to txn objects (freed on
+                    // commit/abort) instead of pinned per thread for its lifetime (#783).
+                    let mut opts = EnvOpenOptions::new().read_txn_without_tls();
+                    opts.map_size(spec.map_size);
+                    opts.max_readers(max_readers());
+                    if spec.max_dbs > 0 {
+                        opts.max_dbs(spec.max_dbs);
+                    }
+                    opts.open(&path)
+                };
+
+                match result {
+                    Ok(env) => {
+                        let entry = Arc::new(PooledEnv {
+                            env,
+                            key: path.clone(),
+                            label: spec.label,
+                            map_size: spec.map_size,
+                            max_dbs: spec.max_dbs,
+                            health: DbHealth::new(),
+                            gc_started: AtomicBool::new(false),
+                            dbi_lock: Mutex::new(()),
+                        });
+                        pool.insert(path.clone(), Arc::downgrade(&entry));
+                        drop(pool);
+                        let shared = Self(entry);
+
+                        match shared.clear_stale_readers() {
+                            Ok(cleared_count) if cleared_count > 0 => {
+                                tracing::info!(
+                                    cleared_count,
+                                    db = spec.label,
+                                    "reclaimed stale LMDB reader slots at open"
+                                );
+                            }
+                            Ok(_) => {}
+                            Err(e) => {
+                                tracing::debug!("clear_stale_readers at open failed: {e}")
+                            }
+                        }
+
+                        return Ok(shared);
+                    }
+                    Err(heed::Error::EnvAlreadyOpened) => open_failed = true,
+                    // special handling cause we know this happens randomly
+                    Err(e)
+                        if is_transient_env_open_error(&e)
+                            && transient_retries < MAX_TRANSIENT_RETRIES =>
+                    {
+                        transient_retries += 1;
+                        tracing::debug!(
+                            path = %path.display(),
+                            transient_retries,
+                            error = ?e,
+                            "transient LMDB env open error, retrying"
+                        );
+                    }
+                    Err(e) => {
+                        return Err(Error::EnvOpen {
+                            db: spec.label,
+                            source: e,
+                        });
+                    }
+                }
+            }
+
+            if open_failed {
+                close_waits += 1;
+                if close_waits > MAX_CLOSE_WAITS {
+                    return Err(Error::EnvOpen {
+                        db: spec.label,
+                        source: heed::Error::EnvAlreadyOpened,
+                    });
+                }
+
+                match heed::env_closing_event(&path) {
+                    Some(event) => {
+                        event.wait_timeout(CLOSE_WAIT);
+                    }
+                    None => thread::sleep(Duration::from_millis(2)),
+                }
+            } else {
+                thread::sleep(TRANSIENT_RETRY_SLEEP);
+            }
+        }
+    }
+
+    pub(crate) fn health(&self) -> &DbHealth {
+        &self.0.health
+    }
+
+    // First caller wins: GC runs once per opened env, not once per tracker.
+    pub(crate) fn try_start_gc(&self) -> bool {
+        !self.0.gc_started.swap(true, Ordering::AcqRel)
+    }
+
+    // LMDB forbids mdb_dbi_open from concurrent txns in the same process.
+    pub(crate) fn lock_dbi_open(&self) -> MutexGuard<'_, ()> {
+        self.0
+            .dbi_lock
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+    }
+
+    pub(crate) fn destroy(&self) -> Result<Option<heed::EnvClosingEvent>> {
+        let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+        let holders = Arc::strong_count(&self.0);
+
+        if holders > 1 {
+            return Err(Error::DbInUse {
+                db: self.0.label,
+                path: self.0.key.clone(),
+                holders: holders - 1,
+            });
+        }
+
+        pool.remove(&self.0.key);
+        Ok(heed::env_closing_event(&self.0.key))
+    }
+}
+
+static POOL: LazyLock<Mutex<HashMap<PathBuf, Weak<PooledEnv>>>> = LazyLock::new(Mutex::default);
+
+const CLOSE_WAIT: Duration = Duration::from_millis(100);
+const MAX_CLOSE_WAITS: u32 = 100;
+const TRANSIENT_RETRY_SLEEP: Duration = Duration::from_millis(50);
+const MAX_TRANSIENT_RETRIES: u32 = 8;
+
+// Concurrent mdb_env_open calls on the same path can race on macOS
+// this is for some reason fixable by simple retry of the open
+// heed's default reader table is 126 slots. In TLS mode each thread pins a slot
+// for its lifetime, so long-lived embedders (Neovim, node agents) that share one
+// lock file across many processes/threads exhaust it (#783). Reader slots are
+// tiny (~64B), so raise the ceiling; `FFF_LMDB_MAX_READERS` lets hosts tune it.
+const DEFAULT_MAX_READERS: u32 = 1024;
+
+fn max_readers() -> u32 {
+    parse_max_readers(std::env::var("FFF_LMDB_MAX_READERS").ok())
+}
+
+// Never drop below heed's default 126; ignore missing/garbage/too-small values.
+fn parse_max_readers(raw: Option<String>) -> u32 {
+    raw.and_then(|v| v.trim().parse::<u32>().ok())
+        .filter(|&n| n >= 126)
+        .unwrap_or(DEFAULT_MAX_READERS)
+}
+
+fn is_transient_env_open_error(err: &heed::Error) -> bool {
+    match err {
+        heed::Error::Io(io) => matches!(
+            io.kind(),
+            std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound
+        ),
+        _ => false,
+    }
+}
+
+fn erase_if_oversized(db_path: &Path, spec: &EnvSpec) {
+    let data = db_path.join("data.mdb");
+    let Ok(meta) = fs::metadata(&data) else {
+        return;
+    };
+
+    if meta.len() <= spec.size_cap_bytes {
+        return;
+    }
+
+    tracing::error!(
+        path = %db_path.display(),
+        size = meta.len(),
+        cap = spec.size_cap_bytes,
+        "LMDB db exceeds size cap, erasing"
+    );
+    let _ = fs::remove_file(&data);
+    let _ = fs::remove_file(db_path.join("lock.mdb"));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DEFAULT_MAX_READERS, parse_max_readers};
+
+    #[test]
+    fn max_readers_parsing() {
+        assert_eq!(parse_max_readers(None), DEFAULT_MAX_READERS);
+        assert_eq!(parse_max_readers(Some("nan".into())), DEFAULT_MAX_READERS);
+        assert_eq!(parse_max_readers(Some("64".into())), DEFAULT_MAX_READERS); // below 126 floor
+        assert_eq!(parse_max_readers(Some(" 512 ".into())), 512);
+        assert_eq!(parse_max_readers(Some("126".into())), 126);
+    }
+}

crates/fff-core/src/dbs/frecency.rs

@@ -1,13 +1,11 @@
 use super::db_healthcheck::DbHealthChecker;
-use super::lmdb::{LmdbStore, is_map_full};
+use super::env_pool::SharedEnv;
 use crate::error::{Error, Result};
 use crate::file_picker::FFFMode;
 use crate::git::is_modified_status;
-use crate::shared::SharedFrecency;
+use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
+use heed::Database;
 use heed::types::{Bytes, SerdeBincode};
-use heed::{Database, Env};
-use std::fs;
-use std::path::PathBuf;
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::{collections::VecDeque, path::Path};
 
@@ -22,8 +20,9 @@ const AI_MAX_HISTORY_DAYS: f64 = 7.0; // Only consider accesses within 7 days
 
 #[derive(Debug)]
 pub struct FrecencyTracker {
-    env: Env,
+    env: SharedEnv,
     db: Database<Bytes, SerdeBincode<VecDeque<u64>>>,
+    health: DbHealth,
 }
 
 const MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [
@@ -44,26 +43,57 @@ const AI_MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [
 ];
 
 impl DbHealthChecker for FrecencyTracker {
-    fn get_env(&self) -> &heed::Env {
+    fn get_env(&self) -> &heed::Env<heed::WithoutTls> {
         &self.env
     }
 
+    fn is_healthy(&self) -> bool {
+        self.health.is_healthy()
+    }
+
     fn count_entries(&self) -> Result<Vec<(&'static str, u64)>> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
-        let count = self.db.len(&rtxn).map_err(Error::DbRead)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count = self.db.len(&rtxn).map_err(|source| Error::DbRead {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok(vec![("absolute_frecency_entries", count)])
     }
 }
 
 impl LmdbStore for FrecencyTracker {
-    const MAX_DBS: u32 = 0;
+    const LABEL: &'static str = "frecency";
     // 10 MiB hard ceiling. Owner's db after years of use is ~560 KiB, so this
     // leaves ~18× headroom while capping runaway growth (see GH issue #437).
     const MAP_SIZE: usize = 10 * 1024 * 1024;
+    const MAX_DBS: u32 = 0;
     // Nuke the db when it exceeds 8 MiB on disk — leaves a small margin under
     // MAP_SIZE so we don't hit MDB_MAP_FULL before the open-time erase fires.
-    const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
+    const SIZE_CAP_BYTES: u64 = 12 * 1024 * 1024;
+
+    fn shared_env(&self) -> &SharedEnv {
+        &self.env
+    }
+
+    fn health(&self) -> &DbHealth {
+        &self.health
+    }
+
+    fn purge_stale_data(env: &SharedEnv) -> Result<()> {
+        let (deleted, pruned) = Self::purge_stale_entries(env)?;
+        if deleted > 0 || pruned > 0 {
+            tracing::info!(deleted, pruned, "Frecency GC purged entries");
+        }
+
+        Ok(())
+    }
 }
 
 impl FrecencyTracker {
@@ -74,11 +104,10 @@ impl FrecencyTracker {
 
     pub fn open(db_path: impl AsRef<Path>) -> Result<Self> {
         let db_path = db_path.as_ref();
-        let env = Self::open_env(db_path)?;
+        let (env, health) = Self::open_env(db_path)?;
 
         let db = Self::open_database_safe(&env, None)?;
-
-        Ok(FrecencyTracker { db, env })
+        Ok(FrecencyTracker { db, env, health })
     }
 
     #[deprecated(
@@ -90,102 +119,41 @@ impl FrecencyTracker {
         Self::open(db_path)
     }
 
-    /// Spawns a background thread to purge stale frecency entries and compact the database.
-    /// Run it once in a while to purge old pages and keep DB file size reasonable.
-    ///
-    /// It's okay to not join this thread since it acquires locks for the db access
-    ///
-    /// ```
-    /// use fff_search::frecency::FrecencyTracker;
-    /// use fff_search::SharedFrecency;
-    /// let shared_frecency: SharedFrecency = Default::default();
-    /// let _ = FrecencyTracker::spawn_gc(shared_frecency, "/path/to/frecency_db".into()).ok();
-    /// ```
-    pub fn spawn_gc(
-        shared: SharedFrecency,
-        db_path: String,
-    ) -> Result<std::thread::JoinHandle<()>> {
-        Ok(std::thread::Builder::new()
-            .name("fff-frecency-gc".into())
-            .spawn(move || Self::run_frecency_gc(shared, db_path))?)
-    }
-
-    #[tracing::instrument(skip(shared), fields(db_path = %db_path))]
-    fn run_frecency_gc(shared: SharedFrecency, db_path: String) {
-        let start = std::time::Instant::now();
-
-        let (deleted, pruned) = {
-            let guard = match shared.read() {
-                Ok(g) => g,
-                Err(e) => {
-                    tracing::debug!("Failed to acquire read lock: {e}");
-                    return;
-                }
-            };
-            let Some(ref tracker) = *guard else {
-                return;
-            };
-
-            // Clear stale readers here (on a background thread) rather than in
-            // open_env — clear_stale_readers needs the writer mutex which can
-            // block indefinitely on a stuck lock if called on the main thread.
-            if let Err(e) = tracker.env.clear_stale_readers() {
-                tracing::debug!("clear_stale_readers failed: {e}");
-            }
-
-            match tracker.purge_stale_entries() {
-                Ok(result) => result,
-                Err(e) => {
-                    tracing::debug!("Purge failed: {e}");
-                    return;
-                }
-            }
-        };
-
-        if deleted > 0 || pruned > 0 {
-            tracing::info!(deleted, pruned, elapsed = ?start.elapsed(), "Frecency GC purged entries");
-        }
-
-        let data_path = PathBuf::from(&db_path).join("data.mdb");
-        let file_size = fs::metadata(&data_path).map(|m| m.len()).unwrap_or(0);
-        if file_size > <Self as LmdbStore>::SIZE_CAP_BYTES {
-            tracing::warn!(
-                size = file_size,
-                cap = <Self as LmdbStore>::SIZE_CAP_BYTES,
-                "Frecency DB exceeds size cap — will be erased on next open"
-            );
-        }
-    }
-
     /// Removes entries where all timestamps are older than MAX_HISTORY_DAYS,
     /// and prunes stale timestamps from entries that still have recent ones.
     /// Returns (deleted_count, pruned_count).
-    fn purge_stale_entries(&self) -> Result<(usize, usize)> {
-        let now = self.get_now();
+    fn purge_stale_entries(env: &SharedEnv) -> Result<(usize, usize)> {
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
         let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64);
 
-        // Collect entries to delete or update
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let db: Database<Bytes, SerdeBincode<VecDeque<u64>>> = Self::open_database_safe(env, None)?;
+
+        let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
+            db: Self::LABEL,
+            source,
+        })?;
         let mut to_delete: Vec<Vec<u8>> = Vec::new();
         let mut to_update: Vec<(Vec<u8>, VecDeque<u64>)> = Vec::new();
 
-        let iter = self.db.iter(&rtxn).map_err(Error::DbRead)?;
+        let iter = db.iter(&rtxn).map_err(|source| Error::DbRead {
+            db: Self::LABEL,
+            source,
+        })?;
         for result in iter {
-            let (key, accesses) = result.map_err(Error::DbRead)?;
+            let (key, accesses) = result.map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
 
-            // Timestamps are chronologically ordered (oldest at front).
-            // Find the first timestamp that is still within the retention window.
+            // Timestamps chronologically ordered (oldest at front).
             let fresh_start = accesses.iter().position(|&ts| ts >= cutoff_time);
             match fresh_start {
-                None => {
-                    // All timestamps are stale — delete the entire entry
-                    to_delete.push(key.to_vec());
-                }
-                Some(0) => {
-                    // All timestamps are fresh — nothing to do
-                }
+                None => to_delete.push(key.to_vec()),
+                Some(0) => {}
                 Some(start) => {
-                    // Some timestamps are stale — keep only the fresh ones
                     let pruned: VecDeque<u64> = accesses.iter().skip(start).copied().collect();
                     to_update.push((key.to_vec(), pruned));
                 }
@@ -197,17 +165,29 @@ impl FrecencyTracker {
             return Ok((0, 0));
         }
 
-        // Apply all changes in a single write transaction
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = env.write_txn().map_err(|source| Error::DbStartWriteTxn {
+            db: Self::LABEL,
+            source,
+        })?;
+
         for key in &to_delete {
-            self.db.delete(&mut wtxn, key).map_err(Error::DbWrite)?;
+            db.delete(&mut wtxn, key).map_err(|source| Error::DbWrite {
+                db: Self::LABEL,
+                source,
+            })?;
         }
+
         for (key, accesses) in &to_update {
-            self.db
-                .put(&mut wtxn, key, accesses)
-                .map_err(Error::DbWrite)?;
+            db.put(&mut wtxn, key, accesses)
+                .map_err(|source| Error::DbWrite {
+                    db: Self::LABEL,
+                    source,
+                })?;
         }
-        wtxn.commit().map_err(Error::DbCommit)?;
+        wtxn.commit().map_err(|source| Error::DbCommit {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok((to_delete.len(), to_update.len()))
     }
@@ -215,9 +195,24 @@ impl FrecencyTracker {
     fn get_accesses(&self, path: &Path) -> Result<Option<VecDeque<u64>>> {
         let key_hash = Self::path_to_hash_bytes(path)?;
 
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
-        let result = self.db.get(&rtxn, &key_hash).map_err(Error::DbRead)?;
-        rtxn.commit().map_err(Error::DbCommit)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        let result = self
+            .db
+            .get(&rtxn, &key_hash)
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        rtxn.commit().map_err(|source| Error::DbCommit {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok(result)
     }
@@ -230,6 +225,16 @@ impl FrecencyTracker {
     }
 
     fn path_to_hash_bytes(path: &Path) -> Result<[u8; 32]> {
+        // On Windows, resolve to the canonical form (short-name/case/symlink)
+        // so the same file always hashes to one key regardless of how the
+        // caller spelled it. Falls back to the raw path when the file no
+        // longer exists (e.g. watcher delete events), so the op is never
+        // dropped. No-op on other platforms.
+        #[cfg(windows)]
+        let canonical: Option<std::path::PathBuf> = crate::path_utils::canonicalize(path).ok();
+        #[cfg(windows)]
+        let path: &Path = canonical.as_deref().unwrap_or(path);
+
         let Some(key) = path.to_str() else {
             return Err(Error::InvalidPath(path.to_path_buf()));
         };
@@ -245,6 +250,11 @@ impl FrecencyTracker {
         Ok(last.map(|ts| self.get_now().saturating_sub(ts)))
     }
 
+    /// Number of tracked access for file path
+    pub fn access_count(&self, path: &Path) -> Result<usize> {
+        Ok(self.get_accesses(path)?.map_or(0, |a| a.len()))
+    }
+
     pub fn track_access(&self, path: &Path) -> Result<()> {
         let key_hash = Self::path_to_hash_bytes(path)?;
         let mut accesses = self.get_accesses(path)?.unwrap_or_default();
@@ -265,9 +275,16 @@ impl FrecencyTracker {
         accesses.push_back(now);
         tracing::debug!(?path, accesses = accesses.len(), "Tracking access");
 
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
         if let Err(e) = self.db.put(&mut wtxn, &key_hash, &accesses) {
             if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on put");
                 tracing::error!(
                     ?path,
                     "Frecency DB hit MDB_MAP_FULL; dropping write — db will be \
@@ -275,19 +292,26 @@ impl FrecencyTracker {
                 );
                 return Ok(());
             }
-            return Err(Error::DbWrite(e));
+            return Err(Error::DbWrite {
+                db: Self::LABEL,
+                source: e,
+            });
         }
 
         wtxn.commit()
             .inspect_err(|e| {
                 if is_map_full(e) {
+                    self.health.mark_unhealthy("MDB_MAP_FULL on commit");
                     tracing::error!(
                         ?path,
                         "Frecency DB hit MDB_MAP_FULL on commit; dropping write"
                     );
                 }
             })
-            .map_err(Error::DbCommit)
+            .map_err(|source| Error::DbCommit {
+                db: Self::LABEL,
+                source,
+            })
     }
 
     pub fn get_access_score(&self, file_path: &Path, mode: FFFMode) -> i64 {
@@ -387,6 +411,15 @@ mod tests {
     use super::*;
     use crate::file_picker::FFFMode;
 
+    // A path that doesn't exist on disk must still hash (canonicalize fails on
+    // Windows → falls back to the raw string), so watcher delete events and
+    // raced files never drop their frecency op.
+    #[test]
+    fn hashes_nonexistent_path_without_error() {
+        let missing = Path::new("/this/path/definitely/does/not/exist/frecency_test_xyz");
+        assert!(FrecencyTracker::path_to_hash_bytes(missing).is_ok());
+    }
+
     fn calculate_test_frecency_score(access_timestamps: &[u64], current_time: u64) -> i64 {
         let mut total_frecency = 0.0;
 

crates/fff-core/src/dbs/lmdb.rs

@@ -1,29 +1,116 @@
-use std::fs;
+use heed::{Database, Env, WithoutTls};
 use std::path::Path;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::sync::atomic::{AtomicU8, Ordering};
 use std::thread;
-use std::time::Duration;
-
-use heed::{Database, Env, EnvOpenOptions};
 
+use super::env_pool::{EnvSpec, SharedEnv};
 use crate::error::{Error, Result};
 
 pub(crate) fn is_map_full(err: &heed::Error) -> bool {
     matches!(err, heed::Error::Mdb(heed::MdbError::MapFull))
 }
 
-// Concurrent `mdb_env_open` calls on the same path can race on macOS
-// this is for some reason fixabtly by simple retry of the open
-fn is_transient_env_open_error(err: &heed::Error) -> bool {
-    match err {
-        heed::Error::Io(io) => matches!(
-            io.kind(),
-            std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound
-        ),
-        _ => false,
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DbHealthState {
+    Pending = 0,
+    Healthy = 1,
+    Degraded = 2,
+}
+
+impl DbHealthState {
+    fn from_u8(v: u8) -> Self {
+        debug_assert!(v <= 2);
+
+        match v {
+            0 => Self::Pending,
+            1 => Self::Healthy,
+            _ => Self::Degraded,
+        }
     }
 }
 
-pub(crate) trait LmdbStore {
+#[derive(Debug, Clone, Default)]
+pub(crate) struct DbHealth(Arc<AtomicU8>);
+
+impl DbHealth {
+    pub(crate) fn new() -> Self {
+        Self(Arc::new(AtomicU8::new(DbHealthState::Pending as u8)))
+    }
+
+    pub(crate) fn is_healthy(&self) -> bool {
+        // Pending counts as unhealthy: if the GC thread never flipped to
+        // Healthy, something's wrong (deadlocked clear_stale_readers, stuck
+        // writer mutex, etc.) and we want that surfaced to the user.
+        DbHealthState::from_u8(self.0.load(Ordering::Acquire)) == DbHealthState::Healthy
+    }
+
+    pub(crate) fn mark_healthy(&self) {
+        let _ = self.0.compare_exchange(
+            DbHealthState::Pending as u8,
+            DbHealthState::Healthy as u8,
+            Ordering::AcqRel,
+            Ordering::Acquire,
+        );
+    }
+
+    pub(crate) fn mark_unhealthy(&self, reason: &'static str) {
+        let prev = self.0.swap(DbHealthState::Degraded as u8, Ordering::AcqRel);
+        if DbHealthState::from_u8(prev) != DbHealthState::Degraded {
+            tracing::error!(reason, "LMDB tracker marked unhealthy");
+        }
+    }
+}
+
+/// Spawns a background thread that is ensuring that the environment that was previously
+/// open is safe, accessible and doesn't have a corrupted lock.md file. If it does this thread will
+/// hang indefinitely but we will have the information that the database is in failure mode
+pub(crate) fn spawn_lmdb_gc<T: LmdbStore>(shared: Arc<RwLock<Option<T>>>) {
+    let thread_shared = shared.clone();
+    let spawn_result = thread::Builder::new()
+        .name("fff-lmdb-gc".into())
+        .spawn(move || {
+            // Holding a read guard blocks `destroy` / re-init's write
+            // guard until this thread finishes — natural serialization.
+            let guard = match thread_shared.read() {
+                Ok(g) => g,
+                Err(e) => {
+                    tracing::debug!("gc: read lock poisoned: {e}");
+                    return;
+                }
+            };
+            let Some(ref tracker) = *guard else {
+                return; // destroyed before we started
+            };
+            // Trackers attaching to an already-pooled env must not repeat the
+            // GC; the first opener's run flips the shared health flag.
+            if !tracker.shared_env().try_start_gc() {
+                return;
+            }
+
+            if let Err(e) = T::purge_stale_data(tracker.shared_env()) {
+                tracing::debug!("purge_stale_data failed: {e}");
+            }
+
+            tracker.health().mark_healthy();
+        });
+
+    if let Err(e) = spawn_result {
+        tracing::debug!(?e, "failed to spawn fff-lmdb-gc thread");
+        // No thread = mark healthy now so healthcheck isn't stuck Pending.
+        if let Ok(guard) = shared.read()
+            && let Some(ref tracker) = *guard
+        {
+            tracker.health().mark_healthy();
+        }
+    }
+}
+
+pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
+    /// Short label used to defferintiate different instances of this trait
+    const LABEL: &'static str;
     /// LMDB map size in bytes. Must be a multiple of the OS page size.
     const MAP_SIZE: usize;
     /// Number of named sub-databases. `0` for single-db envs.
@@ -31,89 +118,80 @@ pub(crate) trait LmdbStore {
     /// Hard cap on `data.mdb` size.
     const SIZE_CAP_BYTES: u64;
 
+    /// Borrow the pooled env handle shared by every tracker of this path.
+    fn shared_env(&self) -> &SharedEnv;
+    /// Borrow the health flag from the tracker.
+    fn health(&self) -> &DbHealth;
+
+    /// Borrow the raw heed env.
+    fn env(&self) -> &Env<WithoutTls> {
+        self.shared_env()
+    }
+
+    /// Override to purge stale rows, compact, etc. Default no-op. Runs on
+    /// the GC thread while a read lock is held against the shared handle,
+    /// so destroy / re-init naturally wait for it.
+    fn purge_stale_data(_env: &SharedEnv) -> Result<()> {
+        Ok(())
+    }
+
+    /// Open (or join) the process-shared LMDB env for `db_path`. The health
+    /// flag is per-env: the GC of the first opener flips it for everyone.
     #[tracing::instrument]
-    fn open_env(db_path: &Path) -> Result<Env> {
-        Self::erase_if_oversized(db_path);
-        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
-
-        const MAX_ATTEMPTS: u32 = 8;
-        let mut attempt = 0u32;
-        loop {
-            let result = unsafe {
-                let mut opts = EnvOpenOptions::new();
-                opts.map_size(Self::MAP_SIZE);
-                if Self::MAX_DBS > 0 {
-                    opts.max_dbs(Self::MAX_DBS);
-                }
-                opts.open(db_path)
-            };
-
-            match result {
-                Ok(env) => return Ok(env),
-                Err(e) if is_transient_env_open_error(&e) && attempt + 1 < MAX_ATTEMPTS => {
-                    attempt += 1;
-                    tracing::debug!(
-                        path = %db_path.display(),
-                        attempt,
-                        error = ?e,
-                        "transient LMDB env open error, retrying"
-                    );
-
-                    thread::sleep(Duration::from_millis(50));
-                }
-                Err(e) => return Err(Error::EnvOpen(e)),
-            }
-        }
+    fn open_env(db_path: &Path) -> Result<(SharedEnv, DbHealth)> {
+        let shared = SharedEnv::get_or_open(
+            db_path,
+            &EnvSpec {
+                label: Self::LABEL,
+                map_size: Self::MAP_SIZE,
+                max_dbs: Self::MAX_DBS,
+                size_cap_bytes: Self::SIZE_CAP_BYTES,
+            },
+        )?;
+        let health = shared.health().clone();
+        Ok((shared, health))
     }
 
     /// Open or create a database without blocking on the LMDB writer mutex
     /// when the database already exists.
-    fn open_database_safe<KC, DC>(env: &Env, name: Option<&str>) -> Result<Database<KC, DC>>
+    fn open_database_safe<KC, DC>(env: &SharedEnv, name: Option<&str>) -> Result<Database<KC, DC>>
     where
         KC: 'static,
         DC: 'static,
     {
-        let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
-        let maybe_db: Option<Database<KC, DC>> =
-            env.open_database(&rtxn, name).map_err(Error::DbOpen)?;
+        let db = Self::LABEL;
+        // mdb_dbi_open must not run from concurrent txns in this process.
+        let _dbi_guard = env.lock_dbi_open();
+
+        let rtxn = env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn { db, source })?;
+        let maybe_db: Option<Database<KC, DC>> = env
+            .open_database(&rtxn, name)
+            .map_err(|source| Error::DbOpen { db, source })?;
 
         // do not drop the DB here
-        rtxn.commit().map_err(Error::DbCommit)?;
+        rtxn.commit()
+            .map_err(|source| Error::DbCommit { db, source })?;
 
         match maybe_db {
-            Some(db) => Ok(db),
+            Some(handle) => Ok(handle),
             None => {
                 // First time: create the database (requires write lock).
                 // unfortunately this CAN be deadlocking and this is what we see happens
                 // if the other part of the code is segfaulting, so the only rule to prevent this
                 // write the good code mf, okay?
-                let mut wtxn = env.write_txn().map_err(Error::DbStartWriteTxn)?;
-                let db = env
+                let mut wtxn = env
+                    .write_txn()
+                    .map_err(|source| Error::DbStartWriteTxn { db, source })?;
+                let handle = env
                     .create_database(&mut wtxn, name)
-                    .map_err(Error::DbCreate)?;
+                    .map_err(|source| Error::DbCreate { db, source })?;
 
-                wtxn.commit().map_err(Error::DbCommit)?;
-                Ok(db)
+                wtxn.commit()
+                    .map_err(|source| Error::DbCommit { db, source })?;
+                Ok(handle)
             }
         }
     }
-
-    fn erase_if_oversized(db_path: &Path) {
-        let data = db_path.join("data.mdb");
-        let Ok(meta) = fs::metadata(&data) else {
-            return;
-        };
-        if meta.len() <= Self::SIZE_CAP_BYTES {
-            return;
-        }
-
-        tracing::error!(
-            path = %db_path.display(),
-            size = meta.len(),
-            cap = Self::SIZE_CAP_BYTES,
-            "LMDB db exceeds size cap, erasing"
-        );
-        let _ = fs::remove_file(&data);
-        let _ = fs::remove_file(db_path.join("lock.mdb"));
-    }
 }

crates/fff-core/src/dbs/mod.rs

@@ -1,4 +1,11 @@
-pub mod db_healthcheck;
-pub mod frecency;
+pub(crate) mod env_pool;
 pub(crate) mod lmdb;
+
+pub mod db_healthcheck;
+pub use db_healthcheck::{DbHealth, DbHealthChecker};
+
+pub mod frecency;
+pub use frecency::*;
+
 pub mod query_tracker;
+pub use query_tracker::*;

crates/fff-core/src/dbs/query_tracker.rs

@@ -1,6 +1,7 @@
 use super::db_healthcheck::DbHealthChecker;
-use super::lmdb::{LmdbStore, is_map_full};
+use super::env_pool::SharedEnv;
 use crate::error::Error;
+use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
 use heed::types::{Bytes, SerdeBincode};
 use heed::{Database, Env};
 use serde::{Deserialize, Serialize};
@@ -27,29 +28,55 @@ struct HistoryEntry {
 
 #[derive(Debug)]
 pub struct QueryTracker {
-    env: Env,
+    env: SharedEnv,
     // Database for (project_path, query) -> QueryMatchEntry mappings
     query_file_db: Database<Bytes, SerdeBincode<QueryMatchEntry>>,
     // Database for project_path -> VecDeque<HistoryEntry> mappings (file picker)
     query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
     // Database for project_path -> VecDeque<HistoryEntry> mappings (grep)
     grep_query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
+    health: DbHealth,
 }
 
 impl DbHealthChecker for QueryTracker {
-    fn get_env(&self) -> &Env {
+    fn get_env(&self) -> &Env<heed::WithoutTls> {
         &self.env
     }
 
-    fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+    fn is_healthy(&self) -> bool {
+        self.health.is_healthy()
+    }
 
-        let count_queries = self.query_file_db.len(&rtxn).map_err(Error::DbRead)?;
-        let count_histories = self.query_history_db.len(&rtxn).map_err(Error::DbRead)?;
-        let count_grep_histories = self
-            .grep_query_history_db
+    fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+
+        let count_queries = self
+            .query_file_db
             .len(&rtxn)
-            .map_err(Error::DbRead)?;
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count_histories = self
+            .query_history_db
+            .len(&rtxn)
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count_grep_histories =
+            self.grep_query_history_db
+                .len(&rtxn)
+                .map_err(|source| Error::DbRead {
+                    db: Self::LABEL,
+                    source,
+                })?;
 
         Ok(vec![
             ("query_file_entries", count_queries),
@@ -60,12 +87,19 @@ impl DbHealthChecker for QueryTracker {
 }
 
 impl LmdbStore for QueryTracker {
+    const LABEL: &'static str = "query";
     // 10 MiB hard ceiling. Same reasoning as FrecencyTracker (GH issue #437).
     const MAP_SIZE: usize = 10 * 1024 * 1024;
     const MAX_DBS: u32 = 16;
-    // Nuke at 4 MiB — query history is bounded per-project but query→file
-    // associations grow unbounded over typing time.
-    const SIZE_CAP_BYTES: u64 = 4 * 1024 * 1024;
+    const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
+
+    fn shared_env(&self) -> &SharedEnv {
+        &self.env
+    }
+
+    fn health(&self) -> &DbHealth {
+        &self.health
+    }
 }
 
 impl QueryTracker {
@@ -76,7 +110,7 @@ impl QueryTracker {
 
     pub fn open(db_path: impl AsRef<Path>) -> Result<Self, Error> {
         let db_path = db_path.as_ref();
-        let env = Self::open_env(db_path)?;
+        let (env, health) = Self::open_env(db_path)?;
 
         let query_file_db = Self::open_database_safe(&env, Some("query_file_associations"))?;
         let query_history_db = Self::open_database_safe(&env, Some("query_history"))?;
@@ -87,6 +121,7 @@ impl QueryTracker {
             query_file_db,
             query_history_db,
             grep_query_history_db,
+            health,
         })
     }
 
@@ -137,7 +172,10 @@ impl QueryTracker {
     ) -> Result<(), Error> {
         let mut history = db
             .get(wtxn, project_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_default();
 
         history.push_back(HistoryEntry {
@@ -149,7 +187,10 @@ impl QueryTracker {
         }
 
         db.put(wtxn, project_key, &history)
-            .map_err(Error::DbWrite)?;
+            .map_err(|source| Error::DbWrite {
+                db: Self::LABEL,
+                source,
+            })?;
         Ok(())
     }
 
@@ -157,15 +198,21 @@ impl QueryTracker {
     /// offset=0 returns most recent, offset=1 returns 2nd most recent, etc.
     fn read_history_at_offset(
         db: &Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
-        env: &Env,
+        env: &Env<heed::WithoutTls>,
         project_key: &[u8; 32],
         offset: usize,
     ) -> Result<Option<String>, Error> {
-        let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
+            db: Self::LABEL,
+            source,
+        })?;
 
         let mut history = db
             .get(&rtxn, project_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_default();
 
         // history is FIFO, last element is most recent
@@ -188,12 +235,21 @@ impl QueryTracker {
         let file_path_buf = file_path.to_path_buf();
 
         let query_key = Self::create_query_key(project_path, query)?;
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         let mut entry = self
             .query_file_db
             .get(&wtxn, &query_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_else(|| QueryMatchEntry {
                 file_path: file_path_buf.clone(),
                 open_count: 0,
@@ -225,6 +281,7 @@ impl QueryTracker {
 
         if let Err(e) = self.query_file_db.put(&mut wtxn, &query_key, &entry) {
             if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on put");
                 tracing::error!(
                     ?query,
                     "Query tracker DB hit MDB_MAP_FULL; dropping write — db will \
@@ -232,7 +289,10 @@ impl QueryTracker {
                 );
                 return Ok(());
             }
-            return Err(Error::DbWrite(e));
+            return Err(Error::DbWrite {
+                db: Self::LABEL,
+                source: e,
+            });
         }
 
         // Update query history database
@@ -240,9 +300,12 @@ impl QueryTracker {
         if let Err(e) =
             Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now)
         {
-            if let Error::DbWrite(ref inner) = e
+            if let Error::DbWrite {
+                source: ref inner, ..
+            } = e
                 && is_map_full(inner)
             {
+                self.health.mark_unhealthy("MDB_MAP_FULL on history append");
                 tracing::error!(?query, "Query tracker DB map full while appending history");
                 return Ok(());
             }
@@ -251,10 +314,14 @@ impl QueryTracker {
 
         if let Err(e) = wtxn.commit() {
             if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on commit");
                 tracing::error!(?query, "Query tracker DB map full on commit");
                 return Ok(());
             }
-            return Err(Error::DbCommit(e));
+            return Err(Error::DbCommit {
+                db: Self::LABEL,
+                source: e,
+            });
         }
 
         tracing::debug!(?query, ?file_path, "Tracked query completion");
@@ -268,12 +335,21 @@ impl QueryTracker {
         min_combo_count: u32,
     ) -> Result<Option<QueryMatchEntry>, Error> {
         let query_key = Self::create_query_key(project_path, query)?;
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         let last_match = self
             .query_file_db
             .get(&rtxn, &query_key)
-            .map_err(Error::DbRead)?;
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
 
         Ok(last_match.filter(|entry| entry.open_count >= min_combo_count))
     }
@@ -287,13 +363,21 @@ impl QueryTracker {
     ) -> Result<i32, Error> {
         let query_key = Self::create_query_key(project_path, query)?;
         tracing::debug!(?query_key, "HASH");
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         match self
             .query_file_db
             .get(&rtxn, &query_key)
-            .map_err(Error::DbRead)?
-        {
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })? {
             Some(entry) => {
                 // Check if the file path matches and return boost
                 if entry.file_path == file_path && entry.open_count >= 2 {
@@ -322,7 +406,13 @@ impl QueryTracker {
     pub fn track_grep_query(&mut self, query: &str, project_path: &Path) -> Result<(), Error> {
         let now = self.get_now();
         let project_key = Self::create_project_key(project_path)?;
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         if let Err(e) = Self::append_to_history(
             &self.grep_query_history_db,
@@ -331,9 +421,13 @@ impl QueryTracker {
             query,
             now,
         ) {
-            if let Error::DbWrite(ref inner) = e
+            if let Error::DbWrite {
+                source: ref inner, ..
+            } = e
                 && is_map_full(inner)
             {
+                self.health
+                    .mark_unhealthy("MDB_MAP_FULL on grep history append");
                 tracing::error!(?query, "Grep query history DB map full; dropping write");
                 return Ok(());
             }
@@ -342,10 +436,14 @@ impl QueryTracker {
 
         if let Err(e) = wtxn.commit() {
             if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on commit");
                 tracing::error!(?query, "Grep query history DB map full on commit");
                 return Ok(());
             }
-            return Err(Error::DbCommit(e));
+            return Err(Error::DbCommit {
+                db: Self::LABEL,
+                source: e,
+            });
         }
 
         tracing::debug!(?query, "Tracked grep query");

crates/fff-core/src/error.rs

@@ -26,26 +26,79 @@ pub enum Error {
         path: std::path::PathBuf,
         source: std::io::Error,
     },
-    #[error("Failed to open frecency database env: {0}")]
-    EnvOpen(#[source] heed::Error),
-    #[error("Failed to create frecency database: {0}")]
-    DbCreate(#[source] heed::Error),
-    #[error("Failed to open frecency database: {0}")]
-    DbOpen(#[source] heed::Error),
-    #[error("Failed to clear stale readers for frecency database: {0}")]
-    DbClearStaleReaders(#[source] heed::Error),
+    #[error("Something is wrong with the local db instance: {0}")]
+    GenericDbError(#[from] heed::Error),
+    #[error("Failed to open {db} database env: {source}")]
+    EnvOpen {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error(
+        "LMDB env at {path} is already open as the '{open_as}' database with different options; requested by '{requested_as}'. Use a distinct path per database."
+    )]
+    EnvSpecMismatch {
+        path: std::path::PathBuf,
+        open_as: &'static str,
+        requested_as: &'static str,
+    },
+    #[error(
+        "The {db} database at {path} is still used by {holders} other tracker(s) in this process"
+    )]
+    DbInUse {
+        db: &'static str,
+        path: std::path::PathBuf,
+        holders: usize,
+    },
+    #[error("Failed to create {db} database: {source}")]
+    DbCreate {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to open {db} database: {source}")]
+    DbOpen {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to clear stale readers for {db} database: {source}")]
+    DbClearStaleReaders {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
 
-    #[error("Failed to start read transaction for frecency database: {0}")]
-    DbStartReadTxn(#[source] heed::Error),
-    #[error("Failed to start write transaction for frecency database: {0}")]
-    DbStartWriteTxn(#[source] heed::Error),
-
-    #[error("Failed to read from frecency database: {0}")]
-    DbRead(#[source] heed::Error),
-    #[error("Failed to write to frecency database: {0}")]
-    DbWrite(#[source] heed::Error),
-    #[error("Failed to commit write transaction to frecency database: {0}")]
-    DbCommit(#[source] heed::Error),
+    #[error("Failed to start read transaction for {db} database: {source}")]
+    DbStartReadTxn {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to start write transaction for {db} database: {source}")]
+    DbStartWriteTxn {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to read from {db} database: {source}")]
+    DbRead {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to write to {db} database: {source}")]
+    DbWrite {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to commit write transaction to {db} database: {source}")]
+    DbCommit {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
     #[error("Failed to start file system watcher: {0}")]
     FileSystemWatch(#[from] notify::Error),
 
@@ -54,6 +107,24 @@ pub enum Error {
 
     #[error("libgit2 error occurred: {0}")]
     Git(#[from] git2::Error),
+
+    #[error("Filesystem walk failed: {0}")]
+    WalkFailed(String),
+
+    #[error("Invalid glob pattern '{pattern}': {reason}")]
+    InvalidGlobPattern { pattern: String, reason: String },
+
+    #[error("File system watching is disabled for this picker")]
+    WatcherDisabled,
+
+    #[error("File system watcher is not ready")]
+    WatcherNotReady,
+
+    #[error("Indexed base path changed while creating the watch subscription")]
+    WatchBaseChanged,
+
+    #[error("Failed to start watch callback dispatcher: {0}")]
+    WatchDispatcherStart(#[source] std::io::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;

crates/fff-core/src/git.rs

@@ -5,7 +5,6 @@ use std::{
     fmt::Debug,
     path::{Path, PathBuf},
 };
-use tracing::debug;
 
 pub(crate) fn default_status_options() -> StatusOptions {
     let mut opts = StatusOptions::new();
@@ -16,6 +15,20 @@ pub(crate) fn default_status_options() -> StatusOptions {
     opts
 }
 
+/// Status options for the initial scan / rescan.
+///
+/// Skips `include_unmodified` because every `FileItem` starts with
+/// `git_status: None` (== clean), so a missing cache entry already means
+/// "clean" — no need to ask libgit2 to enumerate every tracked path.
+/// Saves seconds on huge dirty trees (e.g. chromium with 400k+ entries).
+pub(crate) fn initial_scan_status_options() -> StatusOptions {
+    let mut opts = StatusOptions::new();
+    opts.include_untracked(true)
+        .recurse_untracked_dirs(true)
+        .exclude_submodules(true);
+    opts
+}
+
 #[derive(Debug, Clone, Default)]
 pub(crate) struct GitStatusCache(AHashMap<PathBuf, Status>);
 
@@ -49,7 +62,7 @@ impl GitStatusCache {
 
         let mut entries = AHashMap::with_capacity(statuses.len());
         for entry in &statuses {
-            if let Some(entry_path) = entry.path() {
+            if let Ok(entry_path) = entry.path() {
                 // libgit2 returns entry paths with forward slashes on every platform
                 // fff stores native paths - meaning we have forward slash issue on windows
                 let full_path = crate::path_utils::normalize(repo_path.join(entry_path));
@@ -79,7 +92,7 @@ impl GitStatusCache {
         }
     }
 
-    #[tracing::instrument(skip(repo), level = tracing::Level::DEBUG)]
+    #[tracing::instrument(skip(repo), fields(paths_count = paths.len()), level = tracing::Level::DEBUG)]
     pub fn git_status_for_paths<TPath: AsRef<Path> + Debug>(
         repo: &Repository,
         paths: &[TPath],
@@ -111,11 +124,6 @@ impl GitStatusCache {
         }
 
         let git_status_cache = Self::read_status_impl(repo, &mut status_options)?;
-        debug!(
-            status_len = git_status_cache.statuses_len(),
-            "Multiple files git status"
-        );
-
         Ok(git_status_cache)
     }
 }

crates/fff-core/src/git_status_worker.rs

@@ -0,0 +1,120 @@
+use crate::shared::{SharedFrecency, WeakFilePicker};
+use ahash::AHashSet;
+use parking_lot::{Condvar, Mutex};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+// we don't really need a queue here
+#[derive(Default)]
+struct Pending {
+    paths: AHashSet<PathBuf>,
+    full_rescan: bool,
+    shutdown: bool,
+}
+
+impl Pending {
+    fn has_work(&self) -> bool {
+        self.full_rescan || !self.paths.is_empty()
+    }
+}
+
+/// Condvar based queue that is used for batch processing events
+pub(crate) struct GitStatusWorker {
+    state: Mutex<Pending>,
+    cv: Condvar,
+    consumer_spawned: AtomicBool,
+}
+
+impl GitStatusWorker {
+    pub(crate) fn new() -> Arc<Self> {
+        Arc::new(Self {
+            state: Mutex::new(Pending::default()),
+            cv: Condvar::new(),
+            consumer_spawned: AtomicBool::new(false),
+        })
+    }
+
+    pub(crate) fn spawn_once(
+        self: &Arc<Self>,
+        weak_picker: WeakFilePicker,
+        frecency: SharedFrecency,
+    ) {
+        if self
+            .consumer_spawned
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+        {
+            Self::spawn_consumer(Arc::clone(self), weak_picker, frecency);
+        }
+    }
+
+    pub(crate) fn enqueue_paths<I>(&self, paths: I)
+    where
+        I: IntoIterator<Item = PathBuf>,
+    {
+        let mut guard = self.state.lock();
+        guard.paths.extend(paths);
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    pub(crate) fn request_full_rescan(&self) {
+        let mut guard = self.state.lock();
+        guard.full_rescan = true;
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    pub(crate) fn signal_shutdown(&self) {
+        let mut guard = self.state.lock();
+        guard.shutdown = true;
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    fn wait_and_take(&self) -> Option<Pending> {
+        let mut guard = self.state.lock();
+        while !guard.shutdown && !guard.has_work() {
+            self.cv.wait(&mut guard);
+        }
+        if guard.shutdown {
+            return None;
+        }
+        Some(std::mem::take(&mut *guard))
+    }
+
+    // the problem: git status update can take a lot of time especially on big repositories
+    // and there is unpredictable wait time on the lock file if huge commit is going so we have to
+    // spawn a separate thread to guartee that notify handler is unlocked even if git update takes a
+    // lot of time on every event burst (pretty cheap as this thread is going to sleep 99.9% of time)
+    fn spawn_consumer(
+        mailbox: Arc<GitStatusWorker>,
+        weak_picker: WeakFilePicker,
+        frecency: SharedFrecency,
+    ) {
+        let _ = std::thread::Builder::new()
+            .name("fff-git-status".into())
+            .spawn(move || {
+                while let Some(work) = mailbox.wait_and_take() {
+                    let Some(picker) = weak_picker.upgrade() else {
+                        break;
+                    };
+
+                    if work.full_rescan {
+                        if let Err(e) = picker.refresh_git_status(&frecency) {
+                            tracing::error!("git-status worker: full rescan failed: {e:?}");
+                        }
+                    } else if !work.paths.is_empty() {
+                        let paths: Vec<PathBuf> = work.paths.into_iter().collect();
+                        if let Err(e) = picker.update_git_status_for_paths(&paths, &frecency) {
+                            tracing::error!("git-status worker: path update failed: {e:?}");
+                        }
+                    }
+                }
+
+                tracing::info!("git-status worker stopped");
+            })
+            .inspect_err(|err| tracing::error!(?err, "Failed to spawn git status worker"));
+    }
+}

crates/fff-core/src/grep/classify.rs

@@ -0,0 +1,131 @@
+//! Definition and import line classification (vibe coded POC)
+//!
+//! Byte-level heuristics that tag a matched line as a code definition
+//! (`struct`, `fn`, `class`, …) or an import/use statement. Used to
+//! rank/annotate grep results for AI/MCP consumers. Gated behind the
+//! `definitions` feature since only such consumers need it.
+
+/// Detect if a line looks like a code definition (struct, fn, class, etc.)
+pub fn is_definition_line(line: &str) -> bool {
+    let s = line.trim_start().as_bytes();
+    let s = skip_modifiers(s);
+    is_definition_keyword(s)
+}
+
+/// Modifier keywords that can precede a definition keyword.
+/// Each must be followed by whitespace to be consumed.
+const MODIFIERS: &[&[u8]] = &[
+    b"pub",
+    b"export",
+    b"default",
+    b"async",
+    b"abstract",
+    b"unsafe",
+    b"static",
+    b"protected",
+    b"private",
+    b"public",
+];
+
+/// Definition keywords to detect.
+const DEF_KEYWORDS: &[&[u8]] = &[
+    b"struct",
+    b"fn",
+    b"enum",
+    b"trait",
+    b"impl",
+    b"class",
+    b"interface",
+    b"function",
+    b"def",
+    b"func",
+    b"type",
+    b"module",
+    b"object",
+];
+
+/// Skip zero or more modifier keywords (including `pub(crate)` style visibility).
+fn skip_modifiers(mut s: &[u8]) -> &[u8] {
+    loop {
+        // Handle `pub(...)` — e.g. `pub(crate)`, `pub(super)`
+        if s.starts_with(b"pub(")
+            && let Some(end) = s.iter().position(|&b| b == b')')
+        {
+            s = skip_ws(&s[end + 1..]);
+            continue;
+        }
+        let mut matched = false;
+        for &kw in MODIFIERS {
+            if s.starts_with(kw) {
+                let rest = &s[kw.len()..];
+                if rest.first().is_some_and(|b| b.is_ascii_whitespace()) {
+                    s = skip_ws(rest);
+                    matched = true;
+                    break;
+                }
+            }
+        }
+        if !matched {
+            return s;
+        }
+    }
+}
+
+/// Check if `s` starts with a definition keyword followed by a word boundary.
+fn is_definition_keyword(s: &[u8]) -> bool {
+    for &kw in DEF_KEYWORDS {
+        if s.starts_with(kw) {
+            let after = s.get(kw.len());
+            // Word boundary: end of input, or next byte is not alphanumeric/underscore
+            if after.is_none_or(|b| !b.is_ascii_alphanumeric() && *b != b'_') {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Skip ASCII whitespace.
+#[inline]
+fn skip_ws(s: &[u8]) -> &[u8] {
+    let n = s
+        .iter()
+        .position(|b| !b.is_ascii_whitespace())
+        .unwrap_or(s.len());
+    &s[n..]
+}
+
+/// Detect import/use lines — lower value than definitions or usages.
+///
+/// Checks if the line (after leading whitespace) starts with a common
+/// import statement prefix. Pure byte-level checks, no regex.
+pub fn is_import_line(line: &str) -> bool {
+    let s = line.trim_start().as_bytes();
+    s.starts_with(b"import ")
+        || s.starts_with(b"import\t")
+        || (s.starts_with(b"from ") && s.get(5).is_some_and(|&b| b == b'\'' || b == b'"'))
+        || s.starts_with(b"use ")
+        || s.starts_with(b"use\t")
+        || starts_with_require(s)
+        || starts_with_include(s)
+}
+
+/// Match `require(` or `require (`.
+#[inline]
+fn starts_with_require(s: &[u8]) -> bool {
+    if !s.starts_with(b"require") {
+        return false;
+    }
+    let rest = &s[b"require".len()..];
+    rest.first() == Some(&b'(') || (rest.first() == Some(&b' ') && rest.get(1) == Some(&b'('))
+}
+
+/// Match `# include ` (with optional spaces after `#`).
+#[inline]
+fn starts_with_include(s: &[u8]) -> bool {
+    if s.first() != Some(&b'#') {
+        return false;
+    }
+    let rest = skip_ws(&s[1..]);
+    rest.starts_with(b"include ") || rest.starts_with(b"include\t")
+}

crates/fff-core/src/grep/fuzzy_grep.rs

@@ -0,0 +1,358 @@
+use crate::simd_path::ArenaPtr;
+use crate::types::{ContentCacheBudget, FileItem, MmapSlot};
+use fff_grep::lines::LineStep;
+use rayon::prelude::*;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use super::sink::{
+    char_indices_to_byte_offsets, classify_definition, strip_line_terminators,
+    truncate_display_bytes,
+};
+use super::types::{GrepMatch, GrepResult, GrepSearchOptions};
+
+#[allow(clippy::too_many_arguments)]
+pub(super) fn fuzzy_grep_search<'a>(
+    grep_text: &str,
+    files_to_search: &[&'a FileItem],
+    options: &GrepSearchOptions,
+    total_files: usize,
+    filtered_file_count: usize,
+    case_insensitive: bool,
+    budget: &ContentCacheBudget,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> GrepResult<'a> {
+    // max_typos controls how many *needle* characters can be unmatched.
+    // A transposition (e.g. "shcema" -> "schema") costs ~1 typo with
+    // default gap penalties. We scale max_typos by needle length:
+    //   1-2 chars -> 0 typos (exact subsequence only)
+    //   3-5 chars -> 1 typo
+    //   6+  chars -> 2 typos
+    // Cap at 2: higher values (3+) let the SIMD prefilter pass lines
+    // missing key characters entirely (e.g. query "flvencodeX" matching
+    // lines without 'l' or 'v'). Quality comes from the post-match filters.
+    let max_typos = (grep_text.len() / 3).min(2);
+    let scoring = neo_frizbee::Scoring {
+        // Use default gap penalties. Higher values (e.g. 20) cause
+        // smith-waterman to prefer *dropping needle chars* over paying
+        // gap costs, which inflates the typo count and breaks
+        // transposition matching ("shcema" -> "schema" becomes 3 typos instead of 1)
+        exact_match_bonus: 100,
+        // gap_open_penalty: 4,
+        // gap_extend_penalty: 2,
+        prefix_bonus: 0,
+        capitalization_bonus: if case_insensitive { 0 } else { 4 },
+        ..neo_frizbee::Scoring::default()
+    };
+
+    let matcher = neo_frizbee::Matcher::new(
+        grep_text,
+        &neo_frizbee::Config {
+            // Use the real max_typos so frizbee's SIMD prefilter actually rejects non-matching lines (~2 SIMD instructions per line vs full SW scoring).
+            max_typos: Some(max_typos as u16),
+            sort: false,
+            scoring,
+            ..Default::default()
+        },
+    );
+
+    // Minimum score threshold: 50% of a perfect contiguous match.
+    // With default scoring (match_score=12, matching_case_bonus=4 = 16/char),
+    // a transposition costs ~5 from a gap, keeping the score well above 50%
+    let perfect_score = (grep_text.len() as u16) * 16;
+    let min_score = (perfect_score * 50) / 100;
+
+    // Target identifiers are often longer than the query due to delimiters
+    // (e.g. query "flvencodepicture" -> "ff_flv_encode_picture_header" from ffmpeg)
+    // Allow 3x needle length to accommodate underscore/dot-separated names
+    let max_match_span = grep_text.len() * 3;
+    let needle_len = grep_text.len();
+
+    // Each delimiter (_, .) in the target creates a gap. A typical C/Rust
+    // identifier like "ff_flv_encode_picture_header" has 4-5 underscores.
+    // Scale generously so delimiter gaps don't reject valid matches.
+    let max_gaps = (needle_len / 3).max(2);
+
+    // If a file doesn't contain enough distinct needle characters just skip it
+    let needle_bytes = grep_text.as_bytes();
+    let mut unique_needle_chars: Vec<u8> = Vec::new();
+    for &b in needle_bytes {
+        let lo = b.to_ascii_lowercase();
+        let hi = b.to_ascii_uppercase();
+        if !unique_needle_chars.contains(&lo) {
+            unique_needle_chars.push(lo);
+        }
+        if lo != hi && !unique_needle_chars.contains(&hi) {
+            unique_needle_chars.push(hi);
+        }
+    }
+
+    // How many distinct needle chars must appear in the file.
+    // With max_typos allowed, we need at least (unique_count - max_typos)
+    let unique_count = {
+        let mut seen = [false; 256];
+        for &b in needle_bytes {
+            seen[b.to_ascii_lowercase() as usize] = true;
+        }
+        seen.iter().filter(|&&v| v).count()
+    };
+    let min_chars_required = unique_count.saturating_sub(max_typos);
+
+    let time_budget = if options.time_budget_ms > 0 {
+        Some(std::time::Duration::from_millis(options.time_budget_ms))
+    } else {
+        None
+    };
+    let search_start = std::time::Instant::now();
+    let budget_exceeded = AtomicBool::new(false);
+    let max_matches_per_file = options.max_matches_per_file;
+
+    // for fuzzy match we need a bit smarter chunking as the amount of work we have to perform is
+    // exponentially larger than the original grep (and the nature of work is heavier), so in short we have to
+    // understand if the approximate index prefilter got us a lot of candidates or not
+    //
+    // if we have a few candidates -> likely we have a lot of matches, so verify the check faster
+    // if we have a lot of candidates -> rely on a larger chunk pipelining more parallel lines at once
+    let page_limit = options.page_limit;
+    let base_chunk = rayon::current_num_threads() * 4;
+    let prefilter_strong = total_files > 0 && files_to_search.len() * 2 < total_files;
+    let max_chunk = if prefilter_strong {
+        base_chunk
+    } else {
+        (base_chunk * 256).max(8 * 1024)
+    };
+
+    let growth = if prefilter_strong { 1 } else { 2 };
+    let mut chunk_size = base_chunk;
+    let mut chunk_start = 0;
+    let mut running_matches = 0usize;
+    let mut per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = Vec::new();
+
+    while chunk_start < files_to_search.len() {
+        let chunk_end = (chunk_start + chunk_size).min(files_to_search.len());
+        let chunk = &files_to_search[chunk_start..chunk_end];
+        let chunk_offset = chunk_start;
+        chunk_start = chunk_end;
+        chunk_size = (chunk_size * growth).min(max_chunk);
+
+        // Parallel phase with `map_init`: each rayon worker thread clones the
+        // matcher once and gets a reusable read buffer + mmap slot. Buffer holds
+        // small files, slot holds fresh mmap for cache-miss files ≥ FRESH_MMAP_THRESHOLD.
+        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
+            .par_iter()
+            .enumerate()
+            .map_init(
+                || {
+                    (
+                        matcher.clone(),
+                        Vec::with_capacity(64 * 1024),
+                        MmapSlot::default(),
+                    )
+                },
+                |(matcher, buf, mmap_slot), (local_idx, file)| {
+                    if abort_signal.load(Ordering::Relaxed) {
+                        budget_exceeded.store(true, Ordering::Relaxed);
+                        return None;
+                    }
+
+                    if let Some(budget) = time_budget
+                        && search_start.elapsed() > budget
+                    {
+                        budget_exceeded.store(true, Ordering::Relaxed);
+                        return None;
+                    }
+
+                    let file_arena = if file.is_overflow() {
+                        overflow_arena
+                    } else {
+                        arena
+                    };
+
+                    let file_bytes =
+                        file.get_content_for_search(buf, mmap_slot, file_arena, base_path, budget)?;
+
+                    if min_chars_required > 0 {
+                        let mut chars_found = 0usize;
+                        for &ch in &unique_needle_chars {
+                            if memchr::memchr(ch, file_bytes).is_some() {
+                                chars_found += 1;
+                                if chars_found >= min_chars_required {
+                                    break;
+                                }
+                            }
+                        }
+                        if chars_found < min_chars_required {
+                            return None;
+                        }
+                    }
+
+                    // Validate the whole file as UTF-8 once upfront. Source code
+                    // files are virtually always valid UTF-8; this single check
+                    // replaces per-line from_utf8 calls (~8% of fuzzy grep time)
+                    let file_is_utf8 = std::str::from_utf8(file_bytes).is_ok();
+
+                    let mut stepper = LineStep::new(b'\n', 0, file_bytes.len());
+                    let estimated_lines = (file_bytes.len() / 40).max(64);
+                    let mut file_lines: Vec<&str> = Vec::with_capacity(estimated_lines);
+                    let mut line_meta: Vec<(u64, u64)> = Vec::with_capacity(estimated_lines);
+
+                    let mut line_number: u64 = 1;
+                    while let Some(line_match) = stepper.next_match(file_bytes) {
+                        let byte_offset = line_match.start() as u64;
+                        let trimmed = strip_line_terminators(&file_bytes[line_match]);
+
+                        if !trimmed.is_empty() {
+                            // we know for sure that the file is UTF-8 at this point
+                            let line_str = if file_is_utf8 {
+                                unsafe { std::str::from_utf8_unchecked(trimmed) }
+                            } else if let Ok(s) = std::str::from_utf8(trimmed) {
+                                s
+                            } else {
+                                line_number += 1;
+                                continue;
+                            };
+                            file_lines.push(line_str);
+                            line_meta.push((line_number, byte_offset));
+                        }
+
+                        line_number += 1;
+                    }
+
+                    if file_lines.is_empty() {
+                        return None;
+                    }
+
+                    // Single-pass: score + indices in one Smith-Waterman run per line (not parallel)
+                    let matches_with_indices = matcher.match_list_indices(&file_lines);
+                    let mut file_matches: Vec<GrepMatch> = Vec::new();
+
+                    for mut match_indices in matches_with_indices {
+                        if match_indices.score < min_score {
+                            continue;
+                        }
+
+                        let idx = match_indices.index as usize;
+                        let raw_line = file_lines[idx];
+
+                        let truncated = truncate_display_bytes(raw_line.as_bytes());
+                        let display_line = if truncated.len() < raw_line.len() {
+                            // SAFETY: truncate_display_bytes preserves UTF-8 char boundaries
+                            &raw_line[..truncated.len()]
+                        } else {
+                            raw_line
+                        };
+
+                        // If the line was truncated, re-compute indices on the shorter string.
+                        if display_line.len() < raw_line.len() {
+                            let Some(re_indices) = matcher
+                                .match_list_indices(&[display_line])
+                                .into_iter()
+                                .next()
+                            else {
+                                continue;
+                            };
+                            match_indices = re_indices;
+                        }
+
+                        match_indices.indices.sort_unstable();
+
+                        // Minimum matched chars: at least (needle_len - max_typos)
+                        // characters must appear. This is consistent with the typo
+                        // budget: each typo can drop one needle char from the alignment.
+                        let min_matched = needle_len.saturating_sub(max_typos).max(1);
+                        if match_indices.indices.len() < min_matched {
+                            continue;
+                        }
+
+                        let indices = &match_indices.indices;
+
+                        if let (Some(&first), Some(&last)) = (indices.first(), indices.last()) {
+                            // reject widely scattered matches
+                            let span = last - first + 1;
+                            if span > max_match_span {
+                                continue;
+                            }
+
+                            // Density check: matched chars / span must be dense enough.
+                            // Relaxed for perfect subsequence matches (all needle chars
+                            // present), slightly relaxed for typo matches to handle
+                            // delimiter-heavy targets
+                            // (e.g. "ff_flv_encode_picture_header" has span inflated by underscores w/ density ~68%)
+                            let density = (indices.len() * 100) / span;
+                            let min_density = if indices.len() >= needle_len {
+                                45 // Perfect subsequence relaxed (delimiters inflate span)
+                            } else {
+                                65 // Has typos filter out a long string
+                            };
+                            if density < min_density {
+                                continue;
+                            }
+
+                            // Gap count check: count discontinuities in the indices
+                            let gap_count = indices.windows(2).filter(|w| w[1] != w[0] + 1).count();
+                            if gap_count > max_gaps {
+                                continue;
+                            }
+                        }
+
+                        let (ln, bo) = line_meta[idx];
+                        let match_byte_offsets =
+                            char_indices_to_byte_offsets(display_line, &match_indices.indices);
+                        let col = match_byte_offsets
+                            .first()
+                            .map(|r| r.0 as usize)
+                            .unwrap_or(0);
+
+                        file_matches.push(GrepMatch {
+                            file_index: 0,
+                            line_number: ln,
+                            col,
+                            byte_offset: bo,
+                            is_definition: classify_definition(
+                                options.classify_definitions,
+                                display_line,
+                            ),
+                            line_content: display_line.to_string(),
+                            match_byte_offsets,
+                            fuzzy_score: Some(match_indices.score),
+                            context_before: Vec::new(),
+                            context_after: Vec::new(),
+                        });
+
+                        if max_matches_per_file != 0 && file_matches.len() >= max_matches_per_file {
+                            break;
+                        }
+                    }
+
+                    if file_matches.is_empty() {
+                        return None;
+                    }
+
+                    Some((chunk_offset + local_idx, *file, file_matches))
+                },
+            )
+            .flatten()
+            .collect();
+
+        for result in chunk_results {
+            running_matches += result.2.len();
+            per_file_results.push(result);
+        }
+
+        if running_matches >= page_limit || budget_exceeded.load(Ordering::Relaxed) {
+            break;
+        }
+    }
+
+    GrepResult::collect(
+        per_file_results,
+        files_to_search.len(),
+        options,
+        total_files,
+        filtered_file_count,
+        budget_exceeded.load(Ordering::Relaxed),
+    )
+}

crates/fff-core/src/grep/grep.rs

@@ -0,0 +1,706 @@
+use super::prefilter::prefilter_with_filepath_retry;
+use super::regex::{RegexMatcher, RegexSink, build_regex};
+use super::sink::{SinkState, debug_assert_newline_terminator};
+use super::types::{GrepMatch, GrepMode, GrepResult, GrepSearchOptions};
+use crate::index::{
+    BigramFilter, BigramOverlay, bigram_boundary, fuzzy_candidates, literal_candidates,
+    regex_candidates,
+};
+use crate::simd_string_utils::memmem;
+use crate::types::{ContentCacheBudget, FileItem, FileSliceExt, MmapSlot};
+use fff_grep::{
+    Searcher, SearcherBuilder, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use fff_query_parser::{FFFQuery, GrepConfig, QueryParser};
+use rayon::prelude::*;
+use smallvec::SmallVec;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use tracing::Level;
+
+#[allow(clippy::large_enum_variant)]
+pub(super) enum NeedleFinder<'a> {
+    CaseSensitive(memchr::memmem::Finder<'a>),
+    /// Pre-lowered needle bytes for the SIMD case-insensitive search.
+    CaseInsensitive(&'a [u8]),
+}
+
+impl<'a> NeedleFinder<'a> {
+    fn new(needle: &'a [u8], case_insensitive: bool) -> Self {
+        if case_insensitive {
+            Self::CaseInsensitive(needle)
+        } else {
+            Self::CaseSensitive(memchr::memmem::Finder::new(needle))
+        }
+    }
+
+    #[inline]
+    fn find(&self, haystack: &[u8]) -> Option<usize> {
+        match self {
+            Self::CaseSensitive(finder) => finder.find(haystack),
+            Self::CaseInsensitive(needle_lower) => memmem::find(haystack, needle_lower),
+        }
+    }
+
+    #[inline]
+    fn needle(&self) -> &[u8] {
+        match self {
+            Self::CaseSensitive(finder) => finder.needle(),
+            Self::CaseInsensitive(needle_lower) => needle_lower,
+        }
+    }
+
+    /// Compare `haystack` against a slice of the needle with the same case
+    /// semantics as `find`.
+    #[inline]
+    fn eq_fold(&self, haystack: &[u8], needle_seg: &[u8]) -> bool {
+        match self {
+            Self::CaseSensitive(_) => haystack == needle_seg,
+            Self::CaseInsensitive(_) => {
+                haystack.len() == needle_seg.len() && memmem::find(haystack, needle_seg) == Some(0)
+            }
+        }
+    }
+
+    /// Collect highlight spans for every needle occurrence within a line.
+    /// The case branch is resolved once per line, not once per occurrence.
+    #[inline]
+    fn for_each_occurrence(&self, haystack: &[u8], mut on_match: impl FnMut(usize)) {
+        match self {
+            Self::CaseSensitive(finder) => {
+                let mut start_pos = 0usize;
+                while let Some(pos) = finder.find(&haystack[start_pos..]) {
+                    on_match(start_pos + pos);
+                    start_pos += pos + 1;
+                }
+            }
+            Self::CaseInsensitive(needle_lower) => {
+                let mut start_pos = 0usize;
+                while let Some(pos) = memmem::find(&haystack[start_pos..], needle_lower) {
+                    on_match(start_pos + pos);
+                    start_pos += pos + 1;
+                }
+            }
+        }
+    }
+}
+
+struct PlainTextMatcher<'a> {
+    finder: &'a NeedleFinder<'a>,
+}
+
+impl Matcher for PlainTextMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>, NoError> {
+        let hay = &haystack[at..];
+        let needle_len = self.finder.needle().len();
+
+        Ok(self
+            .finder
+            .find(hay)
+            .map(|pos| Match::new(at + pos, at + pos + needle_len)))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        Some(fff_grep::LineTerminator::byte(b'\n'))
+    }
+}
+
+struct PlainTextSink<'r> {
+    state: SinkState,
+    finder: &'r NeedleFinder<'r>,
+    pattern_len: u32,
+    multiline_segment_len: Option<usize>,
+}
+
+impl Sink for PlainTextSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(
+        &mut self,
+        searcher: &Searcher,
+        sink_match: &SinkMatch<'_>,
+    ) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = sink_match.bytes();
+        let (display_bytes, display_len, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, sink_match);
+
+        let line_content = String::from_utf8_lossy(display_bytes).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        if let Some(seg_len) = self.multiline_segment_len {
+            // Multiline needle: the match starts on this line, so the needle's
+            // first segment must be a suffix of the line. Highlight that suffix.
+            let seg = &self.finder.needle()[..seg_len];
+            if !seg.is_empty()
+                && display_bytes.len() >= seg.len()
+                && self
+                    .finder
+                    .eq_fold(&display_bytes[display_bytes.len() - seg.len()..], seg)
+            {
+                col = display_bytes.len() - seg.len();
+                match_byte_offsets.push((col as u32, display_len));
+            }
+        } else {
+            let pattern_len = self.pattern_len;
+            self.finder.for_each_occurrence(display_bytes, |pos| {
+                let abs_start = pos as u32;
+                let abs_end = (abs_start + pattern_len).min(display_len);
+                if first {
+                    col = pos;
+                    first = false;
+                }
+                match_byte_offsets.push((abs_start, abs_end));
+            });
+        }
+
+        let (context_before, context_after) = self.state.extract_context(sink_match);
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+/// Perform a grep search across all indexed files.
+///
+/// When `query` is empty, returns git-modified/untracked files sorted by
+/// frecency for the "welcome state" UI.
+#[tracing::instrument(skip_all, fields(file_count = files.len()))]
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn grep_search<'a>(
+    files: &'a [FileItem],
+    query: &FFFQuery<'_>,
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let result = grep_search_parsed(
+        files,
+        query,
+        options,
+        budget,
+        bigram_index,
+        bigram_overlay,
+        abort_signal,
+        base_path,
+        arena,
+        overflow_arena,
+    );
+
+    // Constraint parsing can swallow tokens the user meant literally (e.g. `!=`
+    // becoming an exclusion). If the constrained search scanned everything and
+    // found nothing, retry the whole raw query as literal text. This also holds
+    // for later pages: an empty full scan at offset 0 stays empty at any offset,
+    // so paging offsets consistently index the literal search's file list.
+    let full_scan_empty = result.matches.is_empty() && result.next_file_offset == 0;
+    if !full_scan_empty || query.constraints.is_empty() || abort_signal.load(Ordering::Relaxed) {
+        return result;
+    }
+
+    let raw = query.raw_query.trim();
+    if raw.is_empty() {
+        return result;
+    }
+
+    // Keep any explicit FilePath scope (AI mode `path/to/file.ext` prefix) so the
+    // fallback can't leak matches outside the file the user pinned. Only the
+    // swallowed operator/glob tokens are dropped. See issue #756.
+    let scoped_constraints: fff_query_parser::ConstraintVec<'_> = query
+        .constraints
+        .iter()
+        .filter(|c| matches!(c, fff_query_parser::Constraint::FilePath(_)))
+        .cloned()
+        .collect();
+
+    let literal_query = FFFQuery {
+        raw_query: query.raw_query,
+        constraints: scoped_constraints,
+        fuzzy_query: fff_query_parser::FuzzyQuery::Text(raw),
+        location: None,
+    };
+
+    let mut fallback = grep_search_parsed(
+        files,
+        &literal_query,
+        options,
+        budget,
+        bigram_index,
+        bigram_overlay,
+        abort_signal,
+        base_path,
+        arena,
+        overflow_arena,
+    );
+
+    if fallback.matches.is_empty() {
+        result
+    } else {
+        fallback.literal_fallback = true;
+        fallback
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn grep_search_parsed<'a>(
+    files: &'a [FileItem],
+    query: &FFFQuery<'_>,
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let total_files = files.live_count();
+    let constraints_from_query = &query.constraints[..];
+
+    let grep_text = extract_grep_text(query);
+    if grep_text.is_empty() {
+        return GrepResult::empty(total_files, total_files);
+    }
+
+    let case_insensitive = if options.smart_case {
+        !grep_text.chars().any(|c| c.is_uppercase())
+    } else {
+        false
+    };
+
+    let base_count = bigram_boundary(bigram_overlay, files.len());
+
+    let mut regex_fallback_error: Option<String> = None;
+    let regex = match options.mode {
+        GrepMode::PlainText => None,
+        GrepMode::Fuzzy => {
+            let bigram_candidates = fuzzy_candidates(bigram_index, bigram_overlay, &grep_text);
+
+            let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
+                files,
+                constraints_from_query,
+                bigram_candidates.as_deref(),
+                base_count,
+                options,
+                arena,
+                overflow_arena,
+            );
+
+            if files_to_search.is_empty() {
+                return GrepResult::empty(total_files, filtered_file_count);
+            }
+
+            return super::fuzzy_grep::fuzzy_grep_search(
+                &grep_text,
+                &files_to_search,
+                options,
+                total_files,
+                filtered_file_count,
+                case_insensitive,
+                budget,
+                abort_signal,
+                base_path,
+                arena,
+                overflow_arena,
+            );
+        }
+        GrepMode::Regex => build_regex(&grep_text, options.smart_case)
+            .inspect_err(|err| {
+                tracing::warn!("Regex compilation failed for {}. Error {}", grep_text, err);
+
+                regex_fallback_error = Some(err.to_string());
+            })
+            .ok(),
+    };
+
+    let (multiline_segment_len, effective_pattern) = match replace_newline_escapes(&grep_text) {
+        Some((replaced, first_newline_pos)) => (Some(first_newline_pos), replaced),
+        None => (None, grep_text),
+    };
+
+    let is_multiline = multiline_segment_len.is_some();
+
+    // when there is multiple line requested automatically expand the context to include all the lines
+    let after_context = if is_multiline && regex.is_none() && options.after_context == 0 {
+        effective_pattern.bytes().filter(|&b| b == b'\n').count()
+    } else {
+        options.after_context
+    };
+
+    let finder_pattern: Vec<u8> = if case_insensitive {
+        effective_pattern.as_bytes().to_ascii_lowercase()
+    } else {
+        effective_pattern.as_bytes().to_vec()
+    };
+    let finder = NeedleFinder::new(&finder_pattern, case_insensitive);
+    let pattern_len = finder_pattern.len() as u32;
+
+    // PlainText (or regex-fallback-to-plain): literal bigram query.
+    // Regex: decompose the pattern HIR into an AND/OR bigram query tree.
+    let bigram_candidates = if regex.is_none() {
+        literal_candidates(bigram_index, bigram_overlay, &[&effective_pattern])
+    } else {
+        regex_candidates(bigram_index, bigram_overlay, &effective_pattern)
+    };
+
+    let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
+        files,
+        constraints_from_query,
+        bigram_candidates.as_deref(),
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if files_to_search.is_empty() {
+        return GrepResult::empty(total_files, filtered_file_count);
+    }
+
+    // `PlainTextMatcher` is used by the grep-searcher engine for line detection.
+    // `PlainTextSink` / `RegexSink` handle highlight extraction independently via ripgrep create
+    let plain_matcher = PlainTextMatcher { finder: &finder };
+
+    let searcher = {
+        let mut b = SearcherBuilder::new();
+        b.line_number(true).multi_line(is_multiline);
+        b
+    }
+    .build();
+
+    let should_prefilter = regex.is_none();
+    let mut result = perform_grep(
+        &files_to_search,
+        options,
+        &GrepContext {
+            total_files,
+            filtered_file_count,
+            budget,
+            base_path,
+            arena,
+            overflow_arena,
+            prefilter: should_prefilter.then_some(&finder),
+            abort_signal,
+        },
+        // The single sink-selection point: every mode's matcher/sink pairing
+        // is decided here based on the compiled pattern.
+        |file_bytes: &[u8], max_matches: usize| {
+            let state = SinkState {
+                file_index: 0,
+                matches: Vec::with_capacity(4),
+                max_matches,
+                before_context: options.before_context,
+                after_context,
+                classify_definitions: options.classify_definitions,
+            };
+
+            match regex {
+                Some(ref re) => {
+                    let regex_matcher = RegexMatcher {
+                        regex: re,
+                        is_multiline,
+                    };
+                    let mut sink = RegexSink { state, re };
+                    if let Err(e) = searcher.search_slice(&regex_matcher, file_bytes, &mut sink) {
+                        tracing::error!(error = %e, "Grep (regex) search failed");
+                    }
+                    sink.state.matches
+                }
+                None => {
+                    let mut sink = PlainTextSink {
+                        state,
+                        finder: &finder,
+                        pattern_len,
+                        multiline_segment_len,
+                    };
+                    if let Err(e) = searcher.search_slice(&plain_matcher, file_bytes, &mut sink) {
+                        tracing::error!(error = %e, "Grep (plain text) search failed");
+                    }
+                    sink.state.matches
+                }
+            }
+        },
+    );
+    result.regex_fallback_error = regex_fallback_error;
+    result
+}
+
+/// Replace unescaped `\n` escapes with real newlines in a single pass.
+///
+/// Returns `Some((replaced, first_newline_pos))` when the pattern contained at
+/// least one real `\n` escape (the user wants multiline search), where
+/// `first_newline_pos` is the byte offset of the first inserted newline in the
+/// replaced string. Returns `None` when nothing had to be replaced: `\\n` is
+/// preserved as-is (escaped backslash + literal `n`, e.g. `\\nvim-data`).
+pub(super) fn replace_newline_escapes(text: &str) -> Option<(String, usize)> {
+    let bytes = text.as_bytes();
+    let mut result = Vec::with_capacity(bytes.len());
+    let mut first_newline_pos: Option<usize> = None;
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] == b'\\' && i + 1 < bytes.len() {
+            if bytes[i + 1] == b'n' {
+                // Odd number of consecutive backslashes before 'n' -> real \n escape
+                let mut backslash_count = 1;
+                while backslash_count <= i && bytes[i - backslash_count] == b'\\' {
+                    backslash_count += 1;
+                }
+                if backslash_count % 2 == 1 {
+                    first_newline_pos.get_or_insert(result.len());
+                    result.push(b'\n');
+                    i += 2;
+                    continue;
+                }
+            }
+            result.push(bytes[i]);
+            i += 1;
+        } else {
+            result.push(bytes[i]);
+            i += 1;
+        }
+    }
+
+    let first_newline_pos = first_newline_pos?;
+    let replaced = String::from_utf8(result).unwrap_or_else(|_| text.to_string());
+    Some((replaced, first_newline_pos))
+}
+
+pub fn parse_grep_query(query: &str) -> FFFQuery<'_> {
+    let parser = QueryParser::new(GrepConfig);
+    parser.parse(query)
+}
+
+/// Extract the grep pattern text from the parsed query: all non-constraint
+/// tokens joined with spaces, e.g. `"name = *.rs someth"` -> `"name = someth"`
+/// with constraint `Extension("rs")`.
+fn extract_grep_text(query: &FFFQuery<'_>) -> String {
+    if !matches!(query.fuzzy_query, fff_query_parser::FuzzyQuery::Empty) {
+        return query.grep_text();
+    }
+
+    // if constraint-only or empty query we use raw_query for backslash-escape handling
+    let t = query.raw_query.trim();
+    if t.starts_with('\\') && t.len() > 1 {
+        let suffix = &t[1..];
+        let parser = QueryParser::new(GrepConfig);
+        if !parser.parse(suffix).constraints.is_empty() {
+            return suffix.to_string();
+        }
+    }
+    t.to_string()
+}
+
+#[derive(Clone, Copy)]
+pub(super) struct GrepContext<'a, 'b> {
+    pub(super) total_files: usize,
+    pub(super) filtered_file_count: usize,
+    pub(super) budget: &'a ContentCacheBudget,
+    pub(super) base_path: &'a Path,
+    pub(super) arena: crate::simd_path::ArenaPtr,
+    pub(super) overflow_arena: crate::simd_path::ArenaPtr,
+    pub(super) prefilter: Option<&'a NeedleFinder<'b>>,
+    pub(super) abort_signal: &'a AtomicBool,
+}
+
+impl GrepContext<'_, '_> {
+    #[inline]
+    fn arena_for_file(&self, file: &FileItem) -> crate::simd_path::ArenaPtr {
+        if file.is_overflow() {
+            self.overflow_arena
+        } else {
+            self.arena
+        }
+    }
+}
+
+#[tracing::instrument(
+    skip_all,
+    level = Level::DEBUG,
+    fields(prefiltered_count = files_to_search.len())
+)]
+pub(super) fn perform_grep<'a, F>(
+    files_to_search: &[&'a FileItem],
+    options: &GrepSearchOptions,
+    ctx: &GrepContext<'_, '_>,
+    search_file: F,
+) -> GrepResult<'a>
+where
+    F: Fn(&[u8], usize) -> Vec<GrepMatch> + Sync,
+{
+    let time_budget = if options.time_budget_ms > 0 {
+        Some(std::time::Duration::from_millis(options.time_budget_ms))
+    } else {
+        None
+    };
+
+    let search_start = std::time::Instant::now();
+    let page_limit = options.page_limit;
+    let budget_exceeded = AtomicBool::new(false);
+
+    let mut result_files: Vec<&'a FileItem> = Vec::new();
+    let mut all_matches: Vec<GrepMatch> = Vec::new();
+    let mut files_consumed: usize = 0;
+    let mut page_filled = false;
+
+    // Each chunk is a rayon barrier. A flat small chunk over 500k files = ~7800
+    // barriers; x2 growth makes it logarithmic. But a too-aggressive growth
+    // over-scans: when a page fills mid-chunk, the whole submitted chunk still
+    // runs.
+    //
+    // So only grow when the prefilter is weak (large candidate set);
+    // when bigram cut the set in half, keep fixed small chunks for cheap page-fill termination.
+    let base_chunk = rayon::current_num_threads() * 4;
+    let prefilter_strong = ctx.total_files > 0 && files_to_search.len() * 2 < ctx.total_files;
+    let max_chunk = if prefilter_strong {
+        base_chunk
+    } else {
+        (base_chunk * 256).max(8 * 1024)
+    };
+    let growth = if prefilter_strong { 1 } else { 2 };
+    let mut chunk_size = base_chunk;
+    let mut chunk_start = 0;
+
+    while chunk_start < files_to_search.len() {
+        let chunk_end = (chunk_start + chunk_size).min(files_to_search.len());
+        let chunk = &files_to_search[chunk_start..chunk_end];
+        chunk_start = chunk_end;
+        chunk_size = (chunk_size * growth).min(max_chunk);
+        let chunk_offset = files_consumed;
+
+        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
+            .par_iter()
+            .enumerate()
+            .map_init(
+                // tested it out a few times, this is just fine for rayon worker in this specific
+                // case it doesn't reallocate this many times and it is actually faster than using
+                // scoped threads with a predefined local scratch buffers because of spawn cost
+                || (Vec::with_capacity(64 * 1024), MmapSlot::default()),
+                |(buf, mmap_slot), (local_idx, file)| {
+                    // perform all the atomic machinery on every 8th
+                    if local_idx % 8 == 0 {
+                        let mut need_abort = ctx.abort_signal.load(Ordering::Relaxed);
+                        if !need_abort
+                            && let Some(budget) = time_budget
+                            && all_matches.len() > 1
+                            && search_start.elapsed() > budget
+                        {
+                            need_abort = true;
+                        }
+
+                        if need_abort {
+                            budget_exceeded.store(true, Ordering::Relaxed);
+                            return None;
+                        }
+                    }
+
+                    let content = file.get_content_for_search(
+                        buf,
+                        mmap_slot,
+                        ctx.arena_for_file(file),
+                        ctx.base_path,
+                        ctx.budget,
+                    )?;
+
+                    // Fast whole-file memmem check before entering the
+                    // grep-searcher machinery. Skips Vec alloc, Searcher
+                    // setup, and line-splitting for files that can't match.
+                    if let Some(pf) = ctx.prefilter
+                        && pf.find(content).is_none()
+                    {
+                        return None;
+                    }
+
+                    let file_matches = search_file(content, options.max_matches_per_file);
+
+                    if file_matches.is_empty() {
+                        return None;
+                    }
+
+                    Some((chunk_offset + local_idx, *file, file_matches))
+                },
+            )
+            .flatten()
+            .collect();
+
+        // Every file in the chunk was visited by rayon (matched or not).
+        files_consumed = chunk_offset + chunk.len();
+
+        // Flatten this chunk's results into the accumulator.
+        for (batch_idx, file, file_matches) in chunk_results {
+            let file_result_idx = result_files.len();
+            result_files.push(file);
+
+            for mut m in file_matches {
+                m.file_index = file_result_idx;
+                if options.trim_whitespace {
+                    m.trim_leading_whitespace();
+                }
+                all_matches.push(m);
+            }
+
+            if all_matches.len() >= page_limit {
+                // Tighten files_consumed to the file that tipped us over so
+                // the next page resumes right after it.
+                files_consumed = batch_idx + 1;
+                page_filled = true;
+                break;
+            }
+        }
+
+        if page_filled || budget_exceeded.load(Ordering::Relaxed) {
+            break;
+        }
+    }
+
+    // If no file had any match, we searched the entire slice.
+    if result_files.is_empty() {
+        files_consumed = files_to_search.len();
+    }
+
+    let has_more = budget_exceeded.load(Ordering::Relaxed)
+        || (page_filled && files_consumed < files_to_search.len());
+
+    let next_file_offset = if has_more {
+        options.file_offset + files_consumed
+    } else {
+        0
+    };
+
+    GrepResult {
+        matches: all_matches,
+        files_with_matches: result_files.len(),
+        files: result_files,
+        total_files_searched: files_consumed,
+        total_files: ctx.total_files,
+        filtered_file_count: ctx.filtered_file_count,
+        next_file_offset,
+        regex_fallback_error: None,
+        literal_fallback: false,
+    }
+}

crates/fff-core/src/grep/grep_tests.rs

@@ -0,0 +1,518 @@
+use super::grep::replace_newline_escapes;
+use super::*;
+
+use crate::file_picker::{FilePicker, FilePickerOptions};
+use crate::index::BigramIndexBuilder;
+use std::io::Write;
+use std::sync::atomic::AtomicBool;
+
+#[test]
+fn test_replace_newline_escapes() {
+    // Single \n → multiline: replaced with a real newline at byte 3
+    assert_eq!(
+        replace_newline_escapes("foo\\nbar"),
+        Some(("foo\nbar".to_string(), 3))
+    );
+    // \\n → escaped backslash + literal n, NOT multiline
+    // (this is what the user types when grepping Rust source with `\\nvim`)
+    assert_eq!(replace_newline_escapes("foo\\\\nvim-data"), None);
+    // Real-world: source file has literal \\AppData\\Local\\nvim-data
+    // (double backslash in the file, so user types double backslash)
+    assert_eq!(
+        replace_newline_escapes(r#"format!("{}\\AppData\\Local\\nvim-data","#),
+        None
+    );
+    // No \n at all
+    assert_eq!(replace_newline_escapes("hello world"), None);
+    // \\\\n → even number of backslashes before n → NOT multiline
+    assert_eq!(replace_newline_escapes("foo\\\\\\\\nbar"), None);
+    // \\\n → 3 backslashes: first two pair up, third + n = \n → multiline,
+    // newline lands after "foo" + 2 kept backslashes = byte 5
+    assert_eq!(
+        replace_newline_escapes("foo\\\\\\nbar"),
+        Some(("foo\\\\\nbar".to_string(), 5))
+    );
+    // Position is for the FIRST newline when there are several
+    assert_eq!(
+        replace_newline_escapes("a\\nb\\nc"),
+        Some(("a\nb\nc".to_string(), 1))
+    );
+}
+
+#[test]
+fn test_fuzzy_typo_scoring() {
+    // Mirror the config from fuzzy_grep_search
+    let needle = "schema";
+    let max_typos = (needle.len() / 3).min(2); // 2
+    let config = neo_frizbee::Config {
+        max_typos: Some(max_typos as u16),
+        sort: false,
+        scoring: neo_frizbee::Scoring {
+            exact_match_bonus: 100,
+            ..neo_frizbee::Scoring::default()
+        },
+        ..Default::default()
+    };
+    let min_matched = needle.len().saturating_sub(1).max(1); // 5
+    let max_match_span = needle.len() + 4; // 10
+
+    // Helper: check if a match would pass our post-filters
+    let passes = |n: &str, h: &str| -> bool {
+        let Some(mut mi) = neo_frizbee::match_list_indices(n, &[h], &config)
+            .into_iter()
+            .next()
+        else {
+            return false;
+        };
+        // upstream returns indices in reverse order, sort ascending
+        mi.indices.sort_unstable();
+        if mi.indices.len() < min_matched {
+            return false;
+        }
+        if let (Some(&first), Some(&last)) = (mi.indices.first(), mi.indices.last()) {
+            let span = last - first + 1;
+            if span > max_match_span {
+                return false;
+            }
+            let density = (mi.indices.len() * 100) / span;
+            if density < 70 {
+                return false;
+            }
+        }
+        true
+    };
+
+    // Exact match: must pass
+    assert!(passes("schema", "schema"));
+    // Exact in longer line: must pass
+    assert!(passes("schema", "  schema: String,"));
+    // In identifier: must pass
+    assert!(passes("schema", "pub fn validate_schema() {}"));
+    // Transposition: must pass
+    assert!(passes("shcema", "schema"));
+    // Partial "ema" only line: must NOT pass
+    assert!(!passes("schema", "it has ema in it"));
+    // Completely unrelated: must NOT pass
+    assert!(!passes("schema", "hello world foo bar"));
+}
+
+#[test]
+fn test_multi_grep_search() {
+    use crate::file_picker::{FilePicker, FilePickerOptions};
+    use std::io::Write;
+
+    let dir = tempfile::tempdir().unwrap();
+
+    // File 1: has "GrepMode" and "GrepMatch"
+    {
+        let mut f = std::fs::File::create(dir.path().join("grep.rs")).unwrap();
+        writeln!(f, "pub enum GrepMode {{").unwrap();
+        writeln!(f, "    PlainText,").unwrap();
+        writeln!(f, "    Regex,").unwrap();
+        writeln!(f, "}}").unwrap();
+        writeln!(f, "pub struct GrepMatch {{").unwrap();
+        writeln!(f, "    pub line_number: u64,").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    // File 2: has "PlainTextMatcher" only
+    {
+        let mut f = std::fs::File::create(dir.path().join("matcher.rs")).unwrap();
+        writeln!(f, "struct PlainTextMatcher {{").unwrap();
+        writeln!(f, "    needle: Vec<u8>,").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    // File 3: no matches
+    {
+        let mut f = std::fs::File::create(dir.path().join("other.rs")).unwrap();
+        writeln!(f, "fn main() {{").unwrap();
+        writeln!(f, "    println!(\"hello\");").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: dir.path().to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let files = picker.get_files();
+    let arena = picker.arena_base_ptr();
+
+    let options = super::GrepSearchOptions {
+        max_file_size: MAX_FFFILE_SIZE,
+        max_matches_per_file: 0,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 100,
+        mode: super::GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: None,
+    };
+    let no_cancel = AtomicBool::new(false);
+
+    // Test with 3 patterns
+    let result = super::multi_grep_search(
+        files,
+        &["GrepMode", "GrepMatch", "PlainTextMatcher"],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+
+    assert!(
+        result.matches.len() >= 3,
+        "Expected at least 3 matches, got {}",
+        result.matches.len()
+    );
+
+    let has_grep_mode = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("GrepMode"));
+    let has_grep_match = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("GrepMatch"));
+    let has_plain_text_matcher = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("PlainTextMatcher"));
+
+    assert!(has_grep_mode, "Should find GrepMode");
+    assert!(has_grep_match, "Should find GrepMatch");
+    assert!(has_plain_text_matcher, "Should find PlainTextMatcher");
+
+    assert_eq!(result.files.len(), 2, "Should match exactly 2 files");
+
+    // Test with single pattern
+    let result2 = super::multi_grep_search(
+        files,
+        &["PlainTextMatcher"],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+    assert_eq!(
+        result2.matches.len(),
+        1,
+        "Single pattern should find 1 match"
+    );
+
+    // Test with empty patterns
+    let result3 = super::multi_grep_search(
+        files,
+        &[],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+    assert_eq!(
+        result3.matches.len(),
+        0,
+        "Empty patterns should find nothing"
+    );
+}
+
+/// E2E: multiline grep (`\n` in query) and escaped-backslash literals (`\\n`)
+/// through the full picker.grep pipeline, in both PlainText and Regex modes.
+#[test]
+fn test_grep_multiline_and_escaped_newline_e2e() {
+    let dir = tempfile::tempdir().unwrap();
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+
+    // Content spanning two lines: "hello unicorn\nrainbow world"
+    {
+        let mut f = std::fs::File::create(base.join("multi.txt")).unwrap();
+        writeln!(f, "hello unicorn").unwrap();
+        writeln!(f, "rainbow world").unwrap();
+    }
+    // Content with a literal double backslash before "nvim": `C:\\Users\\nvim-data`
+    {
+        let mut f = std::fs::File::create(base.join("winpath.rs")).unwrap();
+        writeln!(f, "let p = \"C:\\\\Users\\\\nvim-data\";").unwrap();
+    }
+    {
+        let mut f = std::fs::File::create(base.join("noise.txt")).unwrap();
+        writeln!(f, "nothing interesting here").unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let options = crate::GrepSearchOptions {
+        page_limit: 100,
+        max_matches_per_file: 0,
+        ..Default::default()
+    };
+
+    // 1. PlainText + `\n`: needle becomes a real newline, match spans two lines
+    let query = super::parse_grep_query("unicorn\\nrainbow");
+    let result = picker.grep(&query, &options);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "multiline plaintext should match multi.txt"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "multi.txt");
+    let m = &result.matches[0];
+    assert_eq!(m.line_content, "hello unicorn");
+    // Auto after-context: the rest of the matched span is returned
+    assert_eq!(m.context_after, vec!["rainbow world"]);
+    // First needle segment highlighted as the line suffix
+    assert_eq!(m.match_byte_offsets.as_slice(), &[(6, 13)]);
+    assert_eq!(m.col, 6);
+
+    // 2. PlainText + `\\n`: literal backslash + n, must NOT be mangled to newline
+    let query = super::parse_grep_query("\\\\nvim-data");
+    let result = picker.grep(&query, &options);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "escaped backslash should match winpath.rs literally"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "winpath.rs");
+    assert!(result.matches[0].line_content.contains("\\\\nvim-data"));
+    assert!(result.matches[0].context_after.is_empty());
+
+    // 3. Regex + `\n`: goes through the MultiLine searcher strategy
+    let regex_options = super::GrepSearchOptions {
+        mode: super::GrepMode::Regex,
+        ..options.clone()
+    };
+    let query = super::parse_grep_query("unicorn\\nrainbow");
+    let result = picker.grep(&query, &regex_options);
+    assert!(result.regex_fallback_error.is_none());
+    assert_eq!(
+        result.files.len(),
+        1,
+        "multiline regex should match multi.txt"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "multi.txt");
+    let m = &result.matches[0];
+    // Blob is normalized: single-line content + remaining lines as context
+    assert_eq!(m.line_content, "hello unicorn");
+    assert_eq!(m.context_after, vec!["rainbow world"]);
+    // Highlight clamped to the visible first line
+    assert_eq!(m.match_byte_offsets.as_slice(), &[(6, 13)]);
+}
+
+/// Regression test for issue #407: Live grep returns duplicate results
+/// when the bigram candidate bitset has trailing bits set beyond
+/// `base_file_count`. The bitset is rounded up to a multiple of 64 bits
+/// so any trailing bit that happens to be set (e.g. from overlay data)
+/// would previously map to an overflow file index, which was then also
+/// unconditionally appended by the overflow loop, producing duplicates.
+#[test]
+fn test_grep_no_duplicates_with_overflow_trailing_bits() {
+    let dir = tempfile::tempdir().unwrap();
+    // Match the picker's internal dunce-canonicalize so paths passed to
+    // on_create_or_modify resolve back to the same base_path on Windows.
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+
+    // Five base files: only three contain the pattern "unicorn".
+    // We need some files WITHOUT the pattern so the bigrams for
+    // "unicorn" aren't treated as ubiquitous (≥90% of files) and
+    // dropped from the index during compress().
+    let base_contents: &[(&str, &str)] = &[
+        ("a.txt", "hello unicorn world"),
+        ("b.txt", "another unicorn line"),
+        ("c.txt", "one more unicorn here"),
+        ("d.txt", "nothing special in here"),
+        ("e.txt", "just some random content"),
+    ];
+    for (name, content) in base_contents {
+        let mut f = std::fs::File::create(base.join(name)).unwrap();
+        writeln!(f, "{}", content).unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+    assert_eq!(picker.get_files().len(), 5);
+
+    // Manually build a bigram index over the 5 base files.
+    let base_count = 5usize;
+    let consec_builder = BigramIndexBuilder::new(base_count);
+    let skip_builder = BigramIndexBuilder::new(base_count);
+    for (i, (_, content)) in base_contents.iter().enumerate() {
+        consec_builder.add_file_content(&skip_builder, i, content.as_bytes());
+    }
+    let mut index = consec_builder.compress(Some(0));
+    index.set_skip_index(skip_builder.compress(Some(0)));
+    picker.set_bigram_index(index);
+
+    // Add three overflow files (new after the bigram index was built),
+    // all containing "unicorn".
+    for name in ["f.txt", "g.txt", "h.txt"] {
+        let path = base.join(name);
+        let mut f = std::fs::File::create(&path).unwrap();
+        writeln!(f, "overflow unicorn entry").unwrap();
+        drop(f);
+        picker.handle_create_or_modify(&path);
+    }
+    assert_eq!(picker.get_files().len(), 8);
+
+    // Inject a trailing bit into the overlay at a file index that
+    // corresponds to an overflow file (i.e. >= base_file_count=5 but
+    // < bitset_word_size=64). Without the fix, the bigram-candidate
+    // merge would set this bit in the bitset, and the bitset loop would
+    // push files[6] while the overflow loop also appends files[5..]
+    // which includes files[6], producing a duplicate.
+    let overflow_rel = "g.txt"; // middle overflow file
+    let overflow_abs = picker
+        .get_files()
+        .iter()
+        .position(|f| f.relative_path(&picker) == overflow_rel)
+        .expect("overflow file should be present");
+    assert!(overflow_abs >= base_count);
+    assert!(
+        overflow_abs < 64,
+        "index must fit in the single bitset word"
+    );
+
+    if let Some(overlay) = picker.bigram_overlay() {
+        overlay
+            .write()
+            .modify_file(overflow_abs, b"overflow unicorn entry");
+    }
+
+    // Run a grep for "unicorn": six files match
+    // (a, b, c in base + f, g, h in overflow).
+    let query = super::parse_grep_query("unicorn");
+    let options = super::GrepSearchOptions {
+        max_file_size: MAX_FFFILE_SIZE,
+        max_matches_per_file: 0,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 100,
+        mode: super::GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: Some(std::sync::Arc::new(AtomicBool::new(false))),
+    };
+    let result = picker.grep(&query, &options);
+
+    // Collect the matched relative paths via the returned files list.
+    let mut paths: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(&picker))
+        .collect();
+    paths.sort();
+
+    // Every file (base + overflow) should match exactly once.
+    let mut dedup = paths.clone();
+    dedup.dedup();
+    assert_eq!(
+        dedup, paths,
+        "grep must not return duplicate results (issue #407): {:?}",
+        paths
+    );
+    assert_eq!(
+        paths,
+        vec!["a.txt", "b.txt", "c.txt", "f.txt", "g.txt", "h.txt"],
+    );
+
+    // And the match count must equal the number of files (one line per
+    // file). A duplicate entry in files_to_search would double-count
+    // matches for the duplicated file.
+    assert_eq!(
+        result.matches.len(),
+        6,
+        "expected exactly one match per file, got {}",
+        result.matches.len()
+    );
+}
+
+/// Issue #756: an AI-mode regex query with an inline FilePath scope and
+/// top-level alternation. The regex fragments are swallowed as bogus Glob
+/// constraints, the constrained search finds nothing, and the literal/regex
+/// fallback must NOT drop the FilePath scope — otherwise the `|` branch leaks
+/// matches into files outside the pinned path.
+#[test]
+fn regex_fallback_keeps_file_path_scope_issue_756() {
+    use fff_query_parser::{AiGrepConfig, QueryParser};
+    let dir = tempfile::tempdir().unwrap();
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+    std::fs::create_dir(base.join("scope")).unwrap();
+    std::fs::write(
+        base.join("scope").join("target.css"),
+        "/* ---------- target ---------- */\n",
+    )
+    .unwrap();
+    std::fs::write(
+        base.join("outside.css"),
+        "/* ---------- outside ---------- */\n",
+    )
+    .unwrap();
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let options = crate::GrepSearchOptions {
+        mode: super::GrepMode::Regex,
+        smart_case: true,
+        max_matches_per_file: 80,
+        page_limit: 100,
+        ..Default::default()
+    };
+
+    let raw = r"scope/target.css ^/\* |^\s*/\* ----------";
+    let query = QueryParser::new(AiGrepConfig).parse(raw);
+    let result = picker.grep(&query, &options);
+    let mut paths: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(&picker))
+        .collect();
+    paths.sort();
+
+    assert_eq!(
+        paths,
+        vec!["scope/target.css"],
+        "regex fallback must not leak outside the FilePath scope"
+    );
+}

crates/fff-core/src/grep/mod.rs

@@ -0,0 +1,27 @@
+//! Live grep. `grep.rs` implements the main plain-text path, the parallel
+//! scan engine, and the `grep_search` entry point that picks the matcher/sink
+//! for every mode in one place; `regex`, `multi_pattern`, and `fuzzy_grep`
+//! hold the mode-specific machinery on top of the shared `prefilter`/`sink`.
+
+#[allow(clippy::module_inception)]
+mod grep;
+pub use grep::*;
+
+mod fuzzy_grep;
+mod multi_pattern;
+mod prefilter;
+mod regex;
+mod sink;
+mod types;
+
+#[cfg(feature = "definitions")]
+mod classify;
+#[cfg(feature = "definitions")]
+pub use classify::*;
+
+pub(crate) use multi_pattern::multi_grep_search;
+pub use regex::has_regex_metacharacters;
+pub use types::*;
+
+#[cfg(test)]
+mod grep_tests;

crates/fff-core/src/grep/multi_pattern.rs

@@ -0,0 +1,191 @@
+use super::grep::{GrepContext, perform_grep};
+use super::prefilter::prefilter_files;
+use super::sink::{SinkState, debug_assert_newline_terminator};
+use super::types::{GrepResult, GrepSearchOptions};
+use crate::index::{BigramFilter, BigramOverlay, bigram_boundary, literal_candidates};
+use crate::types::{ContentCacheBudget, FileItem, FileSliceExt};
+use aho_corasick::AhoCorasick;
+use fff_grep::{
+    Searcher, SearcherBuilder, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use smallvec::SmallVec;
+use std::path::Path;
+use std::sync::atomic::AtomicBool;
+
+/// A `grep_matcher::Matcher` backed by Aho-Corasick for multi-pattern search.
+///
+/// Finds the first occurrence of any pattern starting at the given offset.
+/// Always reports `\n` as the line terminator for the fast candidate-line path.
+struct AhoCorasickMatcher<'a> {
+    ac: &'a AhoCorasick,
+}
+
+impl Matcher for AhoCorasickMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> std::result::Result<Option<Match>, NoError> {
+        let hay = &haystack[at..];
+        let found: Option<aho_corasick::Match> = self.ac.find(hay);
+        Ok(found.map(|m| Match::new(at + m.start(), at + m.end())))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        Some(fff_grep::LineTerminator::byte(b'\n'))
+    }
+}
+
+/// Sink for Aho-Corasick multi-pattern mode.
+///
+/// Collects all pattern match positions on each matched line for highlighting.
+struct AhoCorasickSink<'a> {
+    state: SinkState,
+    ac: &'a AhoCorasick,
+}
+
+impl Sink for AhoCorasickSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(&mut self, searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = mat.bytes();
+        let (display_bytes, display_len, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, mat);
+
+        let line_content = String::from_utf8_lossy(display_bytes).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        for m in self.ac.find_iter(display_bytes as &[u8]) {
+            let abs_start = m.start() as u32;
+            let abs_end = (m.end() as u32).min(display_len);
+            if first {
+                col = abs_start as usize;
+                first = false;
+            }
+            match_byte_offsets.push((abs_start, abs_end));
+        }
+
+        let (context_before, context_after) = self.state.extract_context(mat);
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+/// Multi-pattern OR search using Aho-Corasick.
+///
+/// Builds a single automaton from all patterns and searches each file in one
+/// pass. This is significantly faster than regex alternation for literal text
+/// searches because Aho-Corasick uses SIMD-accelerated multi-needle matching.
+///
+/// Returns the same `GrepResult` type as `grep_search`.
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn multi_grep_search<'a>(
+    files: &'a [FileItem],
+    patterns: &[&str],
+    constraints: &[fff_query_parser::Constraint<'_>],
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let total_files = files.live_count();
+
+    if patterns.is_empty() || patterns.iter().all(|p| p.is_empty()) {
+        return GrepResult::empty(total_files, total_files);
+    }
+
+    let bigram_candidates = literal_candidates(bigram_index, bigram_overlay, patterns);
+    let base_file_count = bigram_boundary(bigram_overlay, files.len());
+
+    // Constraints are separate from patterns, so a miss must not broaden the search.
+    let (files_to_search, filtered_file_count) = prefilter_files(
+        files,
+        constraints,
+        bigram_candidates.as_deref(),
+        base_file_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if files_to_search.is_empty() {
+        return GrepResult::empty(total_files, filtered_file_count);
+    }
+
+    // Smart case: case-insensitive when all patterns are lowercase
+    let case_insensitive = if options.smart_case {
+        !patterns.iter().any(|p| p.chars().any(|c| c.is_uppercase()))
+    } else {
+        false
+    };
+
+    let ac = aho_corasick::AhoCorasickBuilder::new()
+        .ascii_case_insensitive(case_insensitive)
+        .build(patterns)
+        .expect("Aho-Corasick build should not fail for literal patterns");
+
+    let searcher = {
+        let mut b = SearcherBuilder::new();
+        b.line_number(true);
+        b
+    }
+    .build();
+
+    let ac_matcher = AhoCorasickMatcher { ac: &ac };
+    perform_grep(
+        &files_to_search,
+        options,
+        &GrepContext {
+            total_files,
+            filtered_file_count,
+            budget,
+            base_path,
+            arena,
+            overflow_arena,
+            prefilter: None, // no memmem prefilter for multi-pattern search
+            abort_signal,
+        },
+        |file_bytes: &[u8], max_matches: usize| {
+            let state = SinkState {
+                file_index: 0,
+                matches: Vec::with_capacity(4),
+                max_matches,
+                before_context: options.before_context,
+                after_context: options.after_context,
+                classify_definitions: options.classify_definitions,
+            };
+
+            let mut sink = AhoCorasickSink { state, ac: &ac };
+
+            if let Err(e) = searcher.search_slice(&ac_matcher, file_bytes, &mut sink) {
+                tracing::error!(error = %e, "Grep (aho-corasick multi) search failed");
+            }
+
+            sink.state.matches
+        },
+    )
+}

crates/fff-core/src/grep/prefilter.rs

@@ -0,0 +1,204 @@
+use super::types::GrepSearchOptions;
+use crate::index::BigramFilter;
+use crate::index::constraints::{ConstraintPlan, ConstraintsBuffers};
+use crate::sort_buffer::sort_with_buffer;
+use crate::types::FileItem;
+use fff_query_parser::Constraint;
+
+/// Prefilter with a FilePath-constraint fallback: when constraints yield 0
+/// files and the query had FilePath constraints, retry without them (the path
+/// token was likely part of the search text).
+#[allow(clippy::too_many_arguments)]
+pub(super) fn prefilter_with_filepath_retry<'a>(
+    files: &'a [FileItem],
+    constraints: &[Constraint<'_>],
+    bigram_candidates: Option<&[u64]>,
+    base_count: usize,
+    options: &GrepSearchOptions,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> (Vec<&'a FileItem>, usize) {
+    let (files_to_search, filtered_file_count) = prefilter_files(
+        files,
+        constraints,
+        bigram_candidates,
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if !files_to_search.is_empty() {
+        return (files_to_search, filtered_file_count);
+    }
+
+    let Some(stripped) = strip_file_path_constraint_if_present(constraints) else {
+        return (files_to_search, filtered_file_count);
+    };
+
+    prefilter_files(
+        files,
+        &stripped,
+        bigram_candidates,
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    )
+}
+
+/// Single pass prefilter that doesn't involve file reading
+/// allocates only amount of memory required for storing references of the FileItems have to be
+/// opened for grepping unaviodably, in the worst case allocates N * <word> memory if no prefilter needed
+pub(crate) fn prefilter_files<'a>(
+    files: &'a [FileItem],
+    constraints: &[Constraint<'_>],
+    bigram_candidates: Option<&[u64]>,
+    base_count: usize,
+    options: &GrepSearchOptions,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> (Vec<&'a FileItem>, usize) {
+    let max_file_size = options.max_file_size;
+    let plan = if constraints.is_empty() {
+        None
+    } else {
+        Some(ConstraintPlan::build(
+            constraints,
+            files,
+            arena,
+            overflow_arena,
+        ))
+    };
+
+    let mut scratch = ConstraintsBuffers::new();
+
+    #[inline(always)]
+    fn basic_prefilter(file: &FileItem, max: u64) -> bool {
+        !file.is_deleted() && !file.is_binary() && file.size > 0 && file.size <= max
+    }
+
+    // squeeze as much prefilters into a single loop as possible
+    let mut prefiltered: Vec<&FileItem> = match bigram_candidates {
+        Some(candidates) => {
+            let boundary = base_count.min(files.len());
+            let (indexed, tail) = files.split_at(boundary);
+
+            let cap = BigramFilter::count_candidates(candidates) + tail.len();
+            let mut out: Vec<&FileItem> = Vec::with_capacity(cap);
+
+            let full_words = boundary / 64;
+            let last_word_bits = boundary % 64;
+
+            // we need this because we already had a regression of the wrong bit
+            // has been set for the very last word based on the overlay, it's pretty cheap
+            macro_rules! evaluate_bigram_match_word {
+                ($word:expr, $base:expr) => {{
+                    let mut bits: u64 = $word;
+                    while bits != 0 {
+                        let bit = bits.trailing_zeros() as usize;
+                        let file_idx = $base + bit;
+                        bits &= bits - 1;
+
+                        let f = unsafe { indexed.get_unchecked(file_idx) };
+                        if !basic_prefilter(f, max_file_size) {
+                            continue;
+                        }
+                        if let Some(plan) = plan.as_ref()
+                            && !plan.matches(f, file_idx, arena, overflow_arena, &mut scratch)
+                        {
+                            continue;
+                        }
+                        out.push(f);
+                    }
+                }};
+            }
+
+            // Full words: every set bit guaranteed `< boundary`.
+            for (word_idx, &word) in candidates.iter().take(full_words).enumerate() {
+                if word != 0 {
+                    evaluate_bigram_match_word!(word, word_idx * 64);
+                }
+            }
+
+            // Last partial word: mask bits past `boundary` once at word load.
+            if last_word_bits != 0 {
+                // this will get only (mod 64) bits from the last word guarantee that it's 0 padded
+                let last_mask: u64 = (1u64 << last_word_bits) - 1;
+                let word = candidates[full_words] & last_mask;
+                if word != 0 {
+                    evaluate_bigram_match_word!(word, full_words * 64);
+                }
+            }
+
+            // Sequential processing for non-bigrammable files: they are always in the end
+            for (offset, f) in tail.iter().enumerate() {
+                if !basic_prefilter(f, max_file_size) {
+                    continue;
+                }
+                if let Some(ref p) = plan
+                    && !p.matches(f, boundary + offset, arena, overflow_arena, &mut scratch)
+                {
+                    continue;
+                }
+                out.push(f);
+            }
+
+            out
+        }
+        // this will be executed if there is no bigram, in the worst case it will allocate
+        // whole array of files but probability in the real repo of NO preflter working is so
+        // low that we just ignore that, usually there would be at least a few files excluded
+        None => {
+            let mut out: Vec<&FileItem> = Vec::new();
+            for (idx, f) in files.iter().enumerate() {
+                if !basic_prefilter(f, max_file_size) {
+                    continue;
+                }
+                if let Some(ref p) = plan
+                    && !p.matches(f, idx, arena, overflow_arena, &mut scratch)
+                {
+                    continue;
+                }
+                out.push(f);
+            }
+            out
+        }
+    };
+
+    let total_count = prefiltered.len();
+
+    sort_with_buffer(&mut prefiltered, |a, b| {
+        b.total_frecency_score()
+            .cmp(&a.total_frecency_score())
+            .then(b.modified.cmp(&a.modified))
+    });
+
+    if options.file_offset > 0 && options.file_offset < total_count {
+        let paginated = prefiltered.split_off(options.file_offset);
+        (paginated, total_count)
+    } else if options.file_offset >= total_count {
+        (Vec::new(), total_count)
+    } else {
+        (prefiltered, total_count)
+    }
+}
+
+fn strip_file_path_constraint_if_present<'a>(
+    constraints: &[Constraint<'a>],
+) -> Option<fff_query_parser::ConstraintVec<'a>> {
+    if !constraints
+        .iter()
+        .any(|c| matches!(c, Constraint::FilePath(_)))
+    {
+        return None;
+    }
+
+    let filtered: fff_query_parser::ConstraintVec<'a> = constraints
+        .iter()
+        .filter(|c| !matches!(c, Constraint::FilePath(_)))
+        .cloned()
+        .collect();
+
+    Some(filtered)
+}

crates/fff-core/src/grep/regex.rs

@@ -0,0 +1,130 @@
+use super::sink::{SinkState, debug_assert_newline_terminator, split_multiline_blob};
+use fff_grep::{
+    Searcher, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use smallvec::SmallVec;
+
+pub fn has_regex_metacharacters(text: &str) -> bool {
+    regex::escape(text) != text
+}
+
+pub(super) fn build_regex(pattern: &str, smart_case: bool) -> Result<regex::bytes::Regex, String> {
+    if pattern.is_empty() {
+        return Err("empty pattern".to_string());
+    }
+
+    let regex_pattern = if pattern.contains("\\n") {
+        pattern.replace("\\n", "\n")
+    } else {
+        pattern.to_string()
+    };
+
+    let case_insensitive = if smart_case {
+        !pattern.chars().any(|c| c.is_uppercase())
+    } else {
+        false
+    };
+
+    regex::bytes::RegexBuilder::new(&regex_pattern)
+        .case_insensitive(case_insensitive)
+        .multi_line(true)
+        .unicode(false)
+        .build()
+        .map_err(|e| e.to_string())
+}
+
+pub(super) struct RegexMatcher<'r> {
+    pub(super) regex: &'r regex::bytes::Regex,
+    pub(super) is_multiline: bool,
+}
+
+impl Matcher for RegexMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>, NoError> {
+        Ok(self
+            .regex
+            .find_at(haystack, at)
+            .map(|m| Match::new(m.start(), m.end())))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        if self.is_multiline {
+            None
+        } else {
+            Some(fff_grep::LineTerminator::byte(b'\n'))
+        }
+    }
+}
+
+pub(super) struct RegexSink<'r> {
+    pub(super) state: SinkState,
+    pub(super) re: &'r regex::bytes::Regex,
+}
+
+impl Sink for RegexSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(
+        &mut self,
+        searcher: &Searcher,
+        sink_match: &SinkMatch<'_>,
+    ) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = sink_match.bytes();
+        let (display_bytes, _, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, sink_match);
+
+        // MultiLine strategy hands over all matched lines as one blob: keep
+        // `line_content` single-line, the remaining lines become after-context.
+        let (first_line, extra_after) = split_multiline_blob(display_bytes);
+        let first_len = first_line.len() as u32;
+        let line_content = String::from_utf8_lossy(first_line).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        for m in self.re.find_iter(display_bytes) {
+            let abs_start = m.start() as u32;
+            if abs_start >= first_len {
+                continue; // highlight only spans visible in the first line
+            }
+            let abs_end = (m.end() as u32).min(first_len);
+            if first {
+                col = abs_start as usize;
+                first = false;
+            }
+            match_byte_offsets.push((abs_start, abs_end));
+        }
+
+        let (context_before, context_after) = self.state.extract_context(sink_match);
+        let context_after = if extra_after.is_empty() {
+            context_after
+        } else {
+            let mut combined = extra_after;
+            combined.extend(context_after);
+            combined
+        };
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}

crates/fff-core/src/grep/sink.rs

@@ -0,0 +1,242 @@
+use super::types::GrepMatch;
+use fff_grep::{Searcher, SinkMatch};
+use smallvec::SmallVec;
+
+/// Maximum bytes of a matched line to keep for display. Prevents minified
+/// JS or huge single-line files from blowing up memory.
+pub(super) const MAX_LINE_DISPLAY_LEN: usize = 512;
+
+#[cfg(feature = "definitions")]
+#[inline]
+pub(super) fn classify_definition(enabled: bool, line: &str) -> bool {
+    enabled && super::classify::is_definition_line(line)
+}
+
+#[cfg(not(feature = "definitions"))]
+#[inline]
+pub(super) fn classify_definition(_enabled: bool, _line: &str) -> bool {
+    false
+}
+
+#[inline]
+pub(super) fn debug_assert_newline_terminator(searcher: &Searcher) {
+    debug_assert_eq!(
+        searcher.line_terminator(),
+        fff_grep::LineTerminator::byte(b'\n'),
+        "sink helpers assume \\n line terminators (see module invariant)"
+    );
+}
+
+#[inline]
+pub(super) fn strip_line_terminators(bytes: &[u8]) -> &[u8] {
+    let mut len = bytes.len();
+    while len > 0 && matches!(bytes[len - 1], b'\n' | b'\r') {
+        len -= 1;
+    }
+    &bytes[..len]
+}
+
+pub(super) struct SinkState {
+    pub(super) file_index: usize,
+    pub(super) matches: Vec<GrepMatch>,
+    pub(super) max_matches: usize,
+    pub(super) before_context: usize,
+    pub(super) after_context: usize,
+    pub(super) classify_definitions: bool,
+}
+
+impl SinkState {
+    #[inline]
+    pub(super) fn prepare_line<'a>(
+        line_bytes: &'a [u8],
+        mat: &SinkMatch<'_>,
+    ) -> (&'a [u8], u32, u64, u64) {
+        let line_number = mat.line_number().unwrap_or(0);
+        let byte_offset = mat.absolute_byte_offset();
+
+        // Trim trailing newline/CR directly on bytes to avoid UTF-8 conversion.
+        let trimmed_bytes = strip_line_terminators(line_bytes);
+
+        // Truncate for display (floor to a char boundary).
+        let display_bytes = truncate_display_bytes(trimmed_bytes);
+
+        let display_len = display_bytes.len() as u32;
+        (display_bytes, display_len, line_number, byte_offset)
+    }
+
+    #[inline]
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn push_match(
+        &mut self,
+        line_number: u64,
+        col: usize,
+        byte_offset: u64,
+        line_content: String,
+        match_byte_offsets: SmallVec<[(u32, u32); 4]>,
+        context_before: Vec<String>,
+        context_after: Vec<String>,
+    ) {
+        let is_definition = classify_definition(self.classify_definitions, &line_content);
+        self.matches.push(GrepMatch {
+            file_index: self.file_index,
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            fuzzy_score: None,
+            is_definition,
+            context_before,
+            context_after,
+        });
+    }
+
+    /// Extract context lines from the full buffer around a matched region.
+    pub(super) fn extract_context(&self, mat: &SinkMatch<'_>) -> (Vec<String>, Vec<String>) {
+        if self.before_context == 0 && self.after_context == 0 {
+            return (Vec::new(), Vec::new());
+        }
+
+        let buffer = mat.buffer();
+        let range = mat.bytes_range_in_buffer();
+
+        let mut before = Vec::new();
+        if self.before_context > 0 && range.start > 0 {
+            // Walk backward from the start of the match line to find preceding lines
+            let mut pos = range.start;
+            let mut lines_found = 0;
+            while lines_found < self.before_context && pos > 0 {
+                // Skip the newline just before our current position
+                pos -= 1;
+                // Find the previous newline
+                let line_start = match memchr::memrchr(b'\n', &buffer[..pos]) {
+                    Some(nl) => nl + 1,
+                    None => 0,
+                };
+                let line = &buffer[line_start..pos];
+                // Trim trailing \r
+                let line = if line.last() == Some(&b'\r') {
+                    &line[..line.len() - 1]
+                } else {
+                    line
+                };
+                let truncated = truncate_display_bytes(line);
+                before.push(String::from_utf8_lossy(truncated).into_owned());
+                pos = line_start;
+                lines_found += 1;
+            }
+            before.reverse();
+        }
+
+        let mut after = Vec::new();
+        if self.after_context > 0 && range.end < buffer.len() {
+            let mut pos = range.end;
+            let mut lines_found = 0;
+            while lines_found < self.after_context && pos < buffer.len() {
+                // Find the next newline
+                let line_end = match memchr::memchr(b'\n', &buffer[pos..]) {
+                    Some(nl) => pos + nl,
+                    None => buffer.len(),
+                };
+                let line = &buffer[pos..line_end];
+                // Trim trailing \r
+                let line = if line.last() == Some(&b'\r') {
+                    &line[..line.len() - 1]
+                } else {
+                    line
+                };
+                let truncated = truncate_display_bytes(line);
+                after.push(String::from_utf8_lossy(truncated).into_owned());
+                pos = if line_end < buffer.len() {
+                    line_end + 1 // skip past \n
+                } else {
+                    buffer.len()
+                };
+                lines_found += 1;
+            }
+        }
+
+        (before, after)
+    }
+}
+
+/// Truncate a byte slice for display, respecting UTF-8 char boundaries.
+#[inline]
+pub(super) fn truncate_display_bytes(bytes: &[u8]) -> &[u8] {
+    if bytes.len() <= MAX_LINE_DISPLAY_LEN {
+        bytes
+    } else {
+        let mut end = MAX_LINE_DISPLAY_LEN;
+        while end > 0 && !is_utf8_char_boundary(bytes[end]) {
+            end -= 1;
+        }
+        &bytes[..end]
+    }
+}
+
+/// Split a multiline match blob (from the MultiLine searcher strategy) into
+/// the first line and the remaining lines so `line_content` stays single-line.
+pub(super) fn split_multiline_blob(display_bytes: &[u8]) -> (&[u8], Vec<String>) {
+    match memchr::memchr(b'\n', display_bytes) {
+        None => (display_bytes, Vec::new()),
+        Some(pos) => {
+            let first = strip_line_terminators(&display_bytes[..pos + 1]);
+            let extra = display_bytes[pos + 1..]
+                .split(|&b| b == b'\n')
+                .map(|l| String::from_utf8_lossy(strip_line_terminators(l)).into_owned())
+                .collect();
+            (first, extra)
+        }
+    }
+}
+
+/// Convert character-position indices from neo_frizbee into byte-offset
+/// pairs (start, end) suitable for `match_byte_offsets`.
+///
+/// frizbee returns character positions (0-based index into the char
+/// iterator). We need byte ranges because the UI renderer and Lua layer
+/// use byte offsets for extmark highlights.
+///
+/// Each matched character becomes its own (byte_start, byte_end) pair.
+/// Adjacent characters are merged into a single contiguous range.
+pub(super) fn char_indices_to_byte_offsets(
+    line: &str,
+    char_indices: &[usize],
+) -> SmallVec<[(u32, u32); 4]> {
+    if char_indices.is_empty() {
+        return SmallVec::new();
+    }
+
+    // Build a map: char_index -> (byte_start, byte_end) for all chars.
+    // Iterating all chars is O(n) in the line length which is bounded by MAX_LINE_DISPLAY_LEN (512).
+    let char_byte_ranges: Vec<(usize, usize)> = line
+        .char_indices()
+        .map(|(byte_pos, ch)| (byte_pos, byte_pos + ch.len_utf8()))
+        .collect();
+
+    // Convert char indices to byte ranges, merging adjacent ranges
+    let mut result: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(char_indices.len());
+
+    for &ci in char_indices {
+        if ci >= char_byte_ranges.len() {
+            continue; // out of bounds (shouldn't happen with valid data)
+        }
+        let (start, end) = char_byte_ranges[ci];
+        // Merge with previous range if adjacent
+        if let Some(last) = result.last_mut()
+            && last.1 == start as u32
+        {
+            last.1 = end as u32;
+            continue;
+        }
+        result.push((start as u32, end as u32));
+    }
+
+    result
+}
+
+// copied from the rust u8 private method
+#[inline]
+const fn is_utf8_char_boundary(b: u8) -> bool {
+    (b as i8) >= -0x40
+}

crates/fff-core/src/grep/types.rs

@@ -0,0 +1,243 @@
+use crate::types::FileItem;
+use smallvec::SmallVec;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+
+pub use crate::constants::MAX_FFFILE_SIZE;
+
+/// Controls how the grep pattern is interpreted.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum GrepMode {
+    /// Literal plain text match: default path that doesn't require any regex machinery
+    #[default]
+    PlainText,
+    /// Regex mode: uses the same exact matching engine as ripgrep
+    Regex,
+    /// Smart fuzzy mode, allows user to make either a couple of single char typos or long gaps
+    /// e.g. shcema -> shcema, or UserController -> UserAuthController
+    ///
+    /// Significatnly slower than plain text, especially on unindexed FilePicker
+    Fuzzy,
+}
+
+/// A single content match within a file
+#[derive(Debug, Clone)]
+pub struct GrepMatch {
+    /// Index into the deduplicated `files` vec of the GrepResult.
+    pub file_index: usize,
+    /// 1-based line number.
+    pub line_number: u64,
+    /// 0-based byte column of first match start within the line.
+    pub col: usize,
+    /// Absolute byte offset of the matched line from the start of the file.
+    /// Can be used by the preview to seek directly without scanning from the top.
+    pub byte_offset: u64,
+    /// The matched line text, truncated to `MAX_LINE_DISPLAY_LEN`.
+    pub line_content: String,
+    /// Byte offsets `(start, end)` within `line_content` for each match.
+    /// Stack-allocated for the common case of ≤4 spans per line.
+    pub match_byte_offsets: SmallVec<[(u32, u32); 4]>,
+    /// Fuzzy match score from neo_frizbee (only set in Fuzzy grep mode).
+    pub fuzzy_score: Option<u16>,
+    /// Whether the matched line looks like a definition (struct, fn, class, etc.).
+    /// Computed at match time so output formatters don't need to re-scan.
+    pub is_definition: bool,
+    /// Lines before the match (for context display). Empty when context is 0.
+    pub context_before: Vec<String>,
+    /// Lines after the match (for context display). Empty when context is 0.
+    pub context_after: Vec<String>,
+}
+
+impl GrepMatch {
+    /// Strip leading whitespace from `line_content` and all context lines,
+    /// adjusting `col` and `match_byte_offsets` so highlights remain correct.
+    pub fn trim_leading_whitespace(&mut self) {
+        let strip_len = self.line_content.len() - self.line_content.trim_start().len();
+        if strip_len > 0 {
+            self.line_content.drain(..strip_len);
+            let off = strip_len as u32;
+            self.col = self.col.saturating_sub(strip_len);
+            for range in &mut self.match_byte_offsets {
+                range.0 = range.0.saturating_sub(off);
+                range.1 = range.1.saturating_sub(off);
+            }
+        }
+        for line in &mut self.context_before {
+            let n = line.len() - line.trim_start().len();
+            if n > 0 {
+                line.drain(..n);
+            }
+        }
+        for line in &mut self.context_after {
+            let n = line.len() - line.trim_start().len();
+            if n > 0 {
+                line.drain(..n);
+            }
+        }
+    }
+}
+
+/// Options for grep search.
+#[derive(Debug, Clone)]
+pub struct GrepSearchOptions {
+    pub max_file_size: u64,
+    pub max_matches_per_file: usize,
+    pub smart_case: bool,
+    /// File-based pagination offset: index into the sorted/filtered file list
+    /// to start searching from. Pass 0 for the first page, then use
+    /// `GrepResult::next_file_offset` for subsequent pages.
+    pub file_offset: usize,
+    /// Maximum number of matches to collect before stopping.
+    pub page_limit: usize,
+    /// How to interpret the search pattern. Defaults to `PlainText`.
+    pub mode: GrepMode,
+    /// Maximum time in milliseconds to spend searching before returning partial
+    /// results. Prevents UI freezes on pathological queries. 0 = no limit.
+    pub time_budget_ms: u64,
+    /// Number of context lines to include before each match. 0 = disabled.
+    pub before_context: usize,
+    /// Number of context lines to include after each match. 0 = disabled.
+    pub after_context: usize,
+    /// Whether to classify each match as a definition line. Adds ~2% overhead
+    /// on large repos; disable for interactive grep where it is not needed.
+    pub classify_definitions: bool,
+    /// Strip leading whitespace from matched lines and context lines, adjusting
+    /// highlight byte offsets accordingly. Useful for AI/MCP consumers and UIs
+    /// that don't need indentation. Default: false.
+    pub trim_whitespace: bool,
+    /// External abort signal. When provided, overrides the picker's internal
+    /// cancellation flag. Set to `true` to stop the search early and return
+    /// partial results. Omit (or use `..Default::default()`) to let the
+    /// picker manage cancellation.
+    pub abort_signal: Option<Arc<AtomicBool>>,
+}
+
+impl Default for GrepSearchOptions {
+    fn default() -> Self {
+        Self {
+            max_file_size: MAX_FFFILE_SIZE,
+            max_matches_per_file: 200,
+            smart_case: true,
+            file_offset: 0,
+            page_limit: 50,
+            mode: GrepMode::default(),
+            time_budget_ms: 0,
+            before_context: 0,
+            after_context: 0,
+            classify_definitions: false,
+            trim_whitespace: false,
+            abort_signal: None,
+        }
+    }
+}
+
+/// Result of a grep search with a list of matches, list of matched files, and metadata.
+#[derive(Debug, Clone, Default)]
+pub struct GrepResult<'a> {
+    pub matches: Vec<GrepMatch>,
+    /// Deduplicated file references for the returned matches.
+    pub files: Vec<&'a FileItem>,
+    /// Number of files actually searched in this call.
+    pub total_files_searched: usize,
+    /// Total number of indexed files (before filtering).
+    pub total_files: usize,
+    /// Total number of searchable files (after filtering out binary, too-large, etc.).
+    pub filtered_file_count: usize,
+    /// Number of files that contained at least one match.
+    pub files_with_matches: usize,
+    /// The file offset to pass for the next page. `0` if there are no more files.
+    /// Callers should store this and pass it as `file_offset` in the next call.
+    pub next_file_offset: usize,
+    /// When regex mode fails to compile the pattern, the search falls back to
+    /// literal matching and this field contains the compilation error message.
+    /// The UI can display this to inform the user their regex was invalid.
+    pub regex_fallback_error: Option<String>,
+    /// Set to `true` if the constrained query found nothing and the results come from
+    /// retrying the whole raw query as literal text (ignored all the inferred constraints)
+    pub literal_fallback: bool,
+}
+
+impl<'a> GrepResult<'a> {
+    /// Empty result carrying only the file counts (empty query / prefilter miss)
+    pub(crate) fn empty(total_files: usize, filtered_file_count: usize) -> Self {
+        Self {
+            total_files,
+            filtered_file_count,
+            ..Default::default()
+        }
+    }
+
+    pub(crate) fn collect(
+        per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)>,
+        files_to_search_len: usize,
+        options: &GrepSearchOptions,
+        total_files: usize,
+        filtered_file_count: usize,
+        budget_exceeded: bool,
+    ) -> Self {
+        let page_limit = options.page_limit;
+
+        // Each match stores a `file_index` pointing into `result_files` so that
+        // consumers (FFI JSON, Lua) can look up file metadata without duplicating
+        // it across every match from the same file
+        let mut result_files: Vec<&'a FileItem> = Vec::new();
+        let mut all_matches: Vec<GrepMatch> = Vec::new();
+        // files_consumed tracks how far into files_to_search we have advanced,
+        // counting every file whose results were emitted (with or without matches).
+        // We use the batch_idx of the last consumed file + 1, which is correct
+        // because per_file_results only contains files that had matches, and
+        // files between them that had no matches were still searched and can be
+        // safely skipped on the next page
+        let mut files_consumed: usize = 0;
+
+        for (batch_idx, file, file_matches) in per_file_results {
+            // batch_idx is the 0-based position in files_to_search.
+            // Advance files_consumed to include this file and all no-match files before it.
+            files_consumed = batch_idx + 1;
+
+            let file_result_idx = result_files.len();
+            result_files.push(file);
+
+            for mut m in file_matches {
+                m.file_index = file_result_idx;
+                if options.trim_whitespace {
+                    m.trim_leading_whitespace();
+                }
+                all_matches.push(m);
+            }
+
+            // page_limit is a soft cap: we always finish the current file before
+            // stopping, so no matches are dropped. A page may return up to
+            // page_limit + max_matches_per_file - 1 matches in the worst case
+            if all_matches.len() >= page_limit {
+                break;
+            }
+        }
+
+        // If no file had any match, we searched the entire slice.
+        if result_files.is_empty() {
+            files_consumed = files_to_search_len;
+        }
+
+        let has_more = budget_exceeded
+            || (all_matches.len() >= page_limit && files_consumed < files_to_search_len);
+
+        let next_file_offset = if has_more {
+            options.file_offset + files_consumed
+        } else {
+            0
+        };
+
+        Self {
+            matches: all_matches,
+            files_with_matches: result_files.len(),
+            files: result_files,
+            total_files_searched: files_consumed,
+            total_files,
+            filtered_file_count,
+            next_file_offset,
+            regex_fallback_error: None,
+            literal_fallback: false,
+        }
+    }
+}

crates/fff-core/src/ignore.rs

@@ -1,48 +1,75 @@
 use std::path::Path;
 
-pub(crate) const NON_GIT_IGNORED_DIRS: &[&str] = &[
+/// Directories excluded when walking a non-git root. Entries are `cfg`-gated
+/// so a single iteration covers standard + platform-specific overrides.
+pub(crate) const IGNORED_DIRS: &[&str] = &[
+    // various dev tools that can be meet in the developer app
     "node_modules",
     "__pycache__",
     "venv",
     ".venv",
-    // Rust (these are glob-only patterns for non_git_repo_overrides,
-    // is_non_code_directory matches the "target" component separately)
     "target/debug",
     "target/release",
     "target/rust-analyzer",
     "target/criterion",
-];
-
-#[cfg(target_os = "macos")]
-pub(crate) const PLATFORM_IGNORED_DIRS: &[&str] = &[
+    // Language package caches in non-git roots.
+    "go/pkg/mod",
+    ".cargo/registry",
+    ".rustup/toolchains",
+    ".gradle/caches",
+    ".m2/repository",
+    ".npm/_cacache",
+    ".pub-cache",
+    #[cfg(not(target_os = "windows"))]
+    ".local/state", // this contains tons of logs which generate too much watcher noise
+    #[cfg(target_os = "macos")]
     "Library/Application Support",
+    #[cfg(target_os = "macos")]
     "Library/Caches",
-    // App-group sandbox storage — used by iMessage, Photos, Notes, Calendar,
-    // Electron apps, etc. for SQLite-WAL, LevelDB, protobuf files. These are
-    // almost entirely extension-less binary files (~80k on a typical $HOME)
-    // that never need to appear in a fuzzy or grep search.
-    "Library/Group Containers",
-    "Library/Containers",
-];
-
-#[cfg(target_os = "windows")]
-pub(crate) const PLATFORM_IGNORED_DIRS: &[&str] = &[
+    #[cfg(target_os = "macos")]
+    "Library/Containers", // sandboxed apps data
+    #[cfg(target_os = "macos")]
+    "Library/Group Containers", // random application data and networking
+    #[cfg(target_os = "macos")]
+    "Library/pnpm",
+    #[cfg(target_os = "macos")]
+    "Library/Metadata",
+    #[cfg(target_os = "macos")]
+    "Library/Developer/CoreSimulator",
+    #[cfg(target_os = "macos")]
+    "Library/Android",
+    #[cfg(target_os = "macos")]
+    "Library/Logs",
+    #[cfg(target_os = "macos")]
+    "Library/Daemon Containers",
+    #[cfg(target_os = "macos")]
+    "Library/Trial",
+    #[cfg(target_os = "macos")]
+    "Library/Preferences",
+    #[cfg(target_os = "macos")]
+    "Library/Messages",
+    #[cfg(target_os = "macos")]
+    "Library/IdentityServices",
+    #[cfg(target_os = "windows")]
     "bin/Debug",
+    #[cfg(target_os = "windows")]
     "bin/Release",
+    #[cfg(target_os = "windows")]
     "Program Files",
+    #[cfg(target_os = "windows")]
     "Program Files (x86)",
+    #[cfg(target_os = "windows")]
     "AppData/Local",
+    #[cfg(target_os = "windows")]
     "AppData/Roaming",
 ];
 
-#[cfg(not(any(target_os = "macos", target_os = "windows")))]
-pub(crate) const PLATFORM_IGNORED_DIRS: &[&str] = &[];
-
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
 pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrides::Override> {
     use ignore::overrides::OverrideBuilder;
 
     let mut builder = OverrideBuilder::new(base_path);
-    for dir in NON_GIT_IGNORED_DIRS.iter().chain(PLATFORM_IGNORED_DIRS) {
+    for dir in IGNORED_DIRS {
         let pattern = format!("!**/{dir}/");
         if let Err(e) = builder.add(&pattern) {
             tracing::warn!("failed to add ignore pattern {pattern}: {e}");
@@ -54,16 +81,56 @@ pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrid
 
 pub(crate) fn is_non_code_directory(path: &Path) -> bool {
     let path_str = path.as_os_str().to_str().unwrap_or("");
-    NON_GIT_IGNORED_DIRS
-        .iter()
-        .chain(PLATFORM_IGNORED_DIRS)
-        .any(|&dir| {
-            #[cfg(target_os = "windows")]
-            let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
-            #[cfg(target_os = "windows")]
-            return path_str.contains(dir.as_str());
+    IGNORED_DIRS.iter().any(|&dir| {
+        // Entries are gitignore patterns for the walkers; here they are matched
+        // as substrings, so a leading `*` wildcard has to come off first.
+        let dir = dir.strip_prefix('*').unwrap_or(dir);
 
-            #[cfg(not(target_os = "windows"))]
-            path_str.contains(dir)
-        })
+        #[cfg(target_os = "windows")]
+        let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
+        #[cfg(target_os = "windows")]
+        return path_str.contains(dir.as_str());
+
+        #[cfg(not(target_os = "windows"))]
+        path_str.contains(dir)
+    })
+}
+
+#[cfg(all(test, target_os = "macos"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn home_machine_state_is_excluded_but_source_trees_are_not() {
+        // Representative machine state from a home index.
+        for rel in [
+            "Library/pnpm/store/v3/files/00/abcdef",
+            "Library/Preferences/com.apple.finder.plist",
+            "Library/Messages/prewarm.db-shm",
+            "Library/IdentityServices/TetraDB-identityservicesd.db-wal",
+            "Library/Developer/CoreSimulator/Devices/X/data/f",
+            "go/pkg/mod/github.com/x/y@v1/main.go",
+            ".cargo/registry/src/index.crates.io-1/serde-1.0/src/lib.rs",
+            "Library/Android/sdk/platforms/android-34/data/x",
+            ".local/state/nvim/fff+123+456.log",
+        ] {
+            assert!(
+                is_non_code_directory(Path::new(rel)),
+                "{rel} must not reach the index"
+            );
+        }
+
+        // Source trees under $HOME stay searchable.
+        for rel in [
+            "dev/chromium/third_party/blink/renderer/core/dom/node.cc",
+            "dev/fff/crates/fff-core/src/lib.rs",
+            "Documents/notes/todo.md",
+            "dev/myproj/pkg/mod/thing.go",
+        ] {
+            assert!(
+                !is_non_code_directory(Path::new(rel)),
+                "{rel} must stay searchable"
+            );
+        }
+    }
 }

crates/fff-core/src/index/bigram_filter.rs

@@ -1,3 +1,4 @@
+use crate::constants::MAX_INDEXABLE_FILE_SIZE;
 use ahash::AHashMap;
 use rayon::iter::{IndexedParallelIterator, ParallelIterator};
 use rayon::slice::ParallelSlice;
@@ -5,6 +6,8 @@ use std::cell::UnsafeCell;
 use std::sync::OnceLock;
 use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};
 
+use crate::{FileItem, constants};
+
 /// Maximum number of distinct bigrams tracked in the inverted index.
 /// 95 printable ASCII chars (32..=126) after lowercasing → ~70 distinct → 4900 possible.
 /// We cap at 5000 to cover all printable bigrams with margin.
@@ -14,6 +17,19 @@ const MAX_BIGRAM_COLUMNS: usize = 5000;
 /// Sentinel value: bigram has no allocated column.
 const NO_COLUMN: u16 = u16::MAX;
 
+/// 1024 × u64 = 8 KB covers all 65536 possible bigram keys.
+const SEEN_WORDS: usize = 1024;
+
+/// Content size where the branchless two-pass `add_long_content` overtakes
+/// the single-pass `add_short_content`: ~-35% on 4 KB files, but its fixed
+/// flush scan dominates files under ~1 KB. See bigram_bench `bigram_build`.
+const LONG_CONTENT_MIN_LEN: usize = 1024;
+
+thread_local! {
+    static NORM_BUF: std::cell::RefCell<Vec<u8>> =
+        std::cell::RefCell::new(Vec::with_capacity(4096));
+}
+
 /// Temporary sync dense builder for the bigram index.
 /// Builds from the many threads reading file contents in parallel
 pub struct BigramIndexBuilder {
@@ -87,17 +103,6 @@ impl BigramIndexBuilder {
         }
     }
 
-    /// SAFETY: caller must not access the same `word_idx` slot from
-    /// another thread concurrently. Partitioning in
-    /// `file_picker::build_bigram_index` enforces this.
-    #[inline(always)]
-    unsafe fn column_word_ptr(&self, col: u16, word_idx: usize) -> *mut u64 {
-        unsafe {
-            self.col_data_ptr()
-                .add(col as usize * self.words + word_idx)
-        }
-    }
-
     /// Test/bench accessor for a column's raw bitset words. Assumes the
     /// caller has joined all writers (no concurrent mutation).
     #[cfg(test)]
@@ -107,15 +112,7 @@ impl BigramIndexBuilder {
         &slab[start..start + self.words]
     }
 
-    // `pub` (via `#[doc(hidden)]`) only so the criterion bench can drive
-    // `add_file_content` directly. External consumers should use
-    // `build_bigram_index` instead.
-    ///
-    /// SAFETY: concurrent callers must partition `file_idx` by
-    /// word-aligned ranges so that `file_idx / 64` never collides across
-    /// threads. The `file_picker::build_bigram_index` driver enforces
-    /// this via `par_chunks` with a word-aligned chunk size.
-    #[doc(hidden)]
+    #[doc(hidden)] // `pub` (via `#[doc(hidden)]`) only for benchmarking
     pub fn add_file_content(&self, skip_builder: &Self, file_idx: usize, content: &[u8]) {
         if content.len() < 2 {
             return;
@@ -125,75 +122,176 @@ impl BigramIndexBuilder {
         let word_idx = file_idx / 64;
         let bit_mask = 1u64 << (file_idx % 64);
 
-        // Stack-local dedup bitsets: 1024 × u64 = 8 KB each, covers all 65536
-        // bigram keys with margin. Has to fit in L1 cache.
-        let mut seen_consec = [0u64; 1024];
-        let mut seen_skip = [0u64; 1024];
-
-        // Normalise each byte as we stream and carry a 2-byte history
-        // across iterations so each input byte is normalised exactly once
-        // even though it participates in up to three bigrams (as `cur`,
-        // then `prev`, then `skip_prev`). Benchmarked against a NEON
-        // pre-pass variant — the pre-pass needs a heap scratch per call,
-        // which kills throughput unless content is gigantic. Inline
-        // normalisation is the faster choice for realistic file sizes.
-        let bytes = content;
-        let len = bytes.len();
-
-        let mut n0 = normalize_byte_scalar(bytes[0]);
-        let mut n1 = normalize_byte_scalar(bytes[1]);
-
-        if n0 != u16::MAX && n1 != u16::MAX {
-            let key = (n0 << 8) | n1;
-            self.record_bigram(&mut seen_consec, key, word_idx, bit_mask);
-        }
-
-        for &b in &bytes[2..len] {
-            let cur = normalize_byte_scalar(b);
-            if cur != u16::MAX {
-                if n1 != u16::MAX {
-                    let key = (n1 << 8) | cur;
-                    self.record_bigram(&mut seen_consec, key, word_idx, bit_mask);
-                }
-                if n0 != u16::MAX {
-                    let key = (n0 << 8) | cur;
-                    skip_builder.record_bigram(&mut seen_skip, key, word_idx, bit_mask);
-                }
+        NORM_BUF.with_borrow_mut(|buf| {
+            let len = content.len();
+            if buf.len() < len {
+                buf.resize(len.next_power_of_two().max(4096), 0);
             }
-            n0 = n1;
-            n1 = cur;
-        }
+
+            normalize_bytes(content, &mut buf[..len]);
+            let n = &buf[..len];
+
+            // Both paths record the identical bigram set; the split exists
+            // purely for speed (see LONG_CONTENT_MIN_LEN).
+            if len >= LONG_CONTENT_MIN_LEN {
+                self.add_long_content(skip_builder, n, word_idx, bit_mask);
+            } else {
+                self.add_short_content(skip_builder, n, word_idx, bit_mask);
+            }
+        });
 
         self.populated.fetch_add(1, Ordering::Relaxed);
         skip_builder.populated.fetch_add(1, Ordering::Relaxed);
     }
 
-    /// Mark `key` as present for the file whose column-word is `word_idx`
-    /// and bit position is `bit_mask`, de-duplicating via the caller-owned
-    /// `seen` bitmap so we only touch the shared column slab at most once
-    /// per unique bigram per file.
-    ///
-    /// SAFETY: under the partitioning invariant on `add_file_content`
-    /// the `word_idx` slot this touches is owned exclusively by the
-    /// current thread, so a plain `|=` through the raw pointer is
-    /// race-free (no atomic RMW needed).
+    // Branchless two-pass: set every pair in stack-local bitmaps, including
+    // pairs touching the 0 sentinel — flush_seen masks those out. ~-35% vs
+    // the single pass on 4 KB files.
     #[inline(always)]
-    fn record_bigram(&self, seen: &mut [u64; 1024], key: u16, word_idx: usize, bit_mask: u64) {
+    fn add_long_content(&self, skip_builder: &Self, n: &[u8], word_idx: usize, bit_mask: u64) {
+        // Stack-local dedup bitsets: 1024 × u64 = 8 KB each, covers all 65536
+        // bigram keys. Has to fit in L1 cache.
+        let mut seen_consec = [0u64; SEEN_WORDS];
+        let mut seen_skip = [0u64; SEEN_WORDS];
+
+        let mut n0 = n[0];
+        let mut n1 = n[1];
+
+        let key = (n0 as usize) << 8 | n1 as usize;
+        // SAFETY: key < 65536, so key >> 6 < 1024 = SEEN_WORDS.
+        unsafe { *seen_consec.get_unchecked_mut(key >> 6) |= 1u64 << (key & 63) };
+
+        for &cur in &n[2..] {
+            let ck = (n1 as usize) << 8 | cur as usize;
+            let sk = (n0 as usize) << 8 | cur as usize;
+            unsafe {
+                *seen_consec.get_unchecked_mut(ck >> 6) |= 1u64 << (ck & 63);
+                *seen_skip.get_unchecked_mut(sk >> 6) |= 1u64 << (sk & 63);
+            }
+
+            n0 = n1;
+            n1 = cur;
+        }
+
+        self.flush_seen(&seen_consec, word_idx, bit_mask);
+        skip_builder.flush_seen(&seen_skip, word_idx, bit_mask);
+    }
+
+    #[inline(always)]
+    fn add_short_content(&self, skip_builder: &Self, n: &[u8], word_idx: usize, bit_mask: u64) {
+        let mut seen_consec = [0u64; SEEN_WORDS];
+        let mut seen_skip = [0u64; SEEN_WORDS];
+
+        let consec_base = self.col_data_ptr();
+        let consec_words = self.words;
+        let skip_base = skip_builder.col_data_ptr();
+        let skip_words = skip_builder.words;
+
+        let mut n0 = n[0];
+        let mut n1 = n[1];
+
+        if n0 != 0 && n1 != 0 {
+            let key = (n0 as u16) << 8 | n1 as u16;
+            self.record_bigram(
+                &mut seen_consec,
+                key,
+                word_idx,
+                bit_mask,
+                consec_base,
+                consec_words,
+            );
+        }
+
+        for &cur in &n[2..] {
+            if cur != 0 {
+                if n1 != 0 {
+                    let key = (n1 as u16) << 8 | cur as u16;
+                    self.record_bigram(
+                        &mut seen_consec,
+                        key,
+                        word_idx,
+                        bit_mask,
+                        consec_base,
+                        consec_words,
+                    );
+                }
+                if n0 != 0 {
+                    let key = (n0 as u16) << 8 | cur as u16;
+                    skip_builder.record_bigram(
+                        &mut seen_skip,
+                        key,
+                        word_idx,
+                        bit_mask,
+                        skip_base,
+                        skip_words,
+                    );
+                }
+            }
+            n0 = n1;
+            n1 = cur;
+        }
+    }
+
+    #[inline(always)]
+    fn record_bigram(
+        &self,
+        seen: &mut [u64; SEEN_WORDS],
+        key: u16,
+        word_idx: usize,
+        bit_mask: u64,
+        col_base: *mut u64,
+        words: usize,
+    ) {
         let k = key as usize;
         let w = k >> 6;
         let bit = 1u64 << (k & 63);
-        if seen[w] & bit == 0 {
-            seen[w] |= bit;
+        // SAFETY: w = key/64 with key: u16, so w < 1024 = SEEN_WORDS.
+        let prev = unsafe { *seen.get_unchecked(w) };
+        if prev & bit == 0 {
+            unsafe {
+                *seen.get_unchecked_mut(w) = prev | bit;
+            }
             let col = self.get_or_alloc_column(key);
             if col != NO_COLUMN {
                 unsafe {
-                    let p = self.column_word_ptr(col, word_idx);
+                    let p = col_base.add(col as usize * words + word_idx);
                     *p |= bit_mask;
                 }
             }
         }
     }
 
+    fn flush_seen(&self, seen: &[u64; SEEN_WORDS], word_idx: usize, bit_mask: u64) {
+        let col_base = self.col_data_ptr();
+        let words = self.words;
+        // SEEN_WORDS is a multiple of 8, so the remainder is always empty.
+        for (blk, block) in seen.as_chunks::<8>().0.iter().enumerate() {
+            // OR-test whole blocks so the mostly-empty bitmap scans fast.
+            if block.iter().fold(0u64, |a, &w| a | w) == 0 {
+                continue;
+            }
+            for (j, &word_bits) in block.iter().enumerate() {
+                let w = blk * 8 + j;
+                let mut bits = match w & 3 {
+                    _ if w < 4 => 0,
+                    0 => word_bits & !1,
+                    _ => word_bits,
+                };
+                while bits != 0 {
+                    let key = (w << 6 | bits.trailing_zeros() as usize) as u16;
+                    bits &= bits - 1;
+                    let col = self.get_or_alloc_column(key);
+                    if col != NO_COLUMN {
+                        unsafe {
+                            let p = col_base.add(col as usize * words + word_idx);
+                            *p |= bit_mask;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     pub fn is_ready(&self) -> bool {
         self.populated.load(Ordering::Relaxed) > 0
     }
@@ -480,22 +578,117 @@ impl BigramFilter {
     }
 }
 
-/// Map a single input byte to its normalised form used by the bigram
-/// builder: `u16::MAX` when not printable ASCII (outside `32..=126`),
-/// otherwise the lowercased byte value in `0..=126`. The `u16::MAX`
-/// sentinel can never collide with a printable-ASCII byte so the consumer
-/// can test `!= u16::MAX` without false positives.
-///
-/// Branchless and `#[inline(always)]`: LLVM lifts the ASCII-range check
-/// and the conditional-lowercase OR into a handful of instructions per
-/// call, so calling this inside a hot loop matches a hand-unrolled
-/// equivalent.
+/// Single-byte normalize: 0 for non-printable, lowercased byte otherwise.
+/// 0 is a safe sentinel: lowered printable bytes are 32..=126.
 #[inline(always)]
-fn normalize_byte_scalar(b: u8) -> u16 {
+fn normalize_byte_scalar(b: u8) -> u8 {
     let printable = b.wrapping_sub(32) <= 94;
-    // Branchless lowercase: OR 0x20 iff byte is in 'A'..='Z'.
     let lower = b | ((b.wrapping_sub(b'A') < 26) as u8 * 0x20);
-    if printable { lower as u16 } else { u16::MAX }
+    if printable { lower } else { 0 }
+}
+
+/// Bulk version: write `dst[i]` = `normalize_byte_scalar(src[i])` for `i`
+/// in `0..src.len()`. Inlined-scalar so LLVM auto-vectorises with the
+/// build's baseline SIMD; on x86_64 we runtime-dispatch to AVX2.
+/// Caller guarantees `dst.len() >= src.len()`.
+#[inline(always)]
+fn normalize_bytes(src: &[u8], dst: &mut [u8]) {
+    debug_assert!(dst.len() >= src.len());
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::is_x86_feature_detected!("avx2") {
+            unsafe { normalize_bytes_avx2(src, dst) };
+            return;
+        }
+    }
+
+    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
+    {
+        unsafe { normalize_bytes_neon(src, dst) };
+        return;
+    }
+
+    #[allow(unused)]
+    normalize_bytes_scalar(src, dst);
+}
+
+#[inline(always)]
+fn normalize_bytes_scalar(src: &[u8], dst: &mut [u8]) {
+    for (i, &b) in src.iter().enumerate() {
+        dst[i] = normalize_byte_scalar(b);
+    }
+}
+
+/// AVX2 normalize: 32 bytes/iter. AVX2 only has signed cmp, so unsigned
+/// range checks use `min(max(v, lo), hi) == v`.
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn normalize_bytes_avx2(src: &[u8], dst: &mut [u8]) {
+    use std::arch::x86_64::*;
+    let len = src.len();
+    let mut i = 0;
+    let p_lo = _mm256_set1_epi8(32);
+    let p_hi = _mm256_set1_epi8(126u8 as i8);
+    let u_lo = _mm256_set1_epi8(b'A' as i8);
+    let u_hi = _mm256_set1_epi8(b'Z' as i8);
+    let or20 = _mm256_set1_epi8(0x20);
+    while i + 32 <= len {
+        unsafe {
+            let v = _mm256_loadu_si256(src.as_ptr().add(i) as *const __m256i);
+            // printable_mask: v in [32, 126]
+            let clamp_p = _mm256_min_epu8(_mm256_max_epu8(v, p_lo), p_hi);
+            let printable = _mm256_cmpeq_epi8(v, clamp_p);
+            // is_upper_mask: v in [65, 90]
+            let clamp_u = _mm256_min_epu8(_mm256_max_epu8(v, u_lo), u_hi);
+            let is_upper = _mm256_cmpeq_epi8(v, clamp_u);
+            let or_bits = _mm256_and_si256(is_upper, or20);
+            let lower = _mm256_or_si256(v, or_bits);
+            let out = _mm256_and_si256(lower, printable);
+            _mm256_storeu_si256(dst.as_mut_ptr().add(i) as *mut __m256i, out);
+        }
+        i += 32;
+    }
+    while i < len {
+        dst[i] = normalize_byte_scalar(src[i]);
+        i += 1;
+    }
+}
+
+#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
+#[target_feature(enable = "neon")]
+unsafe fn normalize_bytes_neon(src: &[u8], dst: &mut [u8]) {
+    use std::arch::aarch64::*;
+    let len = src.len();
+    let mut i = 0;
+    let v32 = vdupq_n_u8(32);
+    let v127 = vdupq_n_u8(127);
+    let va = vdupq_n_u8(b'A');
+    let vz1 = vdupq_n_u8(b'Z' + 1);
+    let v20 = vdupq_n_u8(0x20);
+
+    while i + 16 <= len {
+        unsafe {
+            let v = vld1q_u8(src.as_ptr().add(i));
+            // printable: v >= 32 AND v < 127
+            let ge32 = vcgeq_u8(v, v32);
+            let lt127 = vcltq_u8(v, v127);
+            let print_mask = vandq_u8(ge32, lt127);
+            // is_upper: v >= 'A' AND v < 'Z'+1
+            let ge_a = vcgeq_u8(v, va);
+            let lt_z1 = vcltq_u8(v, vz1);
+            let upper_mask = vandq_u8(ge_a, lt_z1);
+            let or_bits = vandq_u8(upper_mask, v20);
+            let lower = vorrq_u8(v, or_bits);
+            let out = vandq_u8(lower, print_mask);
+
+            vst1q_u8(dst.as_mut_ptr().add(i), out);
+        }
+        i += 16;
+    }
+    while i < len {
+        dst[i] = normalize_byte_scalar(src[i]);
+        i += 1;
+    }
 }
 
 pub fn extract_bigrams(content: &[u8]) -> Vec<u16> {
@@ -596,7 +789,6 @@ impl BigramOverlay {
     }
 }
 
-pub const BIGRAM_CONTENT_CAP: usize = 64 * 1024;
 const BIGRAM_CHUNK_FILES: usize = 4 * 64;
 
 /// Sparse-column cutoff for the skip-1 sub-index. Rare skip columns add
@@ -605,51 +797,55 @@ const BIGRAM_CHUNK_FILES: usize = 4 * 64;
 const SKIP_INDEX_MIN_DENSITY_PCT: u32 = 12;
 
 thread_local! {
-    /// Per-rayon-worker reusable read buffer. 64 KB is too large to
-    /// keep on the default pthread stack (macOS ships 512 KB), so the
-    /// buffer lives on the heap behind a `Box<[u8; N]>`. TLS keeps the
-    /// allocation alive for the thread's lifetime so we pay the cost
-    /// once, not per file.
-    static READ_BUF: std::cell::RefCell<Box<[u8; BIGRAM_CONTENT_CAP]>> =
-        std::cell::RefCell::new(Box::new([0u8; BIGRAM_CONTENT_CAP]));
+    /// Reusable read buffer that is allocated per thread and used for reading files
+    static READ_BUF: std::cell::RefCell<Box<[u8]>> =
+        std::cell::RefCell::new(vec![0u8; MAX_INDEXABLE_FILE_SIZE].into_boxed_slice());
 }
 
-/// Outcome of processing one file's content.
-enum FileOutcome {
-    /// Content contained a NUL byte — mark the file as binary so future
-    /// greps skip it without re-reading.
-    Binary,
-    /// Read succeeded and the content was fed to the bigram builder.
-    Indexed,
-    /// File was empty or failed to open; nothing to do.
-    Skipped,
+/// Reads bigram chunk, we *SHOULD NOT* use mmap cache here because bigram is built off-lock
+/// if the watcher thread tries to invalidate mmap during the borrow from it - UAB or segfaut
+///
+/// mmap should only be used by the locked version of grep which absolutely minimizes any riscs
+#[inline]
+fn read_bigram_chunk<'a>(
+    file: &FileItem,
+    base_fd: libc::c_int,
+    base_path: &std::path::Path,
+    arena: crate::simd_path::ArenaPtr,
+    buf: &'a mut [u8],
+    path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
+) -> Option<&'a [u8]> {
+    let want = (file.size as usize).min(MAX_INDEXABLE_FILE_SIZE);
+    let filled = file.read_trimmed_into_buf(base_fd, base_path, arena, path_buf, &mut buf[..want]);
+    if filled == 0 {
+        return None;
+    }
+
+    let data = &buf[..filled];
+
+    Some(data)
 }
 
 #[tracing::instrument(skip_all, name = "Building Bigram Index", level = tracing::Level::DEBUG)]
 pub(crate) fn build_bigram_index(
     files: &[crate::types::FileItem],
-    budget: &crate::types::ContentCacheBudget,
     base_path: &std::path::Path,
     arena: crate::simd_path::ArenaPtr,
-) -> (BigramFilter, Vec<usize>) {
-    let start = std::time::Instant::now();
-    tracing::info!("Building bigram index for {} files...", files.len());
-
+) -> BigramFilter {
     let builder = BigramIndexBuilder::new(files.len());
     let skip_builder = BigramIndexBuilder::new(files.len());
 
-    // this does remove a memcpy for every single file + actually reducing open time on macos
     #[cfg(unix)]
     let base_fd: libc::c_int = open_base_dir_fd(base_path);
     #[cfg(not(unix))]
     let base_fd: i32 = -1;
 
-    // `content_binary` is only touched from the Binary branch below, so
-    // the mutex is cold in practice. A lock-free collector wasn't worth
-    // the complexity.
-    let content_binary: std::sync::Mutex<Vec<usize>> = std::sync::Mutex::new(Vec::new());
-
-    crate::file_picker::BACKGROUND_THREAD_POOL.install(|| {
+    // Always reads each file into the thread-local READ_BUF — never aliases the
+    // persistent mmap cache. See `read_bigram_chunk` for the rationale: this
+    // pass runs detached on the background pool without holding the picker
+    // read lock, so a watcher event mutating a `FileItem` would race any
+    // borrow we took from a cached `Mmap`.
+    crate::parallelism::BACKGROUND_THREAD_POOL.install(|| {
         files
             .par_chunks(BIGRAM_CHUNK_FILES)
             .enumerate()
@@ -657,183 +853,82 @@ pub(crate) fn build_bigram_index(
                 let base_idx = chunk_idx * BIGRAM_CHUNK_FILES;
                 for (offset, file) in chunk.iter().enumerate() {
                     let file_idx = base_idx + offset;
-                    let outcome = process_file(
-                        file,
-                        file_idx,
-                        &builder,
-                        &skip_builder,
-                        base_fd,
-                        base_path,
-                        arena,
-                        budget,
-                    );
-                    if matches!(outcome, FileOutcome::Binary) {
-                        content_binary.lock().unwrap().push(file_idx);
+
+                    if file.is_binary() || file.size == 0 {
+                        return;
                     }
+
+                    READ_BUF.with(|read_cell| {
+                        let mut buf = read_cell.borrow_mut();
+                        let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
+
+                        if let Some(content) = read_bigram_chunk(
+                            file,
+                            base_fd,
+                            base_path,
+                            arena,
+                            &mut buf[..],
+                            &mut path_buf,
+                        ) {
+                            // we have to manually ensure that every byte is a valid text byte to
+                            // perform this we have to scan every file, first 512 bytes is not enough
+                            // so basically we rely on the fact that first 2MB will always contain
+                            // an invalid text sequence if this is not a binary file.
+                            //
+                            // Need to find a better way to do this.
+                            file.set_binary(crate::types::detect_binary_content(content));
+
+                            builder.add_file_content(&skip_builder, file_idx, content);
+                        }
+                    });
                 }
             });
     });
 
     #[cfg(unix)]
     if base_fd >= 0 {
-        // SAFETY: we opened `base_fd` at the top of this function and
-        // no worker still references it once the rayon pool joined.
         unsafe { libc::close(base_fd) };
     }
 
-    let content_binary_vec = content_binary.into_inner().unwrap();
-
-    let cols = builder.columns_used();
     let mut index = builder.compress(None);
     let skip_index = skip_builder.compress(Some(SKIP_INDEX_MIN_DENSITY_PCT));
     index.set_skip_index(skip_index);
 
-    // Builder buffers were freed by `compress()` above (one deallocation
-    // each); nudge mimalloc to return them (and any transient allocs)
-    // to the OS.
+    // in progress bigram walk + rust's ignore crate allocates shit ton of garbage memory
+    // all custom allocators would think this is available resource while we do not allocate
+    // after the sync, so it's very important to let the unused memory go back to the OS
     crate::file_picker::hint_allocator_collect();
 
-    tracing::info!(
-        "Bigram index built in {:.2}s — {} dense columns for {} files",
-        start.elapsed().as_secs_f64(),
-        cols,
-        files.len(),
-    );
-    if !content_binary_vec.is_empty() {
-        tracing::info!(
-            "Bigram build detected {} content-binary files (not caught by extension)",
-            content_binary_vec.len(),
-        );
-    }
-
-    (index, content_binary_vec)
+    index
 }
 
-/// Process one file: read up to `BIGRAM_CONTENT_CAP` bytes, feed them
-/// to the bigram builder (or record as binary / skipped).
-///
-/// `base_fd` is the parent-directory fd for the Unix `openat` fast
-/// path, or `-1` to force the portable `std::fs::File::open` fallback.
-#[inline]
-#[allow(clippy::too_many_arguments)]
-fn process_file(
-    file: &crate::types::FileItem,
-    file_idx: usize,
-    builder: &BigramIndexBuilder,
-    skip_builder: &BigramIndexBuilder,
-    base_fd: i32,
+#[tracing::instrument(skip_all, name = "Sniffing Large Files Binary", level = tracing::Level::DEBUG)]
+pub(crate) fn sniff_binary_for_non_indexable(
+    files: &[FileItem],
     base_path: &std::path::Path,
     arena: crate::simd_path::ArenaPtr,
-    budget: &crate::types::ContentCacheBudget,
-) -> FileOutcome {
-    if file.is_binary() || file.size == 0 || file.size > budget.max_file_size {
-        return FileOutcome::Skipped;
-    }
-
-    // Zero-copy fast path: the warmup phase may have cached this file's
-    // content already. Avoid re-reading from disk.
-    if let Some(cached) = file.get_content(arena, base_path, budget) {
-        if crate::file_picker::detect_binary_content(cached) {
-            return FileOutcome::Binary;
-        }
-        let capped = &cached[..cached.len().min(BIGRAM_CONTENT_CAP)];
-        builder.add_file_content(skip_builder, file_idx, capped);
-        return FileOutcome::Indexed;
-    }
-
-    let want = (file.size as usize).min(BIGRAM_CONTENT_CAP);
+    cancelled: &std::sync::atomic::AtomicBool,
+) {
+    // Non-indexable files are few in a typical repo, so a serial pass with a
+    // single reused chunk buffer beats spinning up the thread pool.
     let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
+    let mut chunk = vec![0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE];
+    use std::sync::atomic::Ordering;
 
-    READ_BUF.with(|read_cell| {
-        let mut buf = read_cell.borrow_mut();
-        let filled = read_file_content(
-            file,
-            base_fd,
-            base_path,
-            arena,
-            &mut path_buf,
-            &mut buf[..want],
-        );
-        if filled == 0 {
-            return FileOutcome::Skipped;
+    for (i, file) in files.iter().enumerate() {
+        // check every 256 files to avoid useless work
+        if (i & 0xFF) == 0 && cancelled.load(Ordering::Acquire) {
+            return;
         }
-        let data = &buf[..filled];
-        if crate::file_picker::detect_binary_content(data) {
-            return FileOutcome::Binary;
-        }
-        builder.add_file_content(skip_builder, file_idx, data);
-        FileOutcome::Indexed
-    })
-}
 
-/// Read up to `buf.len()` bytes of `file`'s content into `buf`. Returns
-/// the number of bytes actually read (0 on any error, so callers treat
-/// failures as "skip").
-#[inline]
-fn read_file_content(
-    file: &crate::types::FileItem,
-    base_fd: i32,
-    base_path: &std::path::Path,
-    arena: crate::simd_path::ArenaPtr,
-    path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
-    buf: &mut [u8],
-) -> usize {
-    #[cfg(unix)]
-    {
-        read_file_content_unix(file, base_fd, base_path, arena, path_buf, buf)
-    }
-    #[cfg(not(unix))]
-    {
-        let _ = base_fd;
-        read_file_content_std(file, base_path, arena, path_buf, buf)
-    }
-}
-
-#[cfg(unix)]
-fn read_file_content_unix(
-    file: &crate::types::FileItem,
-    base_fd: libc::c_int,
-    base_path: &std::path::Path,
-    arena: crate::simd_path::ArenaPtr,
-    path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
-    buf: &mut [u8],
-) -> usize {
-    let fd = if base_fd >= 0 {
-        let rel_cstr = file.write_relative_cstr(arena, path_buf);
-        // SAFETY: `rel_cstr` is NUL-terminated, `base_fd` is a valid
-        // directory descriptor owned by the caller.
-        unsafe { libc::openat(base_fd, rel_cstr.as_ptr(), libc::O_RDONLY) }
-    } else {
-        use std::os::unix::io::IntoRawFd;
-        let abs = file.write_absolute_path(arena, base_path, path_buf);
-        match std::fs::File::open(abs) {
-            Ok(f) => f.into_raw_fd(),
-            Err(_) => return 0,
+        // check only the files that we are able to grep
+        if file.size == 0 || file.size > constants::MAX_FFFILE_SIZE {
+            continue;
         }
-    };
-    if fd < 0 {
-        return 0;
-    }
 
-    let mut filled = 0usize;
-    while filled < buf.len() {
-        // SAFETY: `fd` is an owned descriptor, `buf[filled..]` is a
-        // valid writable slice for `buf.len() - filled` bytes.
-        let n = unsafe {
-            libc::read(
-                fd,
-                buf[filled..].as_mut_ptr() as *mut libc::c_void,
-                (buf.len() - filled) as libc::size_t,
-            )
-        };
-        if n <= 0 {
-            break;
-        }
-        filled += n as usize;
+        let abs = file.write_absolute_path(arena, base_path, &mut path_buf);
+        file.detect_binary_per_byte(abs, &mut chunk);
     }
-    // SAFETY: matching close for the owned descriptor.
-    unsafe { libc::close(fd) };
-    filled
 }
 
 /// Open the base directory for the `openat` fast path. Returns `-1` on
@@ -858,33 +953,6 @@ fn open_base_dir_fd(base_path: &std::path::Path) -> libc::c_int {
     }
 }
 
-/// Portable fallback (Windows + non-`openat` Unix): `std::fs::File` +
-/// `Read::read` into `buf`. Used on Windows unconditionally, and on
-/// Unix when the base directory fd could not be opened.
-#[cfg(not(unix))]
-fn read_file_content_std(
-    file: &crate::types::FileItem,
-    base_path: &std::path::Path,
-    arena: crate::simd_path::ArenaPtr,
-    path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
-    buf: &mut [u8],
-) -> usize {
-    use std::io::Read;
-    let abs = file.write_absolute_path(arena, base_path, path_buf);
-    let Ok(mut f) = std::fs::File::open(abs) else {
-        return 0;
-    };
-    let mut filled = 0usize;
-    while filled < buf.len() {
-        match f.read(&mut buf[filled..]) {
-            Ok(0) => break,
-            Ok(n) => filled += n,
-            Err(_) => return 0,
-        }
-    }
-    filled
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1115,6 +1183,25 @@ mod tests {
         run_and_compare(&mixed[..192]); // SIMD path with scalar tail
     }
 
+    #[test]
+    fn add_file_long_short_paths_agree() {
+        // Same mixed content checked just below, at, and above
+        // LONG_CONTENT_MIN_LEN so both add_short_content and add_long_content
+        // are validated against the reference implementation.
+        let mut mixed = Vec::with_capacity(LONG_CONTENT_MIN_LEN * 2);
+        for i in 0..LONG_CONTENT_MIN_LEN * 2 {
+            mixed.push(match i % 11 {
+                0 => 0,
+                1 => 0x7F,
+                2 => b'\n',
+                _ => 32 + ((i * 31) % 95) as u8,
+            });
+        }
+        run_and_compare(&mixed[..LONG_CONTENT_MIN_LEN - 1]);
+        run_and_compare(&mixed[..LONG_CONTENT_MIN_LEN]);
+        run_and_compare(&mixed);
+    }
+
     #[test]
     fn add_file_respects_file_count_boundary() {
         // file_count=100, file_idx=63 (last bit in word 0) and file_idx=64

crates/fff-core/src/index/bigram_query.rs

@@ -1,26 +1,48 @@
-//! Regex → bigram decomposition for the inverted bigram index.
-//!
-//! Parses a regex pattern with `regex-syntax`, walks the HIR to extract
-//! guaranteed bigram keys (u16), and evaluates them as an AND/OR query tree
-//! against [`BigramFilter`]'s inverted posting lists.
-//!
-//! Two bigram types are extracted:
-//! - **Consecutive** (gap=0): adjacent byte pairs `(pattern[i], pattern[i+1])`
-//! - **Sparse-1** (gap=1): pairs across a single-byte wildcard, e.g. `a.b → (a,b)`
-//!
-//! The sparse-1 extraction is the key feature: regex patterns like `foo.bar`
-//! yield the cross-boundary sparse-1 bigram `(o,b)` that provides strong
-//! filtering even when the `.` prevents any consecutive cross-boundary bigram.
-
-use crate::bigram_filter::BigramFilter;
+use crate::index::bigram_filter::BigramFilter;
 use regex_syntax::hir::{Class, Hir, HirKind};
 use smallvec::SmallVec;
 use std::borrow::Cow;
 
-/// Maximum byte values to enumerate from a character class.
-/// Larger classes are treated as unknown (no bigram extractable).
 const MAX_CLASS_EXPAND: usize = 16;
 
+// stack inlined array padded with 0 and tracked length
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct InlineArray {
+    bytes: [u8; MAX_CLASS_EXPAND],
+    len: usize,
+}
+
+impl InlineArray {
+    const fn new() -> Self {
+        Self {
+            bytes: [0; MAX_CLASS_EXPAND],
+            len: 0,
+        }
+    }
+
+    fn from_byte(b: u8) -> Self {
+        let mut set = Self::new();
+        set.push(b);
+        set
+    }
+
+    /// Append a byte; no-op if already full (callers guard against this).
+    fn push(&mut self, b: u8) {
+        if self.len < MAX_CLASS_EXPAND {
+            self.bytes[self.len] = b;
+            self.len += 1;
+        }
+    }
+}
+
+impl std::ops::Deref for InlineArray {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &self.bytes[..self.len]
+    }
+}
+
 #[inline]
 fn consec_key(a: u8, b: u8) -> Option<u16> {
     let al = a.to_ascii_lowercase();
@@ -121,19 +143,15 @@ impl BigramQuery {
                 }
                 let mut result: Option<Vec<u64>> = None;
                 for child in children {
-                    match child.evaluate_cow(index) {
-                        // Any branch can't be filtered → whole OR can't be filtered
-                        None => return None,
-                        Some(child_bits) => {
-                            result = Some(match result {
-                                None => child_bits.into_owned(),
-                                Some(mut r) => {
-                                    bitset_or(&mut r, &child_bits);
-                                    r
-                                }
-                            });
+                    // Any branch can't be filtered -> whole OR can't be filtered
+                    let child_bits = child.evaluate_cow(index)?;
+                    result = Some(match result {
+                        None => child_bits.into_owned(),
+                        Some(mut r) => {
+                            bitset_or(&mut r, &child_bits);
+                            r
                         }
-                    }
+                    });
                 }
                 result.map(Cow::Owned)
             }
@@ -141,14 +159,10 @@ impl BigramQuery {
     }
 }
 
-/// Intermediate state tracked during HIR traversal for bigram extraction.
 struct HirInfo {
     query: BigramQuery,
-    /// Possible first bytes (lowercased, printable ASCII) when this node matches.
-    first: Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>,
-    /// Possible last bytes.
-    last: Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>,
-    /// Whether this node can match the empty string.
+    first: Option<InlineArray>,
+    last: Option<InlineArray>,
     can_be_empty: bool,
 }
 
@@ -185,7 +199,7 @@ pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQue
         return BigramQuery::Any;
     }
 
-    // For very short queries (0 typos), AND all bigrams — exact subsequence.
+    // the simplest case, just check that every bigram is present either consec or not
     if max_typos == 0 {
         return simplify_and(
             bigram_keys
@@ -225,7 +239,7 @@ pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQue
         return simplify_and(probes.iter().map(|&k| BigramQuery::Consec(k)).collect());
     }
 
-    // Generate all C(n, required) subsets → OR(AND(subset), ...)
+    // Generate all C(n, required) subsets as OR(AND(subset), ...)
     let mut branches = Vec::new();
     let mut combo = vec![0u16; required];
     combine(&probes, required, 0, 0, &mut combo, &mut branches);
@@ -282,7 +296,7 @@ fn decompose(hir: &Hir) -> HirInfo {
             match bytes {
                 Some(b) if !b.is_empty() => HirInfo {
                     query: BigramQuery::Any,
-                    first: Some(b.clone()),
+                    first: Some(b),
                     last: Some(b),
                     can_be_empty,
                 },
@@ -344,13 +358,13 @@ fn decompose_literal(bytes: &[u8]) -> HirInfo {
     if lower.len() == 1 {
         let b = lower[0];
         let first = if (32..=126).contains(&b) {
-            Some(SmallVec::from_slice(&[b]))
+            Some(InlineArray::from_byte(b))
         } else {
             None
         };
         return HirInfo {
             query: BigramQuery::Any,
-            first: first.clone(),
+            first,
             last: first,
             can_be_empty: false,
         };
@@ -380,12 +394,12 @@ fn decompose_literal(bytes: &[u8]) -> HirInfo {
     HirInfo {
         query: simplify_and(qs),
         first: if (32..=126).contains(&first_byte) {
-            Some(SmallVec::from_slice(&[first_byte]))
+            Some(InlineArray::from_byte(first_byte))
         } else {
             None
         },
         last: if (32..=126).contains(&last_byte) {
-            Some(SmallVec::from_slice(&[last_byte]))
+            Some(InlineArray::from_byte(last_byte))
         } else {
             None
         },
@@ -401,22 +415,20 @@ fn decompose_concat(parts: &[Hir]) -> HirInfo {
     let infos: Vec<HirInfo> = parts.iter().map(decompose).collect();
     let mut qs: Vec<BigramQuery> = Vec::new();
 
-    // 1. Collect child bigrams
     for info in &infos {
         if !info.query.is_any() {
             qs.push(info.query.clone());
         }
     }
 
-    // 2. Dense cross-boundary between adjacent mandatory parts
+    //  Dense cross-boundary between adjacent mandatory parts
     for pair in infos.windows(2) {
         if !pair[0].can_be_empty && !pair[1].can_be_empty {
             push_cross_consec(&mut qs, pair[0].last.as_deref(), pair[1].first.as_deref());
         }
     }
 
-    // 3. Sparse-1 cross-boundary: across a single 1-byte-wide middle part.
-    //    Catches `foo.bar` → sparse-1 `(o,b)` across the dot.
+    //  Sparse-1 cross-boundary: across a single 1 byte wide middle part
     if parts.len() >= 3 {
         for i in 0..parts.len() - 2 {
             let left = &infos[i];
@@ -464,8 +476,8 @@ fn decompose_alternation(alts: &[Hir]) -> HirInfo {
     }
 }
 
-fn expand_class(class: &Class) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut bytes: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn expand_class(class: &Class) -> Option<InlineArray> {
+    let mut bytes = InlineArray::new();
     match class {
         Class::Bytes(bc) => {
             for range in bc.ranges() {
@@ -473,6 +485,7 @@ fn expand_class(class: &Class) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
                 if bytes.len() + count > MAX_CLASS_EXPAND {
                     return None;
                 }
+
                 for b in range.start()..=range.end() {
                     if (32..=126).contains(&b) {
                         let lower = b.to_ascii_lowercase();
@@ -554,11 +567,11 @@ fn cross_product(last: Option<&[u8]>, first: Option<&[u8]>, skip: bool) -> Optio
     }
 }
 
-fn collect_first(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn collect_first(infos: &[HirInfo]) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
     for info in infos {
         if let Some(ref bytes) = info.first {
-            for &b in bytes {
+            for &b in bytes.iter() {
                 if !result.contains(&b) {
                     if result.len() >= MAX_CLASS_EXPAND {
                         return None;
@@ -580,11 +593,11 @@ fn collect_first(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>
     }
 }
 
-fn collect_last(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn collect_last(infos: &[HirInfo]) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
     for info in infos.iter().rev() {
         if let Some(ref bytes) = info.last {
-            for &b in bytes {
+            for &b in bytes.iter() {
                 if !result.contains(&b) {
                     if result.len() >= MAX_CLASS_EXPAND {
                         return None;
@@ -606,22 +619,17 @@ fn collect_last(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
     }
 }
 
-fn merge_byte_sets<'a>(
-    iter: impl Iterator<Item = &'a Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>>,
-) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn merge_byte_sets<'a>(iter: impl Iterator<Item = &'a Option<InlineArray>>) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
     for opt in iter {
-        match opt {
-            None => return None,
-            Some(bytes) => {
-                for &b in bytes {
-                    if !result.contains(&b) {
-                        if result.len() >= MAX_CLASS_EXPAND {
-                            return None;
-                        }
-                        result.push(b);
-                    }
+        let bytes = opt.as_ref()?;
+
+        for &b in bytes.iter() {
+            if !result.contains(&b) {
+                if result.len() >= MAX_CLASS_EXPAND {
+                    return None;
                 }
+                result.push(b);
             }
         }
     }
@@ -669,7 +677,7 @@ fn simplify_or(children: Vec<BigramQuery>) -> BigramQuery {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bigram_filter::BigramIndexBuilder;
+    use crate::index::bigram_filter::BigramIndexBuilder;
 
     /// Build a tiny index from the given file contents for testing.
     fn build_test_index(files: &[&[u8]]) -> BigramFilter {
@@ -757,7 +765,7 @@ mod tests {
 
     #[test]
     fn sparse1_across_digit() {
-        // "foo\dbar" → sparse-1 (o,b) across \d
+        // "foo\dbar" -> sparse-1 (o,b) across \d
         let idx = build_test_index(&[
             b"foo3bar baz", // 0: has all bigrams
             b"foobar baz",  // 1: has consecutive (o,b) but pattern needs sparse-1
@@ -795,7 +803,6 @@ mod tests {
 
     #[test]
     fn optional_group_excluded() {
-        // (bar)? is optional — its bigrams are not required
         let q = regex_to_bigram_query("foo(bar)?baz");
         assert!(!q.is_any());
 
@@ -813,7 +820,7 @@ mod tests {
 
     #[test]
     fn repetition_min2_cross_boundary() {
-        // (ab){2,} → bigram "ab" + cross-boundary "b","a"
+        // (ab){2,} -> bigram "ab" + cross-boundary "b","a"
         let q = regex_to_bigram_query("(ab){2,}");
         assert!(!q.is_any());
 
@@ -830,16 +837,14 @@ mod tests {
 
     #[test]
     fn two_dots_no_sparse1() {
-        // "a..b" — two 1-byte parts between a and b, not a single 1-byte part
-        // No sparse-1 (a,b) should be extracted
         let q = regex_to_bigram_query("a..b");
-        // Single-char literals with 2 unknown bytes between → Any
+        // Single-char literals with 2 unknown bytes between -> Any
         assert!(q.is_any());
     }
 
     #[test]
     fn character_class_cross_boundary() {
-        // [abc]de → cross-boundary OR(ad,bd,cd) + bigram de
+        // [abc]de -> cross-boundary OR(ad,bd,cd) + bigram de
         // All three class variants must appear in the corpus so the OR
         // branches are tracked in the index (untracked bigrams make the
         // OR conservatively return None, which is correct but untestable).
@@ -861,8 +866,6 @@ mod tests {
         assert!(!BigramFilter::is_candidate(&candidates, 3));
     }
 
-    // ── Helpers for inspecting query trees ──────────────────────────
-
     fn has_consec(q: &BigramQuery, a: u8, b: u8) -> bool {
         let Some(key) = consec_key(a, b) else {
             return false;
@@ -896,13 +899,12 @@ mod tests {
     /// plus typical grep patterns used by agentic tools.
     ///
     /// Each entry: `(regex, Option<&[Bg]>)`.
-    ///   - `None`        → pure classes / unsupported syntax, Any is acceptable.
-    ///   - `Some(&[..])` → must be non-Any, and every listed bigram must appear.
+    ///   - `None`        -> pure classes / unsupported syntax, Any is acceptable.
+    ///   - `Some(&[..])` -> must be non-Any, and every listed bigram must appear.
     #[test]
     fn common_regex_patterns() {
         #[rustfmt::skip]
         let cases: &[(&str, Option<&[Bg]>)] = &[
-            // ── Pure-class / anchor / unsupported → Any is fine ──────
             (r"^\d+$",                                                      None), // 1.  whole numbers
             (r"^\d*\.\d+$",                                                 None), // 2.  decimals
             (r"^\d*(\.\d+)?$",                                              None), // 3.  whole + decimal
@@ -931,11 +933,9 @@ mod tests {
             (r"^[\w,\s-]+\.[A-Za-z]{3}$",                                 None), // 27. filename
             (r"^[A-PR-WY][1-9]\d\s?\d{4}[1-9]$",                         None), // 28. HK ID
 
-            // ── Patterns with extractable literal bigrams ────────────
-
             // 13. URL with required protocol
             (r"https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", Some(&[
-                ("ht", C), ("tt", C), ("tp", C),   // from "http"
+                ("ht", C), ("tt", C), ("tp", C),    // from "http"
                 ("ht", S), ("tp", S),               // from "http" skip-1
                 (":/", C), ("//", C),               // from "://"
             ])),
@@ -943,31 +943,31 @@ mod tests {
             // 29. fn\s+\w+
             (r"fn\s+\w+", Some(&[
                 ("fn", C),                          // from "fn"
-                ("n ", C),                          // cross-boundary: 'n' → \s starts ' '
+                ("n ", C),                          // cross-boundary: 'n' -> \s starts ' '
             ])),
 
             // 30. use\s+crate::
             (r"use\s+crate::", Some(&[
-                ("us", C), ("se", C), ("ue", S),   // from "use"
-                ("cr", C), ("ra", C), ("at", C),   // from "crate"
+                ("us", C), ("se", C), ("ue", S),    // from "use"
+                ("cr", C), ("ra", C), ("at", C),    // from "crate"
                 ("te", C), ("::", C),
-                ("ca", S), ("rt", S), ("ae", S),   // "crate" skip-1
+                ("ca", S), ("rt", S), ("ae", S),    // "crate" skip-1
             ])),
 
             // 31. unwrap\(\)|expect\(
             (r"unwrap\(\)|expect\(", Some(&[
-                ("nw", C), ("wr", C), ("ra", C),   // "unwrap("
+                ("nw", C), ("wr", C), ("ra", C),    // "unwrap("
                 ("ap", C), ("p(", C),
-                ("xp", C), ("pe", C), ("ec", C),   // "expect("
+                ("xp", C), ("pe", C), ("ec", C),    // "expect("
                 ("ct", C), ("t(", C),
             ])),
 
             // 32. TODO|FIXME|HACK
             (r"TODO|FIXME|HACK", Some(&[
-                ("to", C), ("od", C), ("do", C),   // "TODO"
-                ("fi", C), ("ix", C), ("xm", C),   // "FIXME"
+                ("to", C), ("od", C), ("do", C),    // "TODO"
+                ("fi", C), ("ix", C), ("xm", C),    // "FIXME"
                 ("me", C),
-                ("ha", C), ("ac", C), ("ck", C),   // "HACK"
+                ("ha", C), ("ac", C), ("ck", C),    // "HACK"
                 ("hc", S), ("ak", S),               // "HACK" skip-1
             ])),
         ];

crates/fff-core/src/index/candidates.rs

@@ -0,0 +1,118 @@
+use super::{BigramFilter, BigramOverlay, extract_bigrams};
+use super::{fuzzy_to_bigram_query, regex_to_bigram_query};
+
+/// Number of evenly-spaced probe bigrams used by the fuzzy candidate query.
+const FUZZY_PROBE_COUNT: usize = 7;
+
+#[inline]
+fn set_bit(candidates: &mut [u64], file_idx: usize) {
+    let word = file_idx / 64;
+    if word < candidates.len() {
+        candidates[word] |= 1u64 << (file_idx % 64);
+    }
+}
+
+#[inline]
+fn clear_tombstones(candidates: &mut [u64], overlay: &BigramOverlay) {
+    for (r, t) in candidates.iter_mut().zip(overlay.tombstones().iter()) {
+        *r &= !t;
+    }
+}
+
+/// Number of base files covered by the bigram bitset; files past this
+/// boundary (overflow, max 1024) are always scanned.
+#[inline]
+pub(crate) fn bigram_boundary(overlay: Option<&BigramOverlay>, files_len: usize) -> usize {
+    overlay.map(|o| o.base_file_count()).unwrap_or(files_len)
+}
+
+/// Candidate bitset for literal patterns, OR-ed across all of them: a file is
+/// a candidate when it contains the bigrams of ANY pattern. Overlay-modified
+/// files are re-checked against each pattern's bigrams.
+pub(crate) fn literal_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    patterns: &[&str],
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let mut combined: Option<Vec<u64>> = None;
+    for pattern in patterns {
+        if let Some(candidates) = index.query(pattern.as_bytes()) {
+            combined = Some(match combined {
+                None => candidates,
+                Some(mut acc) => {
+                    acc.iter_mut()
+                        .zip(candidates.iter())
+                        .for_each(|(a, b)| *a |= *b);
+                    acc
+                }
+            });
+        }
+    }
+
+    let mut candidates = combined?;
+    if let Some(overlay) = overlay {
+        clear_tombstones(&mut candidates, overlay);
+        for pattern in patterns {
+            let pattern_bigrams = extract_bigrams(pattern.as_bytes());
+            for file_idx in overlay.query_modified(&pattern_bigrams) {
+                set_bit(&mut candidates, file_idx);
+            }
+        }
+    }
+    Some(candidates)
+}
+
+/// Candidate bitset for a regex pattern: the regex HIR is decomposed into an
+/// AND/OR bigram query tree (supports alternation, optional groups, character
+/// classes, and sparse-1 bigrams across single-byte wildcards). Since modified
+/// file contents can't be re-checked against a regex cheaply, all
+/// overlay-modified files are conservatively added.
+pub(crate) fn regex_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    pattern: &str,
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let bq = regex_to_bigram_query(pattern);
+    if bq.is_any() {
+        return None;
+    }
+    let candidates = bq.evaluate(index)?;
+    Some(add_all_modified(candidates, overlay))
+}
+
+/// Candidate bitset for a fuzzy pattern: evenly-spaced probe bigrams with a
+/// typo allowance (widely-spaced probes are far more selective than sliding
+/// windows of adjacent bigrams). All overlay-modified files are added.
+pub(crate) fn fuzzy_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    pattern: &str,
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let bq = fuzzy_to_bigram_query(pattern, FUZZY_PROBE_COUNT);
+    if bq.is_any() {
+        return None;
+    }
+    let candidates = bq.evaluate(index)?;
+    Some(add_all_modified(candidates, overlay))
+}
+
+#[inline]
+fn ready_index(index: Option<&BigramFilter>) -> Option<&BigramFilter> {
+    index.filter(|idx| idx.is_ready())
+}
+
+fn add_all_modified(mut candidates: Vec<u64>, overlay: Option<&BigramOverlay>) -> Vec<u64> {
+    if let Some(overlay) = overlay {
+        clear_tombstones(&mut candidates, overlay);
+        for file_idx in overlay.modified_indices() {
+            set_bit(&mut candidates, file_idx);
+        }
+    }
+    candidates
+}

crates/fff-core/src/index/constraints.rs

@@ -0,0 +1,950 @@
+//! Constraint-based prefiltering for search queries.
+
+use fff_query_parser::{Constraint, GitStatusFilter};
+use smallvec::SmallVec;
+
+use crate::git::is_modified_status;
+use crate::simd_path::ArenaPtr;
+use crate::simd_string_utils::memmem::find_case_insensitive_short;
+
+const PAR_THRESHOLD: usize = 10_000;
+
+pub(crate) trait Constrainable {
+    fn write_file_name(&self, arena: ArenaPtr, out: &mut String);
+    fn git_status(&self) -> Option<git2::Status>;
+    fn write_relative_path(&self, arena: ArenaPtr, out: &mut String);
+    fn is_overflow(&self) -> bool;
+}
+
+/// Stored/canonical paths use `/`; also accept `\` so a Windows user typing
+/// a native separator in a query still matches.
+#[inline]
+fn is_path_sep(b: u8) -> bool {
+    b == b'/' || b == b'\\'
+}
+
+#[inline]
+fn path_slice_eq(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    a.iter().zip(b).all(|(x, y)| {
+        if is_path_sep(*x) && is_path_sep(*y) {
+            true
+        } else {
+            x.eq_ignore_ascii_case(y)
+        }
+    })
+}
+
+/// Path ends with suffix at a path-separator boundary (case-insensitive).
+#[inline]
+pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool {
+    let path_bytes = path.as_bytes();
+    let suffix_bytes = suffix.as_bytes();
+    if path_bytes.len() < suffix_bytes.len() {
+        return false;
+    }
+
+    let start = path.len() - suffix.len();
+
+    // Multi-byte UTF-8 may put `start` inside a char.
+    if !path.is_char_boundary(start) {
+        return false;
+    }
+
+    if !path_slice_eq(&path_bytes[start..], suffix_bytes) {
+        return false;
+    }
+
+    // Exact or preceded by a separator. Scan backward past any multi-byte
+    // continuation bytes to find the preceding ASCII byte.
+    if start == 0 {
+        return true;
+    }
+    let mut i = start;
+    while i > 0 {
+        i -= 1;
+        if path_bytes[i] < 128 {
+            return is_path_sep(path_bytes[i]);
+        }
+    }
+    false
+}
+
+#[inline]
+pub fn file_has_extension(file_name: &str, ext: &str) -> bool {
+    let name_bytes = file_name.as_bytes();
+    let ext_bytes = ext.as_bytes();
+    if name_bytes.len() <= ext_bytes.len() + 1 {
+        return false;
+    }
+    let start = name_bytes.len() - ext_bytes.len() - 1;
+    if start > 0 && !file_name.is_char_boundary(start) {
+        return false;
+    }
+    name_bytes.get(start) == Some(&b'.') && name_bytes[start + 1..].eq_ignore_ascii_case(ext_bytes)
+}
+
+/// Matches multi-segment queries like `libswscale/aarch64`.
+#[inline]
+pub fn path_contains_segment(path: &str, segment: &str) -> bool {
+    let path_bytes = path.as_bytes();
+    let segment_bytes = segment.as_bytes();
+    let segment_len = segment_bytes.len();
+
+    if path_bytes.len() > segment_len
+        && is_path_sep(path_bytes[segment_len])
+        && path.is_char_boundary(segment_len)
+        && path_slice_eq(&path_bytes[..segment_len], segment_bytes)
+    {
+        return true;
+    }
+
+    if path_bytes.len() < segment_len + 2 {
+        return false;
+    }
+
+    for i in 0..path_bytes.len().saturating_sub(segment_len + 1) {
+        if is_path_sep(path_bytes[i]) {
+            let start = i + 1;
+            let end = start + segment_len;
+            if end < path_bytes.len()
+                && is_path_sep(path_bytes[end])
+                && path.is_char_boundary(start)
+                && path.is_char_boundary(end)
+                && path_slice_eq(&path_bytes[start..end], segment_bytes)
+            {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Returns `None` if no constraints are present, `Some(filtered)` otherwise.
+///
+/// Constraint semantics:
+/// - All `Extension` constraints OR together (file matches if ANY extension hits).
+///   They're split out up front so the per-item loop reads the OR predicate as a
+///   single short-circuit check, not as N AND-merged sub-constraints.
+/// - Every other constraint kind ANDs (file matches only if ALL hold). They're
+///   evaluated in order with short-circuit on first failure.
+pub(crate) fn apply_constraints<'a, T: Constrainable + Sync>(
+    items: &'a [T],
+    constraints: &[Constraint<'_>],
+    base_arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> Option<Vec<&'a T>> {
+    if constraints.is_empty() {
+        return None;
+    }
+    let plan = ConstraintPlan::build(constraints, items, base_arena, overflow_arena);
+    Some(plan.run(items, base_arena, overflow_arena))
+}
+
+#[cfg(feature = "zlob")]
+pub(crate) type GlobPattern = zlob::ZlobPattern;
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) type GlobPattern = globset::GlobMatcher;
+
+/// How `Constraint::Glob` is evaluated for each item.
+enum GlobStrategy {
+    /// No Glob constraint present.
+    None,
+    /// Pure-glob workload (no Extension filter to reject items first).
+    /// Batch all paths through zlob/globset once; per-item check is a Vec<bool> lookup.
+    Prepass(Vec<Vec<bool>>),
+    /// Mixed workload (Extension filter present). Compile patterns up front, then
+    /// only run them on items that survive the cheap Extension OR check.
+    /// `None` slot = compile failure -> never matches; preserves index alignment.
+    Inline(Vec<Option<GlobPattern>>),
+}
+
+/// Bundles preprocessed constraints for the per-item evaluator.
+pub(crate) struct ConstraintPlan<'q, 'c> {
+    /// OR semantics — file passes if ANY extension matches. Empty = no ext filter.
+    extensions: SmallVec<[&'q str; 8]>,
+    /// AND semantics — file passes only if ALL match.
+    rest: SmallVec<[&'c Constraint<'q>; 8]>,
+    glob: GlobStrategy,
+}
+
+pub(crate) struct ConstraintsBuffers {
+    fname: String,
+    path: String,
+}
+
+impl ConstraintsBuffers {
+    pub(crate) fn new() -> Self {
+        Self {
+            fname: String::with_capacity(64),
+            path: String::with_capacity(64),
+        }
+    }
+}
+
+impl<'q, 'c> ConstraintPlan<'q, 'c> {
+    pub(crate) fn build<T: Constrainable>(
+        constraints: &'c [Constraint<'q>],
+        items: &[T],
+        base_arena: ArenaPtr,
+        overflow_arena: ArenaPtr,
+    ) -> Self {
+        let mut extensions = SmallVec::new();
+        let mut rest: SmallVec<[&'c Constraint<'q>; 8]> = SmallVec::new();
+        for c in constraints {
+            match c {
+                Constraint::Extension(ext) => extensions.push(*ext),
+                _ => rest.push(c),
+            }
+        }
+        let has_pre_filter = !extensions.is_empty() || rest.iter().any(|&c| !is_glob_node(c));
+        let glob = build_glob_strategy(&rest, has_pre_filter, items, base_arena, overflow_arena);
+
+        Self {
+            extensions,
+            rest,
+            glob,
+        }
+    }
+
+    fn run<'a, T: Constrainable + Sync>(
+        &self,
+        items: &'a [T],
+        base_arean: ArenaPtr,
+        overflow_arena: ArenaPtr,
+    ) -> Vec<&'a T> {
+        if items.len() >= PAR_THRESHOLD {
+            use rayon::prelude::*;
+            items
+                .par_iter()
+                .enumerate()
+                .map_init(ConstraintsBuffers::new, |scratch, (i, item)| {
+                    self.matches(item, i, base_arean, overflow_arena, scratch)
+                        .then_some(item)
+                })
+                .flatten()
+                .collect()
+        } else {
+            let mut scratch = ConstraintsBuffers::new();
+            items
+                .iter()
+                .enumerate()
+                .filter_map(|(i, item)| {
+                    self.matches(item, i, base_arean, overflow_arena, &mut scratch)
+                        .then_some(item)
+                })
+                .collect()
+        }
+    }
+
+    #[inline]
+    pub(crate) fn matches<T: Constrainable>(
+        &self,
+        item: &T,
+        index: usize,
+        base_arena: ArenaPtr,
+        overflow_arena: ArenaPtr,
+        scratch: &mut ConstraintsBuffers,
+    ) -> bool {
+        let arena = if item.is_overflow() {
+            overflow_arena
+        } else {
+            base_arena
+        };
+
+        if !self.passes_extensions(item, arena, scratch) {
+            return false;
+        }
+
+        let mut glob_idx = 0;
+        self.rest.iter().all(|c| {
+            evaluate(
+                item,
+                index,
+                c,
+                &self.glob,
+                &mut glob_idx,
+                false,
+                arena,
+                scratch,
+            )
+        })
+    }
+
+    #[inline]
+    fn passes_extensions<T: Constrainable>(
+        &self,
+        item: &T,
+        arena: ArenaPtr,
+        scratch: &mut ConstraintsBuffers,
+    ) -> bool {
+        if self.extensions.is_empty() {
+            return true;
+        }
+        item.write_file_name(arena, &mut scratch.fname);
+        self.extensions
+            .iter()
+            .any(|ext| file_has_extension(&scratch.fname, ext))
+    }
+}
+
+#[inline]
+#[allow(clippy::too_many_arguments)]
+fn evaluate<T: Constrainable>(
+    item: &T,
+    index: usize,
+    constraint: &Constraint<'_>,
+    glob: &GlobStrategy,
+    glob_idx: &mut usize,
+    negate: bool,
+    arena: ArenaPtr,
+    scratch: &mut ConstraintsBuffers,
+) -> bool {
+    let raw = match constraint {
+        Constraint::Glob(_) => {
+            let m = match glob {
+                GlobStrategy::None => true,
+                GlobStrategy::Prepass(masks) => masks
+                    .get(*glob_idx)
+                    .and_then(|mask| mask.get(index).copied())
+                    .unwrap_or(false),
+                GlobStrategy::Inline(patterns) => {
+                    item.write_relative_path(arena, &mut scratch.path);
+                    patterns
+                        .get(*glob_idx)
+                        .and_then(|p| p.as_ref())
+                        .map(|p| compiled_matches(p, &scratch.path))
+                        .unwrap_or(false)
+                }
+            };
+            *glob_idx += 1;
+            m
+        }
+        // Reachable only via `Not(Extension(_))` — bare extensions are split out
+        // up front and handled in `passes_extensions`.
+        Constraint::Extension(ext) => {
+            item.write_file_name(arena, &mut scratch.fname);
+            file_has_extension(&scratch.fname, ext)
+        }
+        Constraint::PathSegment(segment) => {
+            item.write_relative_path(arena, &mut scratch.path);
+            path_contains_segment(&scratch.path, segment)
+        }
+        Constraint::FilePath(suffix) => {
+            item.write_relative_path(arena, &mut scratch.path);
+            path_ends_with_suffix(&scratch.path, suffix)
+        }
+        Constraint::Text(text) => {
+            // Only meaningful under negation (used as exclude filter).
+            item.write_relative_path(arena, &mut scratch.path);
+            find_case_insensitive_short(scratch.path.as_bytes(), text.as_bytes()).is_some()
+        }
+        Constraint::GitStatus(filter) => matches_git_status(item.git_status(), filter),
+        Constraint::Not(inner) => {
+            return evaluate(item, index, inner, glob, glob_idx, !negate, arena, scratch);
+        }
+        // Pass-throughs — handled at higher levels.
+        Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true,
+    };
+    if negate { !raw } else { raw }
+}
+
+#[inline]
+fn matches_git_status(status: Option<git2::Status>, filter: &GitStatusFilter) -> bool {
+    match (status, filter) {
+        (Some(s), GitStatusFilter::Modified) => is_modified_status(s),
+        (Some(s), GitStatusFilter::Untracked) => s.contains(git2::Status::WT_NEW),
+        (Some(s), GitStatusFilter::Staged) => s.intersects(
+            git2::Status::INDEX_NEW
+                | git2::Status::INDEX_MODIFIED
+                | git2::Status::INDEX_DELETED
+                | git2::Status::INDEX_RENAMED
+                | git2::Status::INDEX_TYPECHANGE,
+        ),
+        (Some(s), GitStatusFilter::Unmodified) => s.is_empty(),
+        (None, GitStatusFilter::Unmodified) => true,
+        (None, _) => false,
+    }
+}
+
+#[inline]
+#[cfg(feature = "zlob")]
+pub(crate) fn compiled_matches(p: &GlobPattern, path: &str) -> bool {
+    p.matches_default(path)
+}
+
+#[inline]
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn compiled_matches(p: &GlobPattern, path: &str) -> bool {
+    p.is_match(path)
+}
+
+/// Append indices (into `rels`) of paths matching `p`, in input order.
+/// zlob backend: ONE FFI call for the whole batch.
+#[cfg(feature = "zlob")]
+pub(crate) fn glob_matches_into(p: &GlobPattern, rels: &[&str], out: &mut Vec<usize>) {
+    match p.match_indices(rels, p.flags()) {
+        Ok(ix) => out.extend_from_slice(ix.as_slice()),
+        Err(e) => {
+            tracing::warn!(?e, "zlob batch match failed, falling back to per-path");
+            out.extend((0..rels.len()).filter(|&i| p.matches_default(rels[i])));
+        }
+    }
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn glob_matches_into(p: &GlobPattern, rels: &[&str], out: &mut Vec<usize>) {
+    out.extend((0..rels.len()).filter(|&i| p.is_match(rels[i])));
+}
+
+/// Decide between batch prepass and inline compiled patterns.
+///
+/// `has_pre_filter` = true when something cheaper than glob can reject items first
+/// (extensions OR non-glob constraints in `rest`). In that case inline pays glob
+/// cost only on survivors and beats prepass on every workload we benched. Pure-glob
+/// (no pre-filter) takes prepass — single batched zlob call beats N inline matches.
+fn build_glob_strategy<T: Constrainable>(
+    rest: &[&Constraint<'_>],
+    has_pre_filter: bool,
+    items: &[T],
+    arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> GlobStrategy {
+    if !contains_glob(rest) {
+        return GlobStrategy::None;
+    }
+    if has_pre_filter {
+        return GlobStrategy::Inline(compile_globs(rest));
+    }
+    let buf = PathBuffer::collect(items, arena, overflow_arena);
+    let path_refs = buf.as_strs();
+    GlobStrategy::Prepass(precompute_masks(rest, &path_refs))
+}
+
+/// `Glob` or `Not(Glob)` — the constraint kinds whose evaluation goes through
+/// the GlobStrategy. Everything else can pre-reject items before glob runs.
+fn is_glob_node(c: &Constraint<'_>) -> bool {
+    match c {
+        Constraint::Glob(_) => true,
+        Constraint::Not(inner) => is_glob_node(inner),
+        _ => false,
+    }
+}
+
+fn contains_glob(rest: &[&Constraint<'_>]) -> bool {
+    rest.iter().any(|c| is_glob_node(c))
+}
+
+/// Contiguous byte buffer holding every item's `relative_path`. Single allocation
+/// instead of N `String`s. On Windows the in-place pass folds `\\` -> `/` so the
+/// glob library sees a canonical separator.
+struct PathBuffer {
+    bytes: Vec<u8>,
+    offsets: Vec<(usize, usize)>,
+}
+
+impl PathBuffer {
+    fn collect<T: Constrainable>(items: &[T], arena: ArenaPtr, overflow_arena: ArenaPtr) -> Self {
+        let mut bytes = Vec::<u8>::new();
+        let mut offsets = Vec::with_capacity(items.len());
+        let mut tmp = String::with_capacity(64);
+        for item in items {
+            let item_arena = if item.is_overflow() {
+                overflow_arena
+            } else {
+                arena
+            };
+            let start = bytes.len();
+            item.write_relative_path(item_arena, &mut tmp);
+            bytes.extend_from_slice(tmp.as_bytes());
+            offsets.push((start, bytes.len() - start));
+        }
+        Self { bytes, offsets }
+    }
+
+    fn as_strs(&self) -> Vec<&str> {
+        self.offsets
+            .iter()
+            .map(|&(off, len)| unsafe {
+                std::str::from_utf8_unchecked(&self.bytes[off..off + len])
+            })
+            .collect()
+    }
+}
+
+fn precompute_masks(rest: &[&Constraint<'_>], paths: &[&str]) -> Vec<Vec<bool>> {
+    let mut out = Vec::new();
+    for c in rest {
+        walk_globs(c, &mut |pattern| {
+            out.push(match_glob_pattern(pattern, paths))
+        });
+    }
+    out
+}
+
+fn compile_globs(rest: &[&Constraint<'_>]) -> Vec<Option<GlobPattern>> {
+    let mut out = Vec::new();
+    for c in rest {
+        walk_globs(c, &mut |pattern| out.push(compile_one(pattern)));
+    }
+    out
+}
+
+/// Visit every Glob (including ones nested under Not) in constraint walk order.
+/// Order matters: `glob_idx` in the per-item evaluator increments by one per Glob node.
+fn walk_globs<F: FnMut(&str)>(c: &Constraint<'_>, f: &mut F) {
+    match c {
+        Constraint::Glob(p) => f(p),
+        Constraint::Not(inner) => walk_globs(inner, f),
+        _ => {}
+    }
+}
+
+#[cfg(feature = "zlob")]
+pub(crate) fn compile_one(pattern: &str) -> Option<GlobPattern> {
+    zlob::ZlobPattern::compile(pattern, zlob::ZlobFlags::RECOMMENDED).ok()
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn compile_one(pattern: &str) -> Option<GlobPattern> {
+    globset::Glob::new(pattern)
+        .ok()
+        .map(|g| g.compile_matcher())
+}
+
+/// Build a `paths.len()`-sized bitmap. Vec<bool> beats AHashSet ~2× in the per-item
+/// filter loop — no hashing, plain array indexing, sequential prefetcher-friendly.
+#[cfg(feature = "zlob")]
+fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec<bool> {
+    let mut mask = vec![false; paths.len()];
+    let Ok(hits) = zlob::zlob_match_paths_indices(pattern, paths, zlob::ZlobFlags::RECOMMENDED)
+    else {
+        return mask;
+    };
+    for i in hits.to_iter() {
+        if i < mask.len() {
+            mask[i] = true;
+        }
+    }
+    mask
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec<bool> {
+    let mut mask = vec![false; paths.len()];
+    let Ok(glob) = globset::Glob::new(pattern) else {
+        return mask;
+    };
+    let matcher = glob.compile_matcher();
+    if paths.len() >= PAR_THRESHOLD {
+        use rayon::prelude::*;
+        mask.par_iter_mut()
+            .zip(paths.par_iter())
+            .for_each(|(slot, p)| *slot = matcher.is_match(p));
+    } else {
+        for (slot, p) in mask.iter_mut().zip(paths.iter()) {
+            *slot = matcher.is_match(p);
+        }
+    }
+    mask
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Clone)]
+    struct TestItem {
+        relative_path: &'static str,
+        file_name: &'static str,
+    }
+
+    impl Constrainable for TestItem {
+        fn write_file_name(&self, _arena: ArenaPtr, out: &mut String) {
+            out.clear();
+            out.push_str(self.file_name);
+        }
+
+        fn write_relative_path(&self, _arena: ArenaPtr, out: &mut String) {
+            out.clear();
+            out.push_str(self.relative_path);
+        }
+
+        fn git_status(&self) -> Option<git2::Status> {
+            None
+        }
+
+        fn is_overflow(&self) -> bool {
+            false
+        }
+    }
+
+    #[test]
+    fn test_file_has_extension() {
+        assert!(file_has_extension("file.rs", "rs"));
+        assert!(file_has_extension("file.RS", "rs")); // case-insensitive
+        assert!(file_has_extension("file.test.rs", "rs"));
+        assert!(file_has_extension("a.rs", "rs"));
+
+        assert!(!file_has_extension("file.tsx", "rs"));
+        assert!(!file_has_extension("rs", "rs")); // too short
+        assert!(!file_has_extension(".rs", "rs")); // just extension
+        assert!(!file_has_extension("file.rsx", "rs")); // different extension
+        assert!(!file_has_extension("filers", "rs")); // no dot
+    }
+
+    #[test]
+    fn test_path_contains_segment() {
+        // Segment at start
+        assert!(path_contains_segment("src/lib.rs", "src"));
+        assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive
+
+        // Segment in middle
+        assert!(path_contains_segment("app/src/lib.rs", "src"));
+        assert!(path_contains_segment("app/SRC/lib.rs", "src"));
+
+        // Multiple levels
+        assert!(path_contains_segment("core/workflow/src/main.rs", "src"));
+        assert!(path_contains_segment(
+            "core/workflow/src/main.rs",
+            "workflow"
+        ));
+        assert!(path_contains_segment("core/workflow/src/main.rs", "core"));
+
+        // Should not match partial segments
+        assert!(!path_contains_segment("source/lib.rs", "src"));
+        assert!(!path_contains_segment("mysrc/lib.rs", "src"));
+
+        // Should not match filename
+        assert!(!path_contains_segment("lib/src", "src"));
+
+        // Multi-segment constraints
+        assert!(path_contains_segment(
+            "libswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "foo/libswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "foo/LibSwscale/AArch64/input.S",
+            "libswscale/aarch64"
+        )); // case-insensitive
+        assert!(!path_contains_segment(
+            "xlibswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        )); // partial match at start
+        assert!(!path_contains_segment(
+            "foo/libswscale/aarch64x/input.S",
+            "libswscale/aarch64"
+        )); // partial match at end
+        assert!(path_contains_segment(
+            "crates/fff-core/src/grep.rs",
+            "fff-core/src"
+        ));
+
+        // Edge cases
+        assert!(!path_contains_segment("", "src"));
+        assert!(!path_contains_segment("src", "src")); // no trailing slash
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn test_path_contains_segment_accepts_backslash() {
+        assert!(path_contains_segment("src\\lib.rs", "src"));
+        assert!(path_contains_segment(
+            "app\\modules\\src\\services\\x.lua",
+            "src"
+        ));
+        assert!(path_contains_segment("app\\SRC\\x.lua", "src"));
+
+        assert!(path_contains_segment(
+            "foo\\libswscale\\aarch64\\input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "crates\\fff-core\\src\\grep.rs",
+            "fff-core/src"
+        ));
+
+        assert!(!path_contains_segment("mysrc\\lib.rs", "src"));
+        assert!(!path_contains_segment(
+            "xlibswscale\\aarch64\\in.S",
+            "libswscale/aarch64"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix() {
+        // Exact match
+        assert!(path_ends_with_suffix(
+            "libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Suffix match at / boundary
+        assert!(path_ends_with_suffix(
+            "foo/libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Deep nesting
+        assert!(path_ends_with_suffix(
+            "a/b/c/libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // No boundary — partial directory name
+        assert!(!path_ends_with_suffix(
+            "xlibswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Case insensitive
+        assert!(path_ends_with_suffix(
+            "foo/LibSwscale/Input.C",
+            "libswscale/input.c"
+        ));
+
+        // Single file name
+        assert!(path_ends_with_suffix("input.c", "input.c"));
+        assert!(!path_ends_with_suffix("xinput.c", "input.c"));
+
+        // Suffix longer than path
+        assert!(!path_ends_with_suffix("input.c", "foo/input.c"));
+
+        // Simple path
+        assert!(path_ends_with_suffix("src/main.rs", "src/main.rs"));
+        assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs"));
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn test_path_ends_with_suffix_accepts_backslash() {
+        assert!(path_ends_with_suffix(
+            "app\\modules\\src\\services\\handler.lua",
+            "services/handler.lua"
+        ));
+        assert!(path_ends_with_suffix(
+            "foo\\libswscale\\input.c",
+            "libswscale/input.c"
+        ));
+        assert!(!path_ends_with_suffix(
+            "xlibswscale\\input.c",
+            "libswscale/input.c"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_does_not_panic_on_unicode_suffix() {
+        assert!(!path_ends_with_suffix("유니코드_파일_테스트.csv", "트.c"));
+        assert!(path_ends_with_suffix(
+            "data/유니코드_파일_테스트.csv",
+            "유니코드_파일_테스트.csv"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_unicode_apostrophe_mismatch() {
+        assert!(!path_ends_with_suffix(
+            "dir/\u{2019}bar/file.txt",
+            "'bar/file.txt"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_unicode_space_mismatch() {
+        assert!(!path_ends_with_suffix(
+            "dir/\u{202f}am/file.txt",
+            " am/file.txt"
+        ));
+    }
+
+    #[test]
+    fn test_path_contains_segment_does_not_panic_on_unicode_segment() {
+        assert!(!path_contains_segment("문서/notes.txt", "문x"));
+        assert!(path_contains_segment("프로젝트/문서/notes.txt", "문서"));
+    }
+
+    #[test]
+    fn test_path_contains_segment_unicode_no_panic() {
+        assert!(!path_contains_segment(
+            "Library/Cloud/Project\u{2019}s Folder/books.ttl",
+            "Project's Folder"
+        ));
+    }
+
+    #[test]
+    fn test_file_has_extension_unicode_no_panic() {
+        assert!(!file_has_extension("cat\u{00e9}.rs", "s"));
+    }
+
+    #[test]
+    fn test_file_has_extension_unicode_filename() {
+        assert!(file_has_extension("운영-가이드.md", "md"));
+        assert!(file_has_extension("테스트.csv", "csv"));
+        assert!(!file_has_extension("테스트.csv", "md"));
+    }
+
+    #[test]
+    fn test_apply_constraints_file_path_with_unicode_suffix() {
+        let arena_ptr = ArenaPtr(std::ptr::null());
+
+        let item = TestItem {
+            relative_path: "data/유니코드_파일_테스트.csv",
+            file_name: "유니코드_파일_테스트.csv",
+        };
+
+        let exact = [Constraint::FilePath("유니코드_파일_테스트.csv")];
+        let mismatch = [Constraint::FilePath("트.c")];
+
+        let exact_items = [item.clone()];
+        let exact_matches = apply_constraints(&exact_items, &exact, arena_ptr, arena_ptr)
+            .expect("constraints applied");
+        assert_eq!(exact_matches.len(), 1);
+
+        let mismatch_items = [item];
+        let mismatch_matches = apply_constraints(&mismatch_items, &mismatch, arena_ptr, arena_ptr)
+            .expect("constraints applied");
+        assert!(mismatch_matches.is_empty());
+    }
+
+    #[test]
+    fn test_unicode_path_no_panic_real_korean_cases() {
+        // Real Korean paths that caused panics
+        let path1 = "Downloads/(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv";
+        let path2 = "hermes-agent-lecture-materials/세부_커리큘럼_최종.csv";
+        let path3 = "projects/fastcampus-hermes-agent-curriculum/chapters/part-02-Hermes-설치-및-기본-사용/section-02-doctor로-설치-상태-검증/research/03-fix가-자동-수정하는-것과-못하는-것.md";
+
+        // These must not panic regardless of segment/suffix used
+        assert!(!path_contains_segment(path1, "작성"));
+        assert!(!path_ends_with_suffix(path1, "작성.csv"));
+        assert!(!path_contains_segment(path2, "최종"));
+        assert!(!path_ends_with_suffix(path2, "최종.csv"));
+        assert!(!path_contains_segment(path3, "수정"));
+        assert!(!path_ends_with_suffix(path3, "것.md"));
+
+        // Positive cases should still work
+        assert!(path_contains_segment(
+            path2,
+            "hermes-agent-lecture-materials"
+        ));
+        assert!(path_ends_with_suffix(
+            path1,
+            "(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv"
+        ));
+        assert!(path_ends_with_suffix(path2, "세부_커리큘럼_최종.csv"));
+    }
+
+    #[test]
+    fn test_negated_glob_excludes_matching_files() {
+        let arena_ptr = ArenaPtr(std::ptr::null());
+
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "src/lib.ts",
+                file_name: "lib.ts",
+            },
+            TestItem {
+                relative_path: "include/fff.h",
+                file_name: "fff.h",
+            },
+        ];
+
+        // Not(Glob("**/*.rs")) should exclude .rs files
+        let constraints = vec![Constraint::Not(Box::new(Constraint::Glob("**/*.rs")))];
+        let result = apply_constraints(&items, &constraints, arena_ptr, arena_ptr).unwrap();
+        let paths: Vec<&str> = result.iter().map(|i| i.relative_path).collect();
+        assert!(
+            !paths.contains(&"src/main.rs"),
+            "rs file should be excluded"
+        );
+        assert!(paths.contains(&"src/lib.ts"), "ts file should be included");
+        assert!(
+            paths.contains(&"include/fff.h"),
+            "h file should be included"
+        );
+    }
+
+    #[test]
+    fn test_inline_glob_path_matches_prepass() {
+        // Mixed (extensions + glob) takes the inline-compiled path.
+        // Pure glob takes the prepass bitmap path. Both must give identical results.
+        let arena_ptr = ArenaPtr(std::ptr::null());
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "src/lib.ts",
+                file_name: "lib.ts",
+            },
+            TestItem {
+                relative_path: "tests/foo.rs",
+                file_name: "foo.rs",
+            },
+            TestItem {
+                relative_path: "docs/readme.md",
+                file_name: "readme.md",
+            },
+        ];
+
+        let mixed = vec![Constraint::Extension("rs"), Constraint::Glob("src/**")];
+        let mixed_paths: Vec<&str> = apply_constraints(&items, &mixed, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert_eq!(mixed_paths, vec!["src/main.rs"]);
+
+        let pure_glob = vec![Constraint::Glob("src/**")];
+        let glob_paths: Vec<&str> = apply_constraints(&items, &pure_glob, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert!(glob_paths.contains(&"src/main.rs"));
+        assert!(glob_paths.contains(&"src/lib.ts"));
+        assert_eq!(glob_paths.len(), 2);
+    }
+
+    #[test]
+    fn test_inline_negated_glob_with_extension() {
+        // Mixed Not(Glob) on inline path — exercise the negate=true branch in
+        // glob_matches_inline through the Not->Glob recursion.
+        let arena_ptr = ArenaPtr(std::ptr::null());
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "vendor/foo.rs",
+                file_name: "foo.rs",
+            },
+            TestItem {
+                relative_path: "vendor/foo.ts",
+                file_name: "foo.ts",
+            },
+        ];
+
+        let constraints = vec![
+            Constraint::Extension("rs"),
+            Constraint::Not(Box::new(Constraint::Glob("vendor/**"))),
+        ];
+        let paths: Vec<&str> = apply_constraints(&items, &constraints, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert_eq!(paths, vec!["src/main.rs"]);
+    }
+}

crates/fff-core/src/index/mod.rs

@@ -0,0 +1,11 @@
+#[doc(hidden)] // for bench
+pub mod bigram_filter;
+pub(crate) use bigram_filter::*;
+
+mod bigram_query;
+pub use bigram_query::*;
+
+mod candidates;
+pub(crate) use candidates::*;
+
+pub mod constraints;

crates/fff-core/src/lib.rs

@@ -1,10 +1,13 @@
 //! # FFF Search — High-performance file finder core
 //!
-//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff.nvim).
+//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff).
 //! It includes filesystem indexing with real-time watching, fuzzy matching powered
 //! by [frizbee](https://docs.rs/neo_frizbee), frecency scoring backed by LMDB,
 //! and multi-mode grep search.
 //!
+//! > [!Important performance information]  
+//! > For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine.
+//!
 //! ## Architecture
 //!
 //! - [`file_picker::FilePicker`] — Main entry point. Indexes a directory tree in a
@@ -17,6 +20,9 @@
 //! - [`grep`] — Live grep search supporting regex, plain-text, and fuzzy modes
 //!   with optional constraint filtering.
 //! - [`git`] — Git status caching and repository detection.
+//! - [`watch`] — Client-facing filesystem watch subscriptions: glob, exact
+//!   path, or directory subtree with normalized batch delivery
+//!   (see [`SharedFilePicker::watch`]).
 //!
 //! ## Shared State
 //!
@@ -91,64 +97,78 @@
 //! # Ok::<(), Box<dyn std::error::Error>>(())
 //! ```
 
-mod background_watcher;
-mod scan;
-// public only for benchmarks — the inverted index is still re-exported via
-// `pub use bigram_filter::*` below for external consumers.
-#[doc(hidden)]
-pub mod bigram_filter;
-pub mod bigram_query;
-mod constraints;
-mod error;
-mod score;
-mod sort_buffer;
-// this is pub only for benchmarks
-pub mod case_insensitive_memmem;
+#[cfg(not(any(feature = "ripgrep", feature = "zlob")))]
+compile_error!(
+    "fff-search requires either the `ripgrep` (default) or `zlob` feature. \
+     Enable one, e.g. `--features ripgrep` or `--features zlob`."
+);
 
-pub(crate) mod simd_path;
+/// Primary entry points with thread-safe [`SharedFilePicker`](shared::FilePicker) instance
+pub mod shared;
+pub use shared::*;
 
-/// Core file picker: filesystem indexing, background watching, and fuzzy search.
-///
+/// Core file picker single thread: filesystem indexing, background watching, and fuzzy search.
 /// See [`FilePicker`](file_picker::FilePicker) for the main entry point.
 pub mod file_picker;
+pub use file_picker::*;
 
 /// Database-backed persistence: frecency, query history, LMDB plumbing.
 pub mod dbs;
-pub use dbs::frecency;
+pub use dbs::*;
 
 /// Git status caching and repository detection utilities.
 pub mod git;
 
 /// Live grep search with regex, plain-text, and fuzzy matching modes.
-///
-/// Supports constraint filtering (file extensions, path segments, globs)
-/// and parallel execution via rayon.
 pub mod grep;
+pub use grep::*;
 
-/// Tracing/logging initialization and panic hook setup.
+/// Tracing/logging initialization
 pub mod log;
 
-/// Path manipulation utilities: cross platform canonicalization, tilde expansion, and
-/// directory distance penalties for search scoring.
+/// Various path utils might be handy for you to work with fff paths
 pub mod path_utils;
 
-pub use dbs::query_tracker;
-
 /// Core data types shared across the crate.
 pub mod types;
-
-mod ignore;
-/// Thread-safe shared handles for [`FilePicker`], [`FrecencyTracker`],
-/// and [`QueryTracker`].
-pub mod shared;
-
-pub use bigram_filter::*;
-pub use dbs::db_healthcheck::{DbHealth, DbHealthChecker};
-pub use error::{Error, Result};
-pub use fff_query_parser::*;
-pub use file_picker::*;
-pub use frecency::*;
-pub use grep::*;
-pub use query_tracker::*;
-pub use shared::*;
 pub use types::*;
+
+pub mod constants;
+
+/// Watcher rescan request accounting.
+pub mod rescan_stats;
+pub use rescan_stats::{RESCAN_STATS_ENABLED, RescanReason, RescanStats};
+
+mod rescan_throttle;
+
+// ==================================
+// these are public only for benchmarks, no backward compatibility guaranteed
+#[doc(hidden)]
+pub use index::bigram_filter;
+#[doc(hidden)]
+pub mod simd_string_utils;
+// ==================================
+
+mod error;
+mod git_status_worker;
+mod ignore;
+mod scan;
+mod score;
+mod sort_buffer;
+
+pub(crate) mod index;
+pub(crate) mod parallelism;
+pub(crate) mod simd_path;
+pub(crate) mod stable_vec;
+pub(crate) mod walk;
+
+/// Filesystem watch subscriptions with glob filtering and batched delivery,
+/// plus the background OS watcher.
+#[path = "watcher/mod.rs"]
+pub mod watch;
+pub use watch::{WatchEvent, WatchEventKind, WatchId, WatchOptions};
+
+// fff error
+pub use error::{Error, Result};
+
+pub use fff_query_parser::*;

crates/fff-core/src/log.rs

@@ -1,31 +1,20 @@
-//! Shared logging utilities for FFF crates.
-//!
-//! Provides file-based tracing initialization and crash handlers (panic hook
-//! + SIGSEGV signal handler) that write diagnostics to both stderr and the
-//! configured log file.
-
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
 use tracing_appender::non_blocking;
 use tracing_subscriber::fmt::format::FmtSpan;
 use tracing_subscriber::{EnvFilter, fmt, prelude::*};
 
-static TRACING_INITIALIZED: std::sync::OnceLock<tracing_appender::non_blocking::WorkerGuard> =
-    std::sync::OnceLock::new();
-
-static CRASH_HANDLERS_INSTALLED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
-
-/// The log file path set by `init_tracing`. Crash handlers append to this file.
-static LOG_FILE_PATH: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
+// Set once on first init_tracing; doubles as the init-once gate.
+static LOG_FILE_PATH: OnceLock<PathBuf> = OnceLock::new();
+static CRASH_HOOKS: OnceLock<()> = OnceLock::new();
 
 fn write_crash_report(header: &str, body: &str) {
     let msg = format!(
-        "\n=== CRASH {} ===\n{}\n=== CRASH END {} ===\n",
+        "\n=== CRASH (this might NOT BE fff related) {} ===\n{}\n=== CRASH END {} ===\n",
         header, body, header
     );
-
     let _ = std::io::Write::write_all(&mut std::io::stderr(), msg.as_bytes());
-
     if let Some(path) = LOG_FILE_PATH.get() {
         let _ = std::fs::OpenOptions::new()
             .create(true)
@@ -35,54 +24,99 @@ fn write_crash_report(header: &str, body: &str) {
     }
 }
 
-extern "C" fn sigsegv_handler(sig: libc::c_int) {
-    let bt = std::backtrace::Backtrace::force_capture();
-    write_crash_report("SIGSEGV", &format!("signal {}\n{}", sig, bt));
+// SIGSEGV handler writes a banner to a pre-opened fd (open(2) inside a signal
+// handler is unsafe due to path-resolution allocs). Unix only.
+#[cfg(unix)]
+mod sigsegv {
+    use std::os::fd::IntoRawFd;
+    use std::path::Path;
+    use std::sync::atomic::{AtomicI32, Ordering};
 
-    unsafe {
-        libc::signal(sig, libc::SIG_DFL);
-        libc::raise(sig);
+    static LOG_FD: AtomicI32 = AtomicI32::new(-1);
+
+    // Must `create(true)` — this runs before init_tracing opens/creates the
+    // writer file, so an append-only open on a non-existent path silently
+    // fails, LOG_FD stays -1, and the SIGSEGV banner never reaches the log.
+    pub fn set_log_fd(path: &Path) {
+        if let Ok(file) = std::fs::OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(path)
+        {
+            let prev = LOG_FD.swap(file.into_raw_fd(), Ordering::Relaxed);
+            if prev >= 0 {
+                unsafe { libc::close(prev) };
+            }
+        }
+    }
+
+    // Body must be async-signal-safe: write(2), atomic load, signal(2). Nothing else.
+    fn handler(_info: &libc::siginfo_t) {
+        const BANNER: &[u8] = b"\n=== CRASH SIGSEGV (fff) ===\n\
+            fff.nvim's rust extension hit a segfault and is about to die.\n\
+            Please file the bug at https://github.com/dmtrKovalenko/fff/issues with this banner attached.\n\
+            === CRASH END SIGSEGV ===\n";
+        unsafe {
+            libc::write(2, BANNER.as_ptr().cast(), BANNER.len());
+            let log_fd = LOG_FD.load(Ordering::Relaxed);
+            if log_fd >= 0 {
+                libc::write(log_fd, BANNER.as_ptr().cast(), BANNER.len());
+            }
+            // Reset to default so handler return → kernel kills us instead of
+            // re-running the faulting instruction in an infinite loop.
+            libc::signal(libc::SIGSEGV, libc::SIG_DFL);
+        }
+    }
+
+    pub fn install() {
+        // signal-hook-registry chains to LuaJIT's prior handler automatically.
+        unsafe {
+            let _ = signal_hook_registry::register_unchecked(libc::SIGSEGV, handler);
+        }
     }
 }
 
-/// Install both the panic hook and the SIGSEGV signal handler.
+#[cfg(not(unix))]
+mod sigsegv {
+    use std::path::Path;
+    pub fn set_log_fd(_path: &Path) {}
+    pub fn install() {}
+}
+
 pub fn install_panic_hook() {
-    CRASH_HANDLERS_INSTALLED.get_or_init(|| {
-        let default_panic = std::panic::take_hook();
-        std::panic::set_hook(Box::new(move |panic_info| {
-            let message = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
-                s.to_string()
-            } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
-                s.clone()
-            } else {
-                "Unknown panic payload".to_string()
-            };
+    CRASH_HOOKS.get_or_init(install_crash_hooks);
+}
 
-            let location = panic_info
-                .location()
-                .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
-                .unwrap_or_else(|| "unknown location".to_string());
+fn install_crash_hooks() {
+    let default_panic = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |panic_info| {
+        let message = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            s.to_string()
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            s.clone()
+        } else {
+            "Unknown panic payload".to_string()
+        };
 
-            tracing::error!(
-                panic.message = %message,
-                panic.location = %location,
-                "PANIC occurred in FFF"
-            );
+        let location = panic_info
+            .location()
+            .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
+            .unwrap_or_else(|| "unknown location".to_string());
 
-            write_crash_report(
-                "RUST PANIC",
-                &format!("Message: {}\nLocation: {}", message, location),
-            );
-            default_panic(panic_info);
-        }));
+        tracing::error!(
+            panic.message = %message,
+            panic.location = %location,
+            "PANIC occurred in FFF"
+        );
 
-        unsafe {
-            libc::signal(
-                libc::SIGSEGV,
-                sigsegv_handler as *const () as libc::sighandler_t,
-            );
-        }
-    });
+        write_crash_report(
+            "RUST PANIC",
+            &format!("Message: {}\nLocation: {}", message, location),
+        );
+        default_panic(panic_info);
+    }));
+
+    sigsegv::install();
 }
 
 /// Parse a log level string into a `tracing::Level`.
@@ -97,54 +131,151 @@ pub fn parse_log_level(level: Option<&str>) -> tracing::Level {
     }
 }
 
-/// Initialize tracing with a single log file.
-pub fn init_tracing(log_file_path: &str, log_level: Option<&str>) -> Result<String, io::Error> {
-    let log_path = Path::new(log_file_path);
-    if let Some(parent) = log_path.parent() {
-        std::fs::create_dir_all(parent)?;
+/// Default retention: how many prior nvim sessions' log files to keep.
+const DEFAULT_RETAIN_RUNS: usize = 20;
+
+pub fn generate_trace_id() -> String {
+    use std::sync::atomic::{AtomicU64, Ordering};
+    static TRACE_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_nanos() as u64)
+        .unwrap_or(0);
+
+    let pid = std::process::id() as u64;
+    let counter = TRACE_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+    // very simple hash functions helps to distinguish trace ids visually
+    let id = nanos ^ (pid.wrapping_mul(0x9E37_79B9_7F4A_7C15)) ^ (counter << 32);
+    format!("{:016x}", id)
+}
+
+pub fn trace_span(trace_id: &str, label: &'static str) -> tracing::Span {
+    tracing::info_span!("fff.trace", trace_id = trace_id, label = label)
+}
+
+fn unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+fn session_path_from_hint(hint: &Path) -> PathBuf {
+    let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff");
+    let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log");
+    let parent = hint.parent().unwrap_or_else(|| Path::new("."));
+    parent.join(format!(
+        "{stem}+{ts}+{pid}.{ext}",
+        ts = unix_secs(),
+        pid = std::process::id(),
+    ))
+}
+
+fn rotate_logs(dir: &Path, stem: &str, ext: &str, retain_runs: usize) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    let prefix = format!("{stem}+");
+    let suffix = format!(".{ext}");
+
+    let mut files: Vec<(std::time::SystemTime, PathBuf)> = entries
+        .filter_map(|res| {
+            let entry = res.ok()?;
+            let name = entry.file_name();
+            let name = name.to_str()?;
+            if !name.starts_with(&prefix) || !name.ends_with(&suffix) {
+                return None;
+            }
+
+            let mtime = entry.metadata().ok()?.modified().ok()?;
+            Some((mtime, entry.path()))
+        })
+        .collect();
+
+    if files.len() <= retain_runs {
+        return;
+    }
+    // Newest first, then drop everything past retain_runs.
+    files.sort_by_key(|(mtime, _)| std::cmp::Reverse(*mtime));
+    for (_, path) in files.into_iter().skip(retain_runs) {
+        let _ = std::fs::remove_file(path);
+    }
+}
+
+/// `log_file_path` is a path-shape hint. Each call writes a unique sibling
+/// `<stem>+<unix-secs>+<pid>.<ext>` so concurrent processes never collide.
+/// Returns the absolute path of the session file.
+pub fn init_tracing(
+    log_file_path: &str,
+    log_level: Option<&str>,
+    retain_runs: Option<usize>,
+) -> Result<String, io::Error> {
+    let hint = Path::new(log_file_path);
+    let session_dir = hint
+        .parent()
+        .unwrap_or_else(|| Path::new("."))
+        .to_path_buf();
+    std::fs::create_dir_all(&session_dir)?;
+
+    let session_path = session_path_from_hint(hint);
+
+    // First init wins; repeat callers no-op and return the original path.
+    if LOG_FILE_PATH.set(session_path.clone()).is_err() {
+        return Ok(LOG_FILE_PATH
+            .get()
+            .map(|p| p.to_string_lossy().into_owned())
+            .unwrap_or_default());
     }
 
-    let _ = LOG_FILE_PATH.set(log_path.to_path_buf());
+    sigsegv::set_log_fd(&session_path);
     install_panic_hook();
 
-    let file_appender = std::fs::OpenOptions::new()
+    let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff");
+    let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log");
+    rotate_logs(
+        &session_dir,
+        stem,
+        ext,
+        retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS),
+    );
+
+    let writer_file = std::fs::OpenOptions::new()
         .create(true)
-        .write(true)
-        .truncate(true) // truncates a file on restart (instead of appending)
-        .open(log_path)?;
+        .append(true)
+        .open(&session_path)?;
 
-    let level = parse_log_level(log_level);
+    // we intinionally leark the guard we don't ever want to stop logging
+    let (non_blocking_appender, guard) = non_blocking(writer_file);
+    Box::leak(Box::new(guard));
 
-    TRACING_INITIALIZED.get_or_init(|| {
-        let (non_blocking_appender, guard) = non_blocking(file_appender);
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            fmt::layer()
+                .with_writer(non_blocking_appender)
+                .with_target(true)
+                .with_thread_ids(false)
+                .with_thread_names(true)
+                .with_ansi(false)
+                .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
+        )
+        .with(
+            EnvFilter::builder()
+                .with_default_directive(parse_log_level(log_level).into())
+                .from_env_lossy(),
+        );
 
-        let subscriber = tracing_subscriber::registry()
-            .with(
-                fmt::layer()
-                    .with_writer(non_blocking_appender)
-                    .with_target(true)
-                    .with_thread_ids(false)
-                    .with_thread_names(false)
-                    .with_ansi(false)
-                    .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
-            )
-            .with(
-                EnvFilter::builder()
-                    .with_default_directive(level.into())
-                    .from_env_lossy(),
-            );
+    if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
+        eprintln!("Failed to set tracing subscriber: {}", e);
+    } else {
+        tracing::info!(
+            "FFF tracing initialized: {} (pid={}, retain_runs={})",
+            session_path.display(),
+            std::process::id(),
+            retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS),
+        );
+    }
 
-        if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
-            eprintln!("Failed to set tracing subscriber: {}", e);
-        } else {
-            tracing::info!(
-                "FFF tracing initialized with log file: {}",
-                log_path.display()
-            );
-        }
-
-        guard
-    });
-
-    Ok(log_file_path.to_string())
+    Ok(session_path.to_string_lossy().into_owned())
 }

crates/fff-core/src/parallelism.rs

@@ -0,0 +1,84 @@
+//! Dedicated rayon pools. The global pool spans every logical core, which
+//! oversubscribes asymmetric chips (Apple P+E): E-cores are ~2× slower and
+//! `open()` contends on a per-VFS lock past P-core count, so a larger pool is
+//! slower on file-heavy work.
+
+use std::sync::LazyLock;
+
+/// Dedicated thread pool for background work (scan, warmup, bigram build).
+pub static BACKGROUND_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    let total = std::thread::available_parallelism()
+        .map(|p| p.get())
+        .unwrap_or(4);
+
+    // Background work is mostly syscall-bound; halving parallelism leaves
+    // cores for search/UI at negligible throughput cost.
+    let bg_threads = (total / 2).max(2);
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(bg_threads)
+        .thread_name(|i| format!("fff-bg-{i}"))
+        .start_handler(|_| {
+            // QoS pin keeps workers on P-cores; the kernel otherwise drifts
+            // them to ~2× slower E-cores.
+            #[cfg(target_os = "macos")]
+            unsafe {
+                let _ = libc::pthread_set_qos_class_self_np(
+                    libc::qos_class_t::QOS_CLASS_USER_INITIATED,
+                    0,
+                );
+            }
+        })
+        .build()
+        .expect("failed to create background rayon pool")
+});
+
+/// Physical performance-core count via sysctl, falling back to logical cores.
+/// On a 12P+4E M4 Max, grep runs 16t=6.2s vs 13t=4.9s — fewer threads win.
+#[cfg(target_os = "macos")]
+fn performance_core_count() -> usize {
+    let mut count: libc::c_int = 0;
+    let mut size = std::mem::size_of::<libc::c_int>();
+    let name = c"hw.perflevel0.physicalcpu";
+    let ok = unsafe {
+        libc::sysctlbyname(
+            name.as_ptr(),
+            &mut count as *mut _ as *mut libc::c_void,
+            &mut size,
+            std::ptr::null_mut(),
+            0,
+        )
+    };
+    if ok == 0 && count > 0 {
+        count as usize
+    } else {
+        std::thread::available_parallelism()
+            .map(|p| p.get())
+            .unwrap_or(4)
+    }
+}
+
+/// Pool for grep content search: P-core sized and QoS-pinned on macOS, full
+/// parallelism elsewhere. Avoids E-core drag and VFS-lock contention.
+pub static SEARCH_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    #[cfg(target_os = "macos")]
+    let threads = performance_core_count();
+    #[cfg(not(target_os = "macos"))]
+    let threads = std::thread::available_parallelism()
+        .map(|p| p.get())
+        .unwrap_or(4);
+
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(threads)
+        .thread_name(|i| format!("fff-search-{i}"))
+        .start_handler(|_| {
+            #[cfg(target_os = "macos")]
+            unsafe {
+                let _ = libc::pthread_set_qos_class_self_np(
+                    libc::qos_class_t::QOS_CLASS_USER_INITIATED,
+                    0,
+                );
+            }
+        })
+        .build()
+        .expect("failed to create search rayon pool")
+});

crates/fff-core/src/path_utils.rs

@@ -10,6 +10,42 @@ pub fn canonicalize(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
     std::fs::canonicalize(path)
 }
 
+/// The index stores relative paths with `/` on every platform. These helpers
+/// convert between that canonical form and the OS-native separator, and are
+/// no-ops on non-Windows where `/` is already native.
+
+/// Fold a relative path to the canonical `/` form (no-op off Windows).
+#[cfg(windows)]
+pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> {
+    if rel.contains('\\') {
+        std::borrow::Cow::Owned(rel.replace('\\', "/"))
+    } else {
+        std::borrow::Cow::Borrowed(rel)
+    }
+}
+
+#[cfg(not(windows))]
+#[inline]
+pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> {
+    std::borrow::Cow::Borrowed(rel)
+}
+
+/// Rewrite canonical `/` bytes to the OS-native separator in place (no-op off
+/// Windows). Used at OS/state boundaries (absolute-path reconstruction).
+#[cfg(windows)]
+#[inline]
+pub fn nativize_slashes_in_place(bytes: &mut [u8]) {
+    for b in bytes {
+        if *b == b'/' {
+            *b = b'\\';
+        }
+    }
+}
+
+#[cfg(not(windows))]
+#[inline]
+pub fn nativize_slashes_in_place(_bytes: &mut [u8]) {}
+
 /// Git requires a normalized forward-slashed paths on windows
 #[cfg(windows)]
 pub fn normalize(path: PathBuf) -> PathBuf {

crates/fff-core/src/rescan_stats.rs

@@ -0,0 +1,215 @@
+#[cfg(rescan_stats)]
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// Whether rescan accounting is compiled in.
+pub const RESCAN_STATS_ENABLED: bool = cfg!(rescan_stats);
+
+/// Cause recorded for a filesystem rescan request.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum RescanReason {
+    /// Requested through the public API (refresh, directory change).
+    Explicit,
+    /// The kernel dropped events and asked us to re-read the subtree.
+    KernelEventLoss,
+    /// A `.gitignore`/`.ignore` changed, so the cached ignore rules are stale.
+    IgnoreFileChanged,
+    /// A single debounce batch touched more paths than we apply incrementally.
+    EventBatchOverflow,
+    /// The picker refused an incremental insert/update.
+    IndexUpdateRejected,
+    /// The post-scan overflow region ran out of slots.
+    OverflowCapacity,
+}
+
+impl RescanReason {
+    pub const ALL: [RescanReason; 6] = [
+        RescanReason::Explicit,
+        RescanReason::KernelEventLoss,
+        RescanReason::IgnoreFileChanged,
+        RescanReason::EventBatchOverflow,
+        RescanReason::IndexUpdateRejected,
+        RescanReason::OverflowCapacity,
+    ];
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            RescanReason::Explicit => "explicit",
+            RescanReason::KernelEventLoss => "kernel_event_loss",
+            RescanReason::IgnoreFileChanged => "ignore_file_changed",
+            RescanReason::EventBatchOverflow => "event_batch_overflow",
+            RescanReason::IndexUpdateRejected => "index_update_rejected",
+            RescanReason::OverflowCapacity => "overflow_capacity",
+        }
+    }
+
+    const fn slot(self) -> usize {
+        match self {
+            RescanReason::Explicit => 0,
+            RescanReason::KernelEventLoss => 1,
+            RescanReason::IgnoreFileChanged => 2,
+            RescanReason::EventBatchOverflow => 3,
+            RescanReason::IndexUpdateRejected => 4,
+            RescanReason::OverflowCapacity => 5,
+        }
+    }
+}
+
+impl std::fmt::Display for RescanReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
+/// Snapshot of rescan requests grouped by reason.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct RescanStats {
+    pub total: usize,
+    /// Requests suppressed during the cooldown.
+    pub throttled: usize,
+    counts: [usize; RescanReason::ALL.len()],
+    throttled_counts: [usize; RescanReason::ALL.len()],
+}
+
+impl RescanStats {
+    pub fn count(&self, reason: RescanReason) -> usize {
+        self.counts[reason.slot()]
+    }
+
+    pub fn count_throttled(&self, reason: RescanReason) -> usize {
+        self.throttled_counts[reason.slot()]
+    }
+
+    /// Admitted requests originating from watcher fallbacks.
+    pub fn watcher_triggered(&self) -> usize {
+        self.total - self.count(RescanReason::Explicit)
+    }
+
+    /// Per-reason delta against an earlier snapshot.
+    pub fn since(&self, earlier: &RescanStats) -> RescanStats {
+        let mut counts = [0usize; RescanReason::ALL.len()];
+        let mut throttled_counts = [0usize; RescanReason::ALL.len()];
+        for slot in 0..RescanReason::ALL.len() {
+            counts[slot] = self.counts[slot].saturating_sub(earlier.counts[slot]);
+            throttled_counts[slot] =
+                self.throttled_counts[slot].saturating_sub(earlier.throttled_counts[slot]);
+        }
+
+        RescanStats {
+            total: self.total.saturating_sub(earlier.total),
+            throttled: self.throttled.saturating_sub(earlier.throttled),
+            counts,
+            throttled_counts,
+        }
+    }
+}
+
+impl std::fmt::Display for RescanStats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} rescan(s)", self.total)?;
+        let mut first = true;
+        for reason in RescanReason::ALL {
+            let count = self.count(reason);
+            if count == 0 {
+                continue;
+            }
+            f.write_str(if first { " [" } else { ", " })?;
+            write!(f, "{reason}={count}")?;
+            first = false;
+        }
+        if !first {
+            f.write_str("]")?;
+        }
+        if self.throttled > 0 {
+            write!(f, ", {} throttled", self.throttled)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(rescan_stats)]
+#[derive(Default)]
+pub(crate) struct RescanCounters {
+    counters: [AtomicUsize; RescanReason::ALL.len()],
+    throttled: [AtomicUsize; RescanReason::ALL.len()],
+}
+
+#[cfg(rescan_stats)]
+impl RescanCounters {
+    pub(crate) fn record(&self, reason: RescanReason) {
+        self.counters[reason.slot()].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn record_throttled(&self, reason: RescanReason) {
+        self.throttled[reason.slot()].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn snapshot(&self) -> RescanStats {
+        let mut stats = RescanStats::default();
+        for reason in RescanReason::ALL {
+            let count = self.counters[reason.slot()].load(Ordering::Relaxed);
+            stats.counts[reason.slot()] = count;
+            stats.total += count;
+
+            let throttled = self.throttled[reason.slot()].load(Ordering::Relaxed);
+            stats.throttled_counts[reason.slot()] = throttled;
+            stats.throttled += throttled;
+        }
+        stats
+    }
+
+    pub(crate) fn reset(&self) {
+        for counter in self.counters.iter().chain(self.throttled.iter()) {
+            counter.store(0, Ordering::Relaxed);
+        }
+    }
+}
+
+// Release builds retain the API without counter storage.
+#[cfg(not(rescan_stats))]
+#[derive(Default)]
+pub(crate) struct RescanCounters;
+
+#[cfg(not(rescan_stats))]
+impl RescanCounters {
+    pub(crate) fn record(&self, _reason: RescanReason) {}
+
+    pub(crate) fn record_throttled(&self, _reason: RescanReason) {}
+
+    pub(crate) fn snapshot(&self) -> RescanStats {
+        RescanStats::default()
+    }
+
+    pub(crate) fn reset(&self) {}
+}
+
+#[cfg(all(test, rescan_stats))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn counters_attribute_and_diff_per_reason() {
+        let counters = RescanCounters::default();
+        counters.record(RescanReason::Explicit);
+        let baseline = counters.snapshot();
+
+        counters.record(RescanReason::IgnoreFileChanged);
+        counters.record(RescanReason::IgnoreFileChanged);
+        counters.record(RescanReason::OverflowCapacity);
+
+        let stats = counters.snapshot();
+        assert_eq!(stats.total, 4);
+        assert_eq!(stats.watcher_triggered(), 3);
+
+        let delta = stats.since(&baseline);
+        assert_eq!(delta.total, 3);
+        assert_eq!(delta.count(RescanReason::Explicit), 0);
+        assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 2);
+        assert_eq!(
+            delta.to_string(),
+            "3 rescan(s) [ignore_file_changed=2, overflow_capacity=1]"
+        );
+
+        counters.reset();
+        assert_eq!(counters.snapshot(), RescanStats::default());
+    }
+}

crates/fff-core/src/rescan_throttle.rs

@@ -0,0 +1,124 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Instant;
+
+use crate::constants::{
+    LARGE_INDEX_FILE_COUNT, RESCAN_MIN_INTERVAL, RESCAN_MIN_INTERVAL_LARGE_INDEX,
+};
+
+const NEVER: u64 = u64::MAX;
+
+// Drops watcher rescan requests inside the cooldown after the last scan.
+// A slightly stale index is fine: the next admitted event rescans everything.
+pub(crate) struct RescanThrottle {
+    epoch: Instant,
+    last_admitted: AtomicU64,
+}
+
+impl Default for RescanThrottle {
+    fn default() -> Self {
+        Self {
+            epoch: Instant::now(),
+            last_admitted: AtomicU64::new(NEVER),
+        }
+    }
+}
+
+impl RescanThrottle {
+    /// Returns `true` if a rescan may start now and records it as the last scan
+    pub(crate) fn admit(&self, live_files: usize, has_git_repo: bool) -> bool {
+        let min_interval = if !has_git_repo && live_files >= LARGE_INDEX_FILE_COUNT {
+            RESCAN_MIN_INTERVAL_LARGE_INDEX
+        } else {
+            RESCAN_MIN_INTERVAL
+        };
+
+        let min_ms = min_interval.as_millis() as u64;
+        let now = self.elapsed_ms();
+
+        loop {
+            let last = self.last_admitted.load(Ordering::Acquire);
+            if last != NEVER && now.saturating_sub(last) < min_ms {
+                return false;
+            }
+            // CAS so two concurrent requests cannot both start a walk.
+            if self
+                .last_admitted
+                .compare_exchange(last, now, Ordering::AcqRel, Ordering::Acquire)
+                .is_ok()
+            {
+                return true;
+            }
+        }
+    }
+
+    /// Records an explicit (unthrottled) scan so watcher requests right after
+    /// it are dropped: the index is already fresh.
+    pub(crate) fn note_explicit_scan(&self) {
+        self.last_admitted
+            .store(self.elapsed_ms(), Ordering::Release);
+    }
+
+    fn elapsed_ms(&self) -> u64 {
+        self.epoch.elapsed().as_millis() as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+
+    fn throttle_at(ms_ago: u64) -> RescanThrottle {
+        let now = Instant::now();
+        RescanThrottle {
+            epoch: now
+                .checked_sub(Duration::from_millis(ms_ago))
+                .expect("monotonic clock older than the rewind"),
+            last_admitted: AtomicU64::new(0),
+        }
+    }
+
+    #[test]
+    fn first_request_is_always_admitted() {
+        let throttle = RescanThrottle::default();
+        assert!(throttle.admit(100, true));
+    }
+
+    #[test]
+    fn requests_inside_the_cooldown_are_dropped() {
+        let throttle = throttle_at(1_000);
+        assert!(!throttle.admit(100, false));
+        assert!(!throttle.admit(100, false));
+    }
+
+    #[test]
+    fn a_large_index_outside_a_git_repo_uses_the_slower_cadence() {
+        // A minute is past the normal cooldown but not the large-index one.
+        let throttle = throttle_at(60_000);
+        assert!(throttle.admit(100, false));
+
+        let throttle = throttle_at(60_000);
+        assert!(!throttle.admit(LARGE_INDEX_FILE_COUNT, false));
+    }
+
+    #[test]
+    fn a_git_repo_keeps_the_normal_cadence_at_any_size() {
+        let throttle = throttle_at(60_000);
+        assert!(throttle.admit(LARGE_INDEX_FILE_COUNT, true));
+    }
+
+    #[test]
+    fn cooldown_expiry_admits_again() {
+        let throttle = throttle_at(RESCAN_MIN_INTERVAL.as_millis() as u64 + 1);
+        assert!(throttle.admit(100, false));
+        // Admission rearms the cooldown.
+        assert!(!throttle.admit(100, false));
+    }
+
+    #[test]
+    fn explicit_scan_rearms_the_cooldown() {
+        let throttle = RescanThrottle::default();
+        throttle.note_explicit_scan();
+        assert!(!throttle.admit(100, false));
+    }
+}

crates/fff-core/src/scan.rs

@@ -1,27 +1,3 @@
-//! Unified scan-phase orchestrator.
-//!
-//! Every (re)index code path — initial scan, FFI-triggered rescan,
-//! watcher overflow rescan — goes through [`ScanJob::run`]. The
-//! orchestrator owns the *sequence* of a scan:
-//!
-//!   1. walk filesystem off-lock
-//!   2. swap `sync_data` under a brief write
-//!   3. apply git status + frecency off-lock
-//!   4. (optional, initial scan only) spawn the filesystem watcher
-//!   5. (optional) post-scan: auto-size cache budget, warmup, bigram
-//!
-//! The picker write lock is held only in step 2 and step 5's index
-//! install — both O(µs-ms), never seconds. Every other FFI caller on
-//! the nvim main thread keeps running.
-//!
-//! ## Entry points
-//!
-//! - [`ScanJob::spawn`] — fire-and-forget from `SharedPicker` state.
-//!   Used by the watcher overflow path and by FFI (`scan_files`).
-//! - [`ScanJob::spawn_initial`] — same, but takes explicit config for
-//!   the very first scan, before the `FilePicker` struct lives inside
-//!   the shared handle.
-
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -29,17 +5,14 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use tracing::{error, info};
 
 use crate::FileSync;
-use crate::background_watcher::BackgroundWatcher;
-use crate::bigram_filter::BigramOverlay;
-use crate::bigram_filter::build_bigram_index;
 use crate::error::Error;
-use crate::file_picker::{self, FFFMode, warmup_mmaps};
+use crate::file_picker::FFFMode;
+use crate::index::{build_bigram_index, sniff_binary_for_non_indexable};
+use crate::parallelism::BACKGROUND_THREAD_POOL;
 use crate::shared::{SharedFilePicker, SharedFrecency};
 use crate::types::ContentCacheBudget;
+use crate::watch::BackgroundWatcher;
 
-/// Shared atomic flags surfaced by the picker for the scan worker to
-/// signal its progress. Grouped so every callsite passes one value,
-/// not four.
 #[derive(Clone, Default)]
 pub(crate) struct ScanSignals {
     /// Set to `true` while any scan phase is running
@@ -48,20 +21,25 @@ pub(crate) struct ScanSignals {
     pub(crate) watcher_ready: Arc<AtomicBool>,
     /// Indicates that that owning picker was requested to shut down
     pub(crate) cancelled: Arc<AtomicBool>,
-    /// Soft lock indicating that the post scan non blocking work is active
-    pub(crate) post_scan_busy: Arc<AtomicBool>,
     /// Used to resolve conflicts if multiple rescans were triggered in a queue
     pub(crate) rescan_pending: Arc<AtomicBool>,
+    /// Set by `post_scan_snapshot`, cleared by `PostScanSnapshot::drop`.
+    /// DO NOT set or clear this manually — it is managed exclusively by the
+    /// PostScanSnapshot lifecycle.
+    pub(crate) post_scan_indexing_active: Arc<AtomicBool>,
 }
 
 /// Which optional phases a scan should run.
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Debug)]
 pub(crate) struct ScanConfig {
     pub(crate) warmup: bool,
     pub(crate) content_indexing: bool,
     pub(crate) watch: bool,
     pub(crate) auto_cache_budget: bool,
     pub(crate) install_watcher: bool,
+    pub(crate) follow_symlinks: bool,
+    pub(crate) enable_fs_root_scanning: bool,
+    pub(crate) enable_home_dir_scanning: bool,
 }
 
 /// A fully-configured scan job ready to run on a background thread.
@@ -80,47 +58,58 @@ pub(crate) struct ScanJob {
     /// side. Reset to 0 at scan start, incremented per-file by the
     /// walker. Shared `Arc` so the UI polls the same atomic.
     scanned_files_counter: Arc<AtomicUsize>,
+    trace_span: tracing::Span,
 }
 
 impl ScanJob {
-    pub fn new(
+    pub fn new_rescan(
         shared_picker: &SharedFilePicker,
         shared_frecency: &SharedFrecency,
-        install_watcher: bool,
     ) -> Result<Option<Self>, Error> {
         let guard = shared_picker.read()?;
         let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?;
 
-        if picker.is_scan_active() {
+        if picker.is_scan_active()
+            || picker
+                .signals
+                .post_scan_indexing_active
+                .load(Ordering::Acquire)
+        {
             return Ok(None);
         }
 
+        let mode = picker.mode();
         let signals = picker.scan_signals();
-        if signals.post_scan_busy.load(Ordering::Acquire) {
-            return Ok(None);
-        }
+        let scanned_files_counter = picker.scanned_files_counter();
+        let base_path = picker.base_path().to_path_buf();
+        let trace_span = picker.trace_span();
+
+        let new_scan_config = ScanConfig {
+            warmup: picker.has_mmap_cache(),
+            content_indexing: picker.has_content_indexing(),
+            watch: picker.has_watcher(),
+            auto_cache_budget: !picker.has_explicit_cache_budget(),
+            install_watcher: false, // the watcher is independent of rescan, it is not restarting EVER
+            follow_symlinks: picker.follows_symlinks(),
+            enable_fs_root_scanning: picker.fs_root_scanning_enabled(),
+            enable_home_dir_scanning: picker.home_dir_scanning_enabled(),
+        };
+
+        drop(guard); // just a sanity check
 
         Ok(Some(Self {
+            mode,
+            signals,
+            base_path,
+            scanned_files_counter,
+            config: new_scan_config,
             shared_picker: shared_picker.clone(),
             shared_frecency: shared_frecency.clone(),
-            base_path: picker.base_path().to_path_buf(),
-            mode: picker.mode(),
-            signals,
-            scanned_files_counter: picker.scanned_files_counter(),
-            config: ScanConfig {
-                warmup: picker.has_mmap_cache(),
-                content_indexing: picker.has_content_indexing(),
-                watch: picker.has_watcher(),
-                auto_cache_budget: !picker.has_explicit_cache_budget(),
-                install_watcher,
-            },
+            trace_span,
         }))
     }
 
-    /// Same as [`new`] but without reading from the picker — caller
-    /// supplies the base path / mode / flags directly. Used by the
-    /// bootstrap scan before the `FilePicker` is published to
-    /// `SharedPicker`.
+    #[allow(clippy::too_many_arguments)]
     pub fn new_initial(
         shared_picker: SharedFilePicker,
         shared_frecency: SharedFrecency,
@@ -128,6 +117,7 @@ impl ScanJob {
         mode: FFFMode,
         signals: ScanSignals,
         scanned_files_counter: Arc<AtomicUsize>,
+        trace_span: tracing::Span,
         config: ScanConfig,
     ) -> Self {
         Self {
@@ -138,16 +128,25 @@ impl ScanJob {
             signals,
             scanned_files_counter,
             config,
+            trace_span,
         }
     }
 
-    /// Spawn the job on a dedicated OS thread. Returns immediately.
-    pub fn spawn(self) -> std::thread::JoinHandle<()> {
+    /// Run the job on `BACKGROUND_THREAD_POOL`. Returns immediately.
+    ///
+    /// Routed through the pool — and not a fresh `std::thread::spawn` — so the
+    /// orchestrator inherits rayon's QoS pin (USER_INITIATED). Without that
+    /// pin, an interactive nvim's USER_INTERACTIVE main thread spawns a child
+    /// at lower QoS, the walker's Zig worker pool inherits the demotion, and
+    /// the kernel drifts those workers onto E-cores. On chromium that turns a
+    /// ~800 ms walk into ~3 s.
+    pub fn spawn(self) {
         self.signals.scanning.store(true, Ordering::Release);
-        std::thread::Builder::new()
-            .name("fff-scan".into())
-            .spawn(move || self.run())
-            .expect("failed to spawn fff-scan thread")
+        let span = self.trace_span.clone();
+        BACKGROUND_THREAD_POOL.spawn(move || {
+            let _g = span.enter();
+            self.run();
+        });
     }
 
     fn run(self) {
@@ -159,23 +158,21 @@ impl ScanJob {
             signals,
             scanned_files_counter,
             config,
+            trace_span: _,
         } = self;
 
-        let _scanning = ScanningGuard::new(&signals, config.install_watcher);
-
-        // Reset the UI-visible counter; the walker bumps it per file
-        // and `get_scan_progress` reads it without locks.
+        let _scanning = ScanningGuard::new(&signals);
         scanned_files_counter.store(0, Ordering::Relaxed);
 
-        // 1. Start git discovery and walk filesystem off-lock.
+        // 1. Walk the file system and collect the list of files
         let git_workdir = FileSync::discover_git_workdir(&base_path);
-        let status_handle = git_workdir.clone().map(FileSync::spawn_git_status);
         let sync = match FileSync::walk_filesystem(
             &base_path,
-            git_workdir,
+            git_workdir.clone(),
             &scanned_files_counter,
             &shared_frecency,
             mode,
+            config.follow_symlinks,
         ) {
             Ok(sync) => sync,
             Err(e) => {
@@ -184,47 +181,65 @@ impl ScanJob {
             }
         };
 
-        if signals.cancelled.load(Ordering::Acquire) {
-            info!("walk completed but picker was replaced, discarding results");
-            return;
-        }
-
-        let git_workdir = sync.git_workdir.clone();
-
-        // 2. Brief write to install the freshly-walked file list.
+        // 2. Populate the file list
+        let git_status_worker;
         if let Ok(mut guard) = shared_picker.write()
             && let Some(picker) = guard.as_mut()
         {
+            if signals.cancelled.load(Ordering::Acquire) {
+                info!("scan cancelled between walk and commit, discarding");
+                return;
+            }
+
+            let live_count = sync.live_count;
             picker.commit_new_sync(sync);
+            git_status_worker = Arc::clone(&picker.git_status_worker);
+
+            if config.auto_cache_budget && !picker.has_explicit_cache_budget() {
+                picker.set_cache_budget(ContentCacheBudget::new_for_repo(live_count));
+            }
         } else {
             error!("failed to install scan results into picker");
             return;
         }
 
-        // Files are now searchable — flip the scan signal *early* so
-        // UI progress polls see the picker as "ready" while we run the
-        // optional post-scan steps in the background.
-        signals.scanning.store(false, Ordering::Relaxed);
+        // Spawn the git status worker once. BUG PINNNING. If the user initiated git in the folder
+        // which is a real use case we need to have a way to start the git worker background thread dynamically
+        if git_workdir.is_some() && !signals.cancelled.load(Ordering::Acquire) {
+            git_status_worker.spawn_once(shared_picker.weaken(), shared_frecency.clone());
+            git_status_worker.request_full_rescan(); // this runs anyway
+        }
+
+        // BUG pinning: take the snapshot *before* the storing the scan=true, otherwise there is a tiny
+        // race window when there scanned is set to true, but `post_scan_indexing_active` flag is `false`
+        let snapshot = if !signals.cancelled.load(Ordering::Acquire) {
+            shared_picker.read().ok().and_then(|guard| {
+                guard
+                    .as_ref()
+                    .and_then(|picker| unsafe { picker.post_scan_snapshot() })
+            })
+        } else {
+            None
+        };
+
+        signals.scanning.store(false, Ordering::Relaxed); // file are searchable
 
         // in case we do a rescan, we have to resubscribe a watcher to the new set of directories
-        // all the already watched directories are not going to be resubscribed
+        // all the already watched directories are not going to be resubscribed (this is internally deduped)
         if !config.install_watcher && !signals.cancelled.load(Ordering::Acquire) {
-            resubscribe_to_new_picker(&shared_picker);
+            rescubscribe_watcher_post_scan(&shared_picker);
         }
 
-        // 3. Apply git status + frecency off-lock.
+        // 3. Runs post scna in parallel with git status collection
         if !signals.cancelled.load(Ordering::Acquire)
-            && let Some(status_handle) = status_handle
+            && let Some(snap) = snapshot.as_ref()
         {
-            file_picker::apply_git_status_and_frecency(
-                &shared_picker,
-                &shared_frecency,
-                status_handle,
-                mode,
-            );
+            Self::run_post_scan(&shared_picker, &signals, &config, snap);
         }
 
-        // 4. Install filesystem watcher (initial scan only).
+        drop(snapshot); // SNAPSHOT SHOULD NOT BE USED AFTER THIS POINT
+
+        // 5. Install filesystem watcher (initial scan only).
         if config.install_watcher && config.watch && !signals.cancelled.load(Ordering::Acquire) {
             let shared_picker: &SharedFilePicker = &shared_picker;
             let shared_frecency: &SharedFrecency = &shared_frecency;
@@ -236,46 +251,40 @@ impl ScanJob {
                 shared_picker.clone(),
                 shared_frecency.clone(),
                 mode,
+                config.enable_fs_root_scanning,
+                config.enable_home_dir_scanning,
+                git_status_worker,
+                tracing::Span::current(),
             ) {
                 Ok(watcher) => {
                     if let Ok(mut guard) = shared_picker.write()
                         && let Some(picker) = guard.as_mut()
+                        && picker.base_path() == base_path
+                        && !signals.cancelled.load(Ordering::Acquire)
                     {
                         picker.background_watcher = Some(watcher);
+                        signals.watcher_ready.store(true, Ordering::Release);
                     }
                 }
                 Err(e) => error!(?e, "failed to initialize background watcher"),
             };
         }
 
-        // 5. Post-scan warmup + bigram build.
-        if (config.warmup || config.content_indexing) && !signals.cancelled.load(Ordering::Acquire)
-        {
-            run_post_scan(&shared_picker, &base_path, &signals, &config);
-        }
-
         // 6. Drain any rescan that arrived while we were busy.
-        //
-        // `trigger_full_rescan_async` sets `rescan_pending` whenever a
-        // caller asks for a rescan while `ScanJob::new` would have
-        // returned `Ok(None)` (scan active *or* post-scan busy). We
-        // consume the flag with `swap` so concurrent requests that land
-        // between the check and the follow-up spawn are still captured
-        // by the next invocation.
+        // if user initiated a new rescan we had no way to cancel current post scan, so do it again
         if !signals.cancelled.load(Ordering::Acquire)
             && signals.rescan_pending.swap(false, Ordering::AcqRel)
         {
-            match Self::new(&shared_picker, &shared_frecency, false) {
+            match Self::new_rescan(&shared_picker, &shared_frecency) {
                 Ok(Some(follow_up)) => {
                     info!("Rescheduling deferred rescan after current scan finished");
                     follow_up.spawn();
                 }
                 Ok(None) => {
-                    // Another scan slipped in between our post-scan exit
-                    // and the `new()` call above. That scan will drain
-                    // the flag we just cleared — but we re-arm it so it
-                    // does.
-                    signals.rescan_pending.store(true, Ordering::Release);
+                    // this should be practically impossible because we do not have any
+                    // queue, but if somehow a new rescan was triggered JUST IN THIS MOMENT
+                    // just ignore it because the ongoing one is fresh enough
+                    tracing::warn!("Post scan was re-triggered, ignoring");
                 }
                 Err(e) => {
                     error!(?e, "Failed to reschedule deferred rescan");
@@ -283,109 +292,95 @@ impl ScanJob {
             }
         }
     }
+
+    /// THIS IS VERY VERY IMPORTANT THAT ANYTHING INSIDE THIS FUNCTION TO NOT READ ANYTHING CLEARABLE OUTSIDE
+    /// this is a very silly off lock implementation that actually matters, and that's why it is crafted
+    /// to never read anything from the picker, it can only WRITE information using single instructions
+    ///
+    /// Things that are safe and immutable - file list, indexes of files, paths, and signals.
+    #[tracing::instrument(skip_all, fields(warmup = ?config.warmup, indexing = ?config.content_indexing))]
+    fn run_post_scan(
+        shared_picker: &SharedFilePicker,
+        signals: &ScanSignals,
+        config: &ScanConfig,
+        unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot,
+    ) {
+        let Some(arena) = unsafe_snapshot
+            .arena // we are never touching overlays so this arena is always correct
+            .as_ref()
+            .map(|s| s.as_arena_ptr())
+        else {
+            tracing::error!("Failed to run post scan: arena is invalid");
+            return;
+        };
+
+        let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
+        if signals.cancelled.load(Ordering::Acquire) {
+            return;
+        }
+
+        if config.content_indexing {
+            let indexable_count = unsafe_snapshot.indexable_count.min(files.len());
+            let (indexable_files, non_indexable_files) = files.split_at(indexable_count);
+            let index = build_bigram_index(indexable_files, &unsafe_snapshot.base_path, arena);
+
+            if let Ok(mut guard) = shared_picker.write()
+                && let Some(picker) = guard.as_mut()
+            {
+                picker.set_bigram_index(index);
+            }
+
+            // Bigram only sniffs files <= MAX_INDEXABLE_FILE_SIZE; large
+            // unknown-extension binaries slip past it and would otherwise be
+            // grep-able as text. Cheap header sniff catches those.
+            if !signals.cancelled.load(Ordering::Acquire) {
+                sniff_binary_for_non_indexable(
+                    non_indexable_files,
+                    &unsafe_snapshot.base_path,
+                    arena,
+                    &signals.cancelled,
+                );
+            }
+        } else {
+            // this potentially a long running as we are not parallelizing it but it's okay
+            sniff_binary_for_non_indexable(
+                files,
+                &unsafe_snapshot.base_path,
+                arena,
+                &signals.cancelled,
+            );
+        }
+
+        // TODO Skipped as potentially unsafe - figure this out later
+        // if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
+        //     warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
+        // }
+    }
 }
 
-/// RAII helper that flips the `scanning` signal on construction and
-/// resets it on drop (so early-returns can't leave it stuck on `true`).
-/// Also drives the `watcher_ready` signal on the initial-scan path.
+// Ensures early returns clear the scanning signal.
 struct ScanningGuard<'a> {
     signals: &'a ScanSignals,
-    release_watcher_ready_on_drop: bool,
 }
 
 impl<'a> ScanningGuard<'a> {
-    fn new(signals: &'a ScanSignals, release_watcher_ready_on_drop: bool) -> Self {
+    fn new(signals: &'a ScanSignals) -> Self {
         signals.scanning.store(true, Ordering::Relaxed);
-        Self {
-            signals,
-            release_watcher_ready_on_drop,
-        }
+        Self { signals }
     }
 }
 
 impl Drop for ScanningGuard<'_> {
     fn drop(&mut self) {
         self.signals.scanning.store(false, Ordering::Relaxed);
-        if self.release_watcher_ready_on_drop {
-            self.signals.watcher_ready.store(true, Ordering::Release);
-        }
     }
 }
 
-fn run_post_scan(
-    shared_picker: &SharedFilePicker,
-    base_path: &std::path::Path,
-    signals: &ScanSignals,
-    config: &ScanConfig,
-) {
-    let phase_start = std::time::Instant::now();
-
-    // Auto-scale the cache budget before we take the files snapshot —
-    // warmup needs the final budget.
-    if config.auto_cache_budget
-        && !signals.cancelled.load(Ordering::Acquire)
-        && let Ok(mut guard) = shared_picker.write()
-        && let Some(picker) = guard.as_mut()
-        && !picker.has_explicit_cache_budget()
-    {
-        let (files, _, _) = picker.sync_data_snapshot();
-        picker.set_cache_budget(ContentCacheBudget::new_for_repo(files.len()));
-    }
-
-    let Some((files, indexable_count, budget, arena, _busy_guard)) = shared_picker
-        .read()
-        .ok()
-        .and_then(|guard| guard.as_ref().map(|p| snapshot_sync_data(p, signals)))
-    else {
-        return;
-    };
-
-    if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
-        let t = std::time::Instant::now();
-        warmup_mmaps(files, &budget, base_path, arena);
-        info!(
-            "Warmup completed in {:.2}s (cached {} files, {} bytes)",
-            t.elapsed().as_secs_f64(),
-            budget.cached_count.load(Ordering::Relaxed),
-            budget.cached_bytes.load(Ordering::Relaxed),
-        );
-    }
-
-    if config.content_indexing && !signals.cancelled.load(Ordering::Acquire) {
-        let indexable_files = &files[..indexable_count.min(files.len())];
-        let (index, content_binary) =
-            build_bigram_index(indexable_files, &budget, base_path, arena);
-
-        if let Ok(mut guard) = shared_picker.write()
-            && let Some(picker) = guard.as_mut()
-        {
-            for &idx in &content_binary {
-                if let Some(file) = picker.get_file_mut(idx) {
-                    file.set_binary(true);
-                }
-            }
-            picker.set_bigram_index(index, BigramOverlay::new(indexable_count));
-        }
-    }
-
-    info!(
-        "Post-scan phase total: {:.2}s (warmup={}, content_indexing={})",
-        phase_start.elapsed().as_secs_f64(),
-        config.warmup,
-        config.content_indexing,
-    );
-}
-
-struct PostScanBusyGuard<'a>(&'a AtomicBool);
-impl Drop for PostScanBusyGuard<'_> {
-    fn drop(&mut self) {
-        self.0.store(false, Ordering::Release);
-    }
-}
-
-/// Re-registers all the directories at the watcher
+/// If the scan encounters new directories created we have to add them to the watch list
+/// this is fine because the watcher does deduplicate the entries and doesn't add a lot of
+/// garbage notify watchers / fs events streams
 #[tracing::instrument(skip_all)]
-fn resubscribe_to_new_picker(shared_picker: &SharedFilePicker) {
+fn rescubscribe_watcher_post_scan(shared_picker: &SharedFilePicker) {
     let Ok(guard) = shared_picker.read() else {
         return;
     };
@@ -396,46 +391,8 @@ fn resubscribe_to_new_picker(shared_picker: &SharedFilePicker) {
         return;
     };
 
-    // Base path first — this is the watch that delivers `Create(Folder)`
-    // events for brand-new top-level subdirs. On rescan paths this
-    // watch is still alive (the BackgroundWatcher survives rescans), so
-    // the call is idempotent. Including it explicitly protects against
-    // any future refactor that could drop the initial base-path watch.
-    watcher.request_watch_dir(picker.base_path().to_path_buf());
-
     picker.for_each_dir(|dir: &std::path::Path| {
         watcher.request_watch_dir(dir.to_path_buf());
         std::ops::ControlFlow::Continue(())
     });
 }
-
-/// Take a `'static`-lifetime snapshot of `sync_data` pinned by a
-/// post-scan busy guard. Concurrent rescans short-circuit while the
-/// returned guard is alive, so the raw slice can't be freed from under
-/// the warmup + bigram build that consumes it.
-fn snapshot_sync_data<'a>(
-    picker: &crate::file_picker::FilePicker,
-    signals: &'a ScanSignals,
-) -> (
-    &'static [crate::types::FileItem],
-    usize,
-    Arc<ContentCacheBudget>,
-    crate::simd_path::ArenaPtr,
-    PostScanBusyGuard<'a>,
-) {
-    signals.post_scan_busy.store(true, Ordering::Release);
-    let busy = PostScanBusyGuard(&signals.post_scan_busy);
-
-    let (files, indexable_count, arena) = picker.sync_data_snapshot();
-    let ptr = files.as_ptr();
-    let len = files.len();
-    let static_files: &'static [crate::types::FileItem] =
-        unsafe { std::slice::from_raw_parts(ptr, len) };
-    (
-        static_files,
-        indexable_count,
-        picker.cache_budget_arc(),
-        arena,
-        busy,
-    )
-}

crates/fff-core/src/score.rs

@@ -1,14 +1,15 @@
 use crate::{
-    constraints::apply_constraints,
     git::is_modified_status,
+    index::constraints::apply_constraints,
     path_utils::calculate_distance_penalty,
-    simd_path::ArenaPtr,
+    simd_path::{ArenaPtr, MAX_PATH_CHUNKS},
     sort_buffer::{sort_by_key_with_buffer, sort_with_buffer},
     types::{DirItem, FileItem, Score, ScoringContext},
 };
-use fff_query_parser::FuzzyQuery;
+use fff_query_parser::{FFFQuery, FuzzyQuery};
 use neo_frizbee::Scoring;
 use rayon::prelude::*;
+use smallvec::SmallVec;
 use std::{borrow::Cow, path::MAIN_SEPARATOR};
 
 enum FileItems<'a> {
@@ -32,7 +33,7 @@ impl<'a> FileItems<'a> {
 fn resolve_file_chunks(
     file: &FileItem,
     arena: ArenaPtr,
-    buf: &mut [*const u8; 32],
+    buf: &mut [*const u8; MAX_PATH_CHUNKS],
 ) -> Option<(usize, u16)> {
     if file.is_deleted() {
         return None;
@@ -60,15 +61,15 @@ fn match_fuzzy_parts(
         return vec![];
     }
 
-    let resolve = |file: &FileItem, buf: &mut [*const u8; 32]| -> Option<(usize, u16)> {
-        resolve_file_chunks(file, arena, buf)
-    };
+    let resolve = |file: &FileItem,
+                   buf: &mut [*const u8; MAX_PATH_CHUNKS]|
+     -> Option<(usize, u16)> { resolve_file_chunks(file, arena, buf) };
 
     // because we reassemble the vec of reference we have to use a different type
     // to narrow down the [&FileItem] which would be resolved by frizbee as &&
-    let resolve_ref = |file: &&FileItem, buf: &mut [*const u8; 32]| -> Option<(usize, u16)> {
-        resolve_file_chunks(file, arena, buf)
-    };
+    let resolve_ref = |file: &&FileItem,
+                       buf: &mut [*const u8; MAX_PATH_CHUNKS]|
+     -> Option<(usize, u16)> { resolve_file_chunks(file, arena, buf) };
 
     let first_part_matches = match working_files {
         FileItems::All(files) => neo_frizbee::match_list_parallel_resolved(
@@ -168,13 +169,141 @@ pub(crate) fn fuzzy_match_and_score_files<'a>(
     sort_and_paginate(results, context)
 }
 
+pub(crate) fn fuzzy_match_byte_offsets_for_page<'q>(
+    query: &'q FFFQuery<'q>,
+    items: &[&FileItem],
+    max_typos: u16,
+    base_arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> Vec<SmallVec<[(u32, u32); 4]>> {
+    let parts: Vec<&str> = match &query.fuzzy_query {
+        FuzzyQuery::Text(text) if text.len() >= 2 => vec![*text],
+        FuzzyQuery::Parts(parts) => parts.iter().copied().filter(|p| p.len() >= 2).collect(),
+        _ => Vec::new(),
+    };
+
+    let mut ranges_by_item = vec![SmallVec::new(); items.len()];
+    if parts.is_empty() || items.is_empty() {
+        return ranges_by_item;
+    }
+
+    let paths: Vec<String> = items
+        .iter()
+        .map(|item| {
+            let arena = if item.is_overflow() {
+                overflow_arena
+            } else {
+                base_arena
+            };
+            let mut path = String::with_capacity(item.relative_path_len());
+            item.write_relative_path_from_arena(arena, &mut path);
+            path
+        })
+        .collect();
+
+    let has_uppercase = parts
+        .iter()
+        .any(|part| part.chars().any(|ch| ch.is_uppercase()));
+    let config = neo_frizbee::Config {
+        max_typos: Some(max_typos),
+        sort: false,
+        scoring: Scoring {
+            capitalization_bonus: if has_uppercase { 8 } else { 0 },
+            matching_case_bonus: if has_uppercase { 4 } else { 0 },
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+
+    for (idx, part) in parts.iter().copied().enumerate() {
+        let mut part_config = config;
+        if idx > 0 {
+            part_config.max_typos = config.max_typos.map(|t| t.min(part.len() as u16));
+        }
+
+        let mut matcher = neo_frizbee::Matcher::new(part, &part_config);
+        for mut matched in matcher.match_list_indices(&paths) {
+            let item_idx = matched.index as usize;
+            let Some(path) = paths.get(item_idx) else {
+                continue;
+            };
+
+            matched.indices.sort_unstable();
+            ranges_by_item[item_idx].extend(char_indices_to_byte_offsets(path, &matched.indices));
+        }
+    }
+
+    for ranges in &mut ranges_by_item {
+        *ranges = merge_byte_offsets(std::mem::take(ranges));
+    }
+
+    ranges_by_item
+}
+
+fn char_indices_to_byte_offsets(line: &str, char_indices: &[usize]) -> SmallVec<[(u32, u32); 4]> {
+    let char_byte_ranges: Vec<(usize, usize)> = line
+        .char_indices()
+        .map(|(byte_pos, ch)| (byte_pos, byte_pos + ch.len_utf8()))
+        .collect();
+    let mut result: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(char_indices.len());
+
+    for &char_idx in char_indices {
+        let Some(&(start, end)) = char_byte_ranges.get(char_idx) else {
+            continue;
+        };
+
+        if let Some(last) = result.last_mut()
+            && last.1 == start as u32
+        {
+            last.1 = end as u32;
+            continue;
+        }
+
+        result.push((start as u32, end as u32));
+    }
+
+    result
+}
+
+fn merge_byte_offsets(mut ranges: SmallVec<[(u32, u32); 4]>) -> SmallVec<[(u32, u32); 4]> {
+    if ranges.len() <= 1 {
+        return ranges;
+    }
+
+    ranges.sort_unstable_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
+    let mut merged: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(ranges.len());
+
+    for (start, end) in ranges {
+        if end <= start {
+            continue;
+        }
+
+        if let Some(last) = merged.last_mut()
+            && start <= last.1
+        {
+            last.1 = last.1.max(end);
+            continue;
+        }
+
+        merged.push((start, end));
+    }
+
+    merged
+}
+
 /// Resolve a DirItem's chunked path into frizbee's pointer buffer.
 #[inline]
 fn resolve_dir_chunks(
     dir: &DirItem,
     arena: ArenaPtr,
-    buf: &mut [*const u8; 32],
+    overflow_arena: ArenaPtr,
+    buf: &mut [*const u8; MAX_PATH_CHUNKS],
 ) -> Option<(usize, u16)> {
+    let arena = if dir.is_overflow() {
+        overflow_arena
+    } else {
+        arena
+    };
     let ptrs = dir.path.resolve_ptrs(arena, buf);
     Some((ptrs.len(), dir.path.byte_len))
 }
@@ -187,6 +316,7 @@ fn match_fuzzy_parts_dirs(
     options: &neo_frizbee::Config,
     max_threads: usize,
     arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
 ) -> Vec<neo_frizbee::Match> {
     let valid_parts: Vec<&str> = fuzzy_parts
         .iter()
@@ -199,8 +329,8 @@ fn match_fuzzy_parts_dirs(
     }
 
     let resolve_chunks_for_frizbee =
-        |dir: &&DirItem, buf: &mut [*const u8; 32]| -> Option<(usize, u16)> {
-            resolve_dir_chunks(dir, arena, buf)
+        |dir: &&DirItem, buf: &mut [*const u8; MAX_PATH_CHUNKS]| -> Option<(usize, u16)> {
+            resolve_dir_chunks(dir, arena, overflow_arena, buf)
         };
 
     let first_part_matches = neo_frizbee::match_list_parallel_resolved(
@@ -267,19 +397,23 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
     dirs: &'a [DirItem],
     context: &ScoringContext,
     arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
 ) -> (Vec<&'a DirItem>, Vec<Score>, usize) {
     if dirs.is_empty() {
         return (vec![], vec![], 0);
     }
 
     let parsed_query = context.query;
+    // Ghost dirs (all files tombstoned) never surface in search results.
     let working_dirs: Vec<&DirItem> = if parsed_query.constraints.is_empty() {
-        dirs.iter().collect()
+        dirs.iter().filter(|d| !d.is_deleted()).collect()
     } else {
-        match apply_constraints(dirs, &parsed_query.constraints, arena) {
-            Some(filtered) if !filtered.is_empty() => filtered,
+        match apply_constraints(dirs, &parsed_query.constraints, arena, overflow_arena) {
+            Some(filtered) if !filtered.is_empty() => {
+                filtered.into_iter().filter(|d| !d.is_deleted()).collect()
+            }
             Some(_) => return (vec![], vec![], 0),
-            None => dirs.iter().collect(),
+            None => dirs.iter().filter(|d| !d.is_deleted()).collect(),
         }
     };
 
@@ -291,20 +425,6 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
         }
     };
 
-    // See `score_files` — stored dir paths are platform-native on Windows.
-    #[cfg(windows)]
-    let fuzzy_parts_owned: Option<Vec<String>> = if fuzzy_parts.iter().any(|p| p.contains('/')) {
-        Some(fuzzy_parts.iter().map(|p| p.replace('/', "\\")).collect())
-    } else {
-        None
-    };
-    #[cfg(windows)]
-    let fuzzy_parts_refs: Option<Vec<&str>> = fuzzy_parts_owned
-        .as_ref()
-        .map(|v| v.iter().map(String::as_str).collect());
-    #[cfg(windows)]
-    let fuzzy_parts: &[&str] = fuzzy_parts_refs.as_deref().unwrap_or(fuzzy_parts);
-
     let valid_parts: Vec<&str> = fuzzy_parts
         .iter()
         .copied()
@@ -327,6 +447,7 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
             matching_case_bonus: if has_uppercase { 4 } else { 0 },
             ..Default::default()
         },
+        ..Default::default()
     };
 
     let path_matches = match_fuzzy_parts_dirs(
@@ -335,6 +456,7 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
         &options,
         context.max_threads,
         arena,
+        overflow_arena,
     );
 
     let main_needle = valid_parts[0].as_bytes();
@@ -347,12 +469,17 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
         .into_iter()
         .map(|path_match| {
             let dir = working_dirs[path_match.index as usize];
+            let dir_arena = if dir.is_overflow() {
+                overflow_arena
+            } else {
+                arena
+            };
             let base_score = path_match.score as i32;
             let frecency_boost = base_score.saturating_mul(dir.max_access_frecency()) / 100;
 
             // Distance penalty from current file's directory.
             let distance_penalty = if context.current_file.is_some() {
-                dir.path.write_to_string(arena, &mut dir_buf);
+                dir.path.write_to_string(dir_arena, &mut dir_buf);
                 calculate_distance_penalty(context.current_file, &dir_buf)
             } else {
                 0
@@ -363,7 +490,7 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
             let match_start_approx = path_match.end_col.saturating_sub(main_needle_len - 1);
             let is_dirname_match = match_start_approx >= last_seg_offset;
 
-            dir.write_dir_name(arena, &mut dirname_buf);
+            dir.write_dir_name(dir_arena, &mut dirname_buf);
             let dirname_len = dirname_buf.len();
             let is_exact_dirname = is_dirname_match
                 && main_needle_len as usize == dirname_len
@@ -488,7 +615,7 @@ fn match_and_score_in_arena<'a>(
     let working_files: FileItems<'a> = if parsed.constraints.is_empty() {
         FileItems::All(files)
     } else {
-        match apply_constraints(files, &parsed.constraints, arena) {
+        match apply_constraints(files, &parsed.constraints, arena, arena) {
             Some(filtered) if !filtered.is_empty() => FileItems::Filtered(filtered),
             Some(_) => {
                 return vec![];
@@ -505,22 +632,6 @@ fn match_and_score_in_arena<'a>(
         }
     };
 
-    // On Windows, stored relative paths use the native `\\` separator while
-    // users type `/`. Translate so frizbee sees the same bytes it would on
-    // a path stored by the walker.
-    #[cfg(windows)]
-    let fuzzy_parts_owned: Option<Vec<String>> = if fuzzy_parts.iter().any(|p| p.contains('/')) {
-        Some(fuzzy_parts.iter().map(|p| p.replace('/', "\\")).collect())
-    } else {
-        None
-    };
-    #[cfg(windows)]
-    let fuzzy_parts_refs: Option<Vec<&str>> = fuzzy_parts_owned
-        .as_ref()
-        .map(|v| v.iter().map(String::as_str).collect());
-    #[cfg(windows)]
-    let fuzzy_parts: &[&str] = fuzzy_parts_refs.as_deref().unwrap_or(fuzzy_parts);
-
     debug_assert!(!fuzzy_parts.is_empty());
     let has_uppercase = fuzzy_parts
         .iter()
@@ -539,6 +650,7 @@ fn match_and_score_in_arena<'a>(
             matching_case_bonus: if has_uppercase { 4 } else { 0 },
             ..Default::default()
         },
+        ..Default::default()
     };
 
     let path_matches = match_fuzzy_parts(
@@ -841,13 +953,17 @@ fn score_filtered_by_frecency<'a>(
     match files {
         FileItems::All(s) => s
             .par_iter()
-            .filter(|f| !f.is_deleted())
-            .map(&score_file)
+            .filter_map(|f| {
+                let live = !f.is_deleted();
+                live.then_some(score_file(f))
+            })
             .collect(),
         FileItems::Filtered(v) => v
             .iter()
-            .filter(|f| !f.is_deleted())
-            .map(|&file| score_file(file))
+            .filter_map(|f| {
+                let live = !f.is_deleted();
+                live.then_some(score_file(f))
+            })
             .collect(),
     }
 }
@@ -1341,6 +1457,31 @@ mod filename_bonus_tests {
         );
     }
 
+    /// Regression: PR #652 / field panic in pi-fff v0.9.6.
+    /// A path >512 bytes (but within PATH_MAX) overflows the fixed
+    /// `[*const u8; 32]` chunk-pointer buffer during scoring and panics with
+    /// "index out of bounds: the len is 32 but the index is 32".
+    #[test]
+    fn test_path_longer_than_512_bytes_does_not_panic_and_matches() {
+        let mut long_path = String::new();
+        while long_path.len() < 600 {
+            long_path.push_str("deeply_nested_directory_segment/");
+        }
+        long_path.push_str("needle_file.rs");
+        assert!(long_path.len() > 512 && long_path.len() < crate::simd_path::PATH_BUF_SIZE);
+
+        let (files, arena) = make_files(&[long_path.as_str(), "src/other.rs"]);
+
+        // Panics here on unfixed code: frizbee resolves chunk ptrs per file.
+        let results = search(&files, "needle", arena);
+
+        assert!(
+            results.iter().any(|(p, _)| p == &long_path),
+            "filename at the tail of a >512-byte path must still match, got: {:?}",
+            results.iter().map(|(p, _)| p).collect::<Vec<_>>()
+        );
+    }
+
     #[test]
     fn test_single_path_matching() {
         let path = "core_workflow_service/kafka_event_consumer/src/ai_part_extraction_request/ai_part_extraction_request_handler.rs";

crates/fff-core/src/shared.rs

@@ -2,12 +2,17 @@ use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak};
 use std::time::{Duration, Instant};
 
+use crate::dbs::lmdb::{LmdbStore, spawn_lmdb_gc};
 use crate::error::Error;
 use crate::file_picker::FilePicker;
 use crate::frecency::FrecencyTracker;
 use crate::git::GitStatusCache;
 use crate::query_tracker::QueryTracker;
+use crate::rescan_stats::{RescanCounters, RescanReason, RescanStats};
+use crate::rescan_throttle::RescanThrottle;
 use crate::scan::ScanJob;
+use crate::watch::{WatchEvent, WatchId, WatchOptions, WatchRegistry};
+use git2::Repository;
 
 /// Poll `.git/index.lock` until it disappears (git write completed), giving up
 /// after [`GIT_LOCK_MAX_WAIT`]. Used by [`SharedPicker::refresh_git_status`]
@@ -37,6 +42,19 @@ fn wait_for_git_index_lock_release(git_root: &Path) {
     }
 }
 
+/// Poll `done` every 10ms until it returns `true`, or until `timeout` elapses.
+/// Returns `true` if the condition was met, `false` on timeout.
+fn poll_until(timeout: Duration, mut done: impl FnMut() -> bool) -> bool {
+    let start = Instant::now();
+    while !done() {
+        if start.elapsed() >= timeout {
+            return false;
+        }
+        std::thread::sleep(Duration::from_millis(10));
+    }
+    true
+}
+
 /// Thread-safe shared handle to the [`FilePicker`] instance.
 /// This accumulates only asynchronous non-blocking operations against the
 /// file picker: creating, triggering various rescans and so on.
@@ -58,12 +76,20 @@ pub struct SharedFilePicker(pub(crate) Arc<SharedPickerInner>);
 
 pub struct SharedPickerInner {
     picker: parking_lot::RwLock<Option<FilePicker>>,
+    /// Watch subscriptions live outside the picker lock so delivery and
+    /// (un)subscribing never contend with searches.
+    watchers: Arc<WatchRegistry>,
+    rescans: RescanCounters,
+    rescan_throttle: RescanThrottle,
 }
 
 impl Default for SharedPickerInner {
     fn default() -> Self {
         Self {
             picker: parking_lot::RwLock::new(None),
+            watchers: Arc::new(WatchRegistry::default()),
+            rescans: RescanCounters::default(),
+            rescan_throttle: RescanThrottle::default(),
         }
     }
 }
@@ -98,6 +124,16 @@ impl SharedFilePicker {
         Ok(self.0.picker.write())
     }
 
+    /// Signal the background scan to cancel. Non-blocking: post-scan
+    /// threads check this flag and bail out at their next cancellation point.
+    pub fn cancel(&self) {
+        if let Ok(guard) = self.read()
+            && let Some(picker) = guard.as_ref()
+        {
+            picker.cancel();
+        }
+    }
+
     /// Produce a non-owning handle to the same inner picker.
     /// Use it if you don't need to block internal threads from dropping while owning this ref
     pub(crate) fn weaken(&self) -> WeakFilePicker {
@@ -124,14 +160,9 @@ impl SharedFilePicker {
             }
         };
 
-        let start = std::time::Instant::now();
-        while signal.load(std::sync::atomic::Ordering::Acquire) {
-            if start.elapsed() >= timeout {
-                return false;
-            }
-            std::thread::sleep(Duration::from_millis(10));
-        }
-        true
+        poll_until(timeout, || {
+            !signal.load(std::sync::atomic::Ordering::Acquire)
+        })
     }
 
     /// Block until the background file watcher is ready.
@@ -145,28 +176,74 @@ impl SharedFilePicker {
             }
         };
 
-        let start = std::time::Instant::now();
-        while !watch_ready_signal.load(std::sync::atomic::Ordering::Acquire) {
-            if start.elapsed() >= timeout {
-                return false;
+        poll_until(timeout, || {
+            watch_ready_signal.load(std::sync::atomic::Ordering::Acquire)
+        })
+    }
+
+    /// Blocks until both the filesystem walk and post-scan indexing are done.
+    /// Returns true once scanning=false AND post_scan_indexing_active=false.
+    pub fn wait_for_indexing_complete(&self, timeout: Duration) -> bool {
+        let (scanning, post_scan_active) = {
+            let guard = self.0.picker.read();
+            match &*guard {
+                Some(picker) => (
+                    Arc::clone(&picker.signals.scanning),
+                    Arc::clone(&picker.signals.post_scan_indexing_active),
+                ),
+                None => return true,
             }
-            std::thread::sleep(Duration::from_millis(10));
-        }
-        true
+        };
+
+        poll_until(timeout, || {
+            !scanning.load(std::sync::atomic::Ordering::Acquire)
+                && !post_scan_active.load(std::sync::atomic::Ordering::Acquire)
+        })
     }
 
     /// Trigger a full filesystem rescan without blocking the caller.
     /// Performs a safe async rescan. Guarantees only single active rescan per picker.
     /// If many rescans requested the last one guaranteed to be finished.
     pub fn trigger_full_rescan_async(&self, shared_frecency: &SharedFrecency) -> Result<(), Error> {
-        match ScanJob::new(self, shared_frecency, /*install_watcher=*/ false)? {
+        self.trigger_full_rescan_with_reason(shared_frecency, RescanReason::Explicit)
+            .map(|_| ())
+    }
+
+    /// Returns admitted and throttled rescan requests by reason.
+    /// Counters start at picker creation or the last reset.
+    pub fn rescan_stats(&self) -> RescanStats {
+        self.0.rescans.snapshot()
+    }
+
+    pub fn reset_rescan_stats(&self) {
+        self.0.rescans.reset();
+    }
+
+    /// Returns `Ok(true)` when a rescan was started (or queued behind an
+    /// active scan) and `Ok(false)` when the request was throttled — the
+    /// caller must then fall back to incremental event processing.
+    pub(crate) fn trigger_full_rescan_with_reason(
+        &self,
+        shared_frecency: &SharedFrecency,
+        reason: RescanReason,
+    ) -> Result<bool, Error> {
+        // for giant folders we have no other choice other than throttling rescans
+        // if user is running application in millions of files with a ton of rescan events
+        // we drop / throttle some of requests to avoid constant burst of IO
+        if reason == RescanReason::Explicit {
+            self.0.rescan_throttle.note_explicit_scan();
+        } else if !self.check_rescan_throttle(reason) {
+            return Ok(false);
+        }
+
+        self.0.rescans.record(reason);
+
+        match ScanJob::new_rescan(self, shared_frecency)? {
             Some(job) => {
                 job.spawn();
             }
             None => {
-                // A scan is already in flight — mark a follow-up as
-                // needed. The running scan's `run()` drains this flag
-                // and reschedules itself.
+                // we can not abort the ongoing sync, but if the events
                 if let Ok(guard) = self.read()
                     && let Some(picker) = guard.as_ref()
                 {
@@ -181,38 +258,116 @@ impl SharedFilePicker {
                 }
             }
         }
-        Ok(())
+        Ok(true)
     }
 
-    /// Refresh git statuses for all indexed files.
+    fn check_rescan_throttle(&self, reason: RescanReason) -> bool {
+        let (live_files, has_git) = self
+            .read()
+            .ok()
+            .and_then(|guard| {
+                guard
+                    .as_ref()
+                    .map(|picker| (picker.live_file_count(), picker.has_git_repo()))
+            })
+            .unwrap_or((0, false));
+
+        if self.0.rescan_throttle.admit(live_files, has_git) {
+            return true;
+        }
+
+        self.0.rescans.record_throttled(reason);
+        tracing::debug!(%reason, live_files, "Rescan throttled, skipping");
+        false
+    }
+
+    /// Subscribe to filesystem changes matching `pattern`.
+    ///
+    /// Patterns may be base-relative globs (./ works), exact paths inside the indexed
+    /// tree, or existing directories. An empty pattern watches the whole tree.
+    ///
+    /// Events are debounced and submitted in batches per 100-ms window at most 128 events.
+    /// Gitignored and other ignored files are never triggering watcher.
+    pub fn watch(
+        &self,
+        pattern: &str,
+        options: WatchOptions,
+        callback: impl Fn(WatchId, &[WatchEvent]) + Send + Sync + 'static,
+    ) -> Result<WatchId, Error> {
+        let (base_path, has_watcher, watcher_ready) = {
+            let guard = self.read()?;
+            let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?;
+
+            (
+                picker.base_path().to_path_buf(),
+                picker.has_watcher(),
+                picker.is_watcher_ready(),
+            )
+        };
+
+        if !has_watcher {
+            return Err(Error::WatcherDisabled);
+        }
+        if !watcher_ready {
+            return Err(Error::WatcherNotReady);
+        }
+
+        self.0
+            .watchers
+            .subscribe(&base_path, pattern, options, Box::new(callback))
+    }
+
+    /// Remove a watch subscription. Returns `true` if the id was active.
+    pub fn unwatch(&self, id: WatchId) -> bool {
+        self.0.watchers.unsubscribe(id)
+    }
+
+    /// Return whether a watch subscription is active.
+    pub fn is_watch_active(&self, id: WatchId) -> bool {
+        self.0.watchers.contains(id)
+    }
+
+    /// Remove every subscription without waiting for an executing callback.
+    pub fn shutdown_watches(&self) {
+        self.0.watchers.shutdown();
+    }
+
+    /// Remove every subscription and wait for an executing callback.
+    /// When called by that callback, it does not wait on itself.
+    pub fn shutdown_watches_and_wait(&self) {
+        self.0.watchers.shutdown_and_wait();
+    }
+
+    pub(crate) fn rebase_watches(&self, base_path: &Path) {
+        self.0.watchers.rebase(base_path);
+    }
+
+    pub(crate) fn watch_registry(&self) -> &Arc<WatchRegistry> {
+        &self.0.watchers
+    }
+
+    /// Refresh git statuses for all indexed files
+    #[tracing::instrument(level = "info", skip_all)]
     pub fn refresh_git_status(&self, shared_frecency: &SharedFrecency) -> Result<usize, Error> {
         use tracing::debug;
 
         let git_status = {
-            let guard = self.read()?;
-            let Some(ref picker) = *guard else {
-                return Err(Error::FilePickerMissing);
+            let git_root = {
+                let guard = self.read()?;
+                let Some(ref picker) = *guard else {
+                    return Err(Error::FilePickerMissing);
+                };
+                picker.git_root().map(|p| p.to_path_buf())
             };
 
-            debug!(
-                "Refreshing git statuses for picker: {:?}",
-                picker.git_root()
-            );
+            debug!(?git_root, "Refreshing git status for picker");
 
-            // Wait briefly for any in-progress git operation to release
-            // its `.git/index.lock`. libgit2 reads `.git/index` directly
-            // and does NOT coordinate with the filesystem lock; if a
-            // writer is mid-atomic-rename (lock file exists, new index
-            // not yet swapped in), we would observe stale status data.
-            // This matters most for the background watcher, which
-            // typically fires refresh in response to the very events
-            // produced by that in-flight git write.
-            if let Some(root) = picker.git_root() {
+            if let Some(ref root) = git_root {
                 wait_for_git_index_lock_release(root);
             }
 
             GitStatusCache::read_git_status(
-                picker.git_root(),
+                git_root.as_deref(),
                 &mut crate::git::default_status_options(),
             )
         };
@@ -230,16 +385,62 @@ impl SharedFilePicker {
 
         Ok(statuses_count)
     }
+
+    /// Recompute and apply git status for a specific set of paths.
+    pub fn update_git_status_for_paths(
+        &self,
+        paths: &[PathBuf],
+        shared_frecency: &SharedFrecency,
+    ) -> Result<(), Error> {
+        if paths.is_empty() {
+            return Ok(());
+        }
+
+        let git_root = {
+            let guard = self.read()?;
+            let Some(ref picker) = *guard else {
+                return Err(Error::FilePickerMissing);
+            };
+            picker.git_root().map(|p| p.to_path_buf())
+        };
+        let Some(git_root) = git_root else {
+            return Ok(());
+        };
+
+        wait_for_git_index_lock_release(&git_root);
+
+        let repo = Repository::open(&git_root)?;
+        let status = GitStatusCache::git_status_for_paths(&repo, paths)?;
+
+        let mut guard = self.write()?;
+        let picker = guard.as_mut().ok_or(Error::FilePickerMissing)?;
+        picker.update_git_statuses(status, shared_frecency)
+    }
 }
 
-/// Thread-safe shared handle to the [`FrecencyTracker`] instance.
-#[derive(Clone)]
-pub struct SharedFrecency {
-    inner: Arc<RwLock<Option<FrecencyTracker>>>,
+/// Thread-safe shared handle to an LMDB-backed store. A disabled (`noop`)
+/// instance silently ignores writes. See the [`SharedFrecency`] and
+/// [`SharedQueryTracker`] aliases.
+///
+/// `LmdbStore` is intentionally crate-private, so the store type is sealed:
+/// only `FrecencyTracker` / `QueryTracker` can ever instantiate this.
+#[allow(private_bounds)]
+pub struct SharedDb<T: LmdbStore> {
+    inner: Arc<RwLock<Option<T>>>,
     enabled: bool,
 }
 
-impl Default for SharedFrecency {
+// Hand-written to avoid a spurious `T: Clone` bound — `Arc` is always `Clone`.
+impl<T: LmdbStore> Clone for SharedDb<T> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            enabled: self.enabled,
+        }
+    }
+}
+
+impl<T: LmdbStore> Default for SharedDb<T> {
     fn default() -> Self {
         Self {
             inner: Arc::new(RwLock::new(None)),
@@ -248,13 +449,14 @@ impl Default for SharedFrecency {
     }
 }
 
-impl std::fmt::Debug for SharedFrecency {
+impl<T: LmdbStore> std::fmt::Debug for SharedDb<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("SharedFrecency").field(&"..").finish()
+        f.debug_tuple("SharedDb").field(&T::LABEL).finish()
     }
 }
 
-impl SharedFrecency {
+#[allow(private_bounds)]
+impl<T: LmdbStore> SharedDb<T> {
     /// Creates a disabled instance that silently ignores all writes.
     pub fn noop() -> Self {
         Self {
@@ -263,46 +465,57 @@ impl SharedFrecency {
         }
     }
 
-    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<FrecencyTracker>>, Error> {
+    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<T>>, Error> {
         self.inner.read().map_err(|_| Error::AcquireFrecencyLock)
     }
 
-    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<FrecencyTracker>>, Error> {
+    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<T>>, Error> {
         self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
     }
 
-    /// Initialize the frecency tracker. No-op if this is a disabled instance.
-    pub fn init(&self, tracker: FrecencyTracker) -> Result<(), Error> {
+    /// Initialize the store + spawn GC in the background. No-op when disabled.
+    pub fn init(&self, tracker: T) -> Result<(), Error> {
         if !self.enabled {
             return Ok(());
         }
-        let mut guard = self.write()?;
-        *guard = Some(tracker);
-        Ok(())
-    }
 
-    /// Spawn a background GC thread for this frecency tracker.
-    pub fn spawn_gc(&self, db_path: String) -> crate::Result<std::thread::JoinHandle<()>> {
-        FrecencyTracker::spawn_gc(self.clone(), db_path)
+        {
+            let mut guard = self.write()?;
+            *guard = Some(tracker);
+        }
+
+        // GC holds a read guard on this lock, so destroy / re-init wait won't race
+        spawn_lmdb_gc(self.inner.clone());
+        Ok(())
     }
 
     /// Drop the in-memory tracker and delete the on-disk database directory.
     ///
-    /// Acquires the write lock, ensuring all readers (including any active mmap
-    /// access) are finished before the LMDB environment is closed and the files
-    /// are removed.
-    ///
-    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no
-    /// tracker was initialized.
+    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no tracker was initialized.
     pub fn destroy(&self) -> Result<Option<PathBuf>, Error> {
         let mut guard = self.write()?;
         let Some(tracker) = guard.take() else {
             return Ok(None);
         };
-        let db_path = tracker.db_path().to_path_buf();
+
+        let closing_event = match tracker.shared_env().destroy() {
+            Ok(closing) => closing,
+            Err(e) => {
+                *guard = Some(tracker);
+                return Err(e);
+            }
+        };
+
+        let db_path = tracker.env().path().to_path_buf();
         // Drop closes the LMDB env and unmaps the files
         drop(tracker);
         drop(guard);
+
+        // Deleting before mdb_env_close finishes would race the unmap.
+        if let Some(event) = closing_event {
+            event.wait_timeout(Duration::from_secs(5));
+        }
+
         std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir {
             path: db_path.clone(),
             source,
@@ -311,75 +524,8 @@ impl SharedFrecency {
     }
 }
 
+/// Thread-safe shared handle to the [`FrecencyTracker`] instance.
+pub type SharedFrecency = SharedDb<FrecencyTracker>;
+
 /// Thread-safe shared handle to the [`QueryTracker`] instance.
-#[derive(Clone)]
-pub struct SharedQueryTracker {
-    inner: Arc<RwLock<Option<QueryTracker>>>,
-    enabled: bool,
-}
-
-impl Default for SharedQueryTracker {
-    fn default() -> Self {
-        Self {
-            inner: Arc::new(RwLock::new(None)),
-            enabled: true,
-        }
-    }
-}
-
-impl std::fmt::Debug for SharedQueryTracker {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("SharedQueryTracker").field(&"..").finish()
-    }
-}
-
-impl SharedQueryTracker {
-    /// Creates a disabled instance that silently ignores all writes.
-    pub fn noop() -> Self {
-        Self {
-            inner: Arc::new(RwLock::new(None)),
-            enabled: false,
-        }
-    }
-
-    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<QueryTracker>>, Error> {
-        self.inner.read().map_err(|_| Error::AcquireFrecencyLock)
-    }
-
-    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<QueryTracker>>, Error> {
-        self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
-    }
-
-    /// Initialize the query tracker. No-op if this is a disabled instance.
-    pub fn init(&self, tracker: QueryTracker) -> Result<(), Error> {
-        if !self.enabled {
-            return Ok(());
-        }
-        let mut guard = self.write()?;
-        *guard = Some(tracker);
-        Ok(())
-    }
-
-    /// Drop the in-memory tracker and delete the on-disk database directory.
-    ///
-    /// Acquires the write lock, ensuring all readers (including any active mmap
-    /// access) are finished before the LMDB environment is closed and the files
-    /// are removed.
-    ///
-    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no
-    /// tracker was initialized.
-    pub fn destroy(&self) -> Result<Option<PathBuf>, Error> {
-        let mut guard = self.write()?;
-        let Some(tracker) = guard.take() else {
-            return Ok(None);
-        };
-        let db_path = tracker.db_path().to_path_buf();
-        drop(tracker);
-        drop(guard);
-        std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir {
-            path: db_path.clone(),
-            source,
-        })?;
-        Ok(Some(db_path))
-    }
-}
+pub type SharedQueryTracker = SharedDb<QueryTracker>;

crates/fff-core/src/simd_path.rs

@@ -60,7 +60,10 @@ impl std::fmt::Debug for SimdChunk {
     }
 }
 
-pub const PATH_BUF_SIZE: usize = 4096;
+pub use crate::constants::PATH_BUF_SIZE;
+
+/// Chunk pointer capacity needed for the longest path the platform allows.
+pub(crate) const MAX_PATH_CHUNKS: usize = PATH_BUF_SIZE.div_ceil(SIMD_CHUNK_BYTES);
 
 /// Indices into a shared `SimdChunk` arena representing a file path.
 ///
@@ -98,14 +101,10 @@ impl ChunkedString {
     }
 
     #[inline]
-    pub fn resolve_ptrs<'a>(
-        &self,
-        arena: ArenaPtr,
-        buf: &'a mut [*const u8; 32],
-    ) -> &'a [*const u8] {
-        let count = self.indices.len();
+    pub fn resolve_ptrs<'a>(&self, arena: ArenaPtr, buf: &'a mut [*const u8]) -> &'a [*const u8] {
+        let count = self.indices.len().min(buf.len());
         let base = arena.as_ptr();
-        for (i, &idx) in self.indices.iter().enumerate() {
+        for (i, &idx) in self.indices[..count].iter().enumerate() {
             buf[i] = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
         }
         &buf[..count]
@@ -295,10 +294,11 @@ pub(crate) struct ChunkedPathStoreBuilder {
 
 impl ChunkedPathStoreBuilder {
     pub fn new(estimated_files: usize) -> Self {
-        let est_chunks = estimated_files * 3;
+        let est_chunks = estimated_files * INLINE_CHUNKS; // we know that most of repos will fit
+        // most paths into 64 = 16 * INLINE_CHUNKS
         Self {
-            arena: Vec::with_capacity(est_chunks / 2),
-            chunk_dedup: AHashMap::with_capacity(est_chunks / 2),
+            arena: Vec::with_capacity(est_chunks),
+            chunk_dedup: AHashMap::with_capacity(est_chunks),
         }
     }
 
@@ -415,7 +415,10 @@ mod tests {
 
     #[test]
     fn test_chunked_string_full_path() {
-        let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);
+        let (store, strings, _files) = build_test_store(&[
+            "src/components/Button.tsx",
+            "src/components/Button.test.tsx",
+        ]);
         let arena = store.as_arena_ptr();
         let cs = &strings[0];
 
@@ -423,6 +426,15 @@ mod tests {
         assert_eq!(cs.read_to_buf(arena, &mut buf), "src/components/Button.tsx");
         assert_eq!(cs.byte_len, 25);
         assert_eq!(cs.filename_offset, 15);
+
+        let cs = &strings[1];
+        let mut buf = [0u8; 512];
+        assert_eq!(
+            cs.read_to_buf(arena, &mut buf),
+            "src/components/Button.test.tsx"
+        );
+        assert_eq!(cs.byte_len, 30);
+        assert_eq!(cs.filename_offset, 15);
     }
 
     #[test]
@@ -459,7 +471,7 @@ mod tests {
         let arena = store.as_arena_ptr();
         let cs = &strings[0];
 
-        let mut ptrs = [std::ptr::null::<u8>(); 32];
+        let mut ptrs = [std::ptr::null::<u8>(); MAX_PATH_CHUNKS];
         let resolved = cs.resolve_ptrs(arena, &mut ptrs);
         assert_eq!(resolved.len(), 2); // 25 bytes = 2 chunks
 
@@ -477,6 +489,43 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_resolve_ptrs_path_exceeding_512_bytes() {
+        // Regression: a fixed 32-ptr buffer covered only 512 bytes while
+        // PATH_BUF_SIZE (libc::PATH_MAX) allows longer paths, panicking with
+        // "index out of bounds: the len is 32 but the index is 32"
+        let mut path = String::new();
+        while path.len() < 600 {
+            path.push_str("deeply_nested_directory_segment/");
+        }
+        path.push_str("needle_file.rs");
+        assert!(path.len() > 512 && path.len() < PATH_BUF_SIZE);
+
+        let (store, strings, _files) = build_test_store(&[path.as_str()]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+        assert!(cs.chunk_count() > 32, "path must span more than 32 chunks");
+
+        let mut ptrs = [std::ptr::null::<u8>(); MAX_PATH_CHUNKS];
+        let resolved = cs.resolve_ptrs(arena, &mut ptrs);
+
+        // Truncation is not acceptable either: it silently drops the tail of
+        // the path (including the filename here) from fuzzy matching.
+        assert_eq!(
+            resolved.len(),
+            cs.chunk_count(),
+            "resolve_ptrs must resolve every chunk of a PATH_MAX-legal path"
+        );
+
+        let total = cs.byte_len as usize;
+        let mut reconstructed = Vec::with_capacity(total);
+        for (i, &ptr) in resolved.iter().enumerate() {
+            let take = SIMD_CHUNK_BYTES.min(total - i * SIMD_CHUNK_BYTES);
+            reconstructed.extend_from_slice(unsafe { std::slice::from_raw_parts(ptr, take) });
+        }
+        assert_eq!(std::str::from_utf8(&reconstructed).unwrap(), path);
+    }
+
     #[test]
     fn test_filename_cow_mid_chunk() {
         let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);

crates/fff-core/src/simd_string_utils/case.rs

@@ -0,0 +1,168 @@
+#[inline]
+pub fn ascii_swap_case(b: u8) -> u8 {
+    b ^ 0x20
+}
+
+#[inline]
+fn eq_lowered_scalar(h: *const u8, needle_lower: &[u8]) -> bool {
+    for (i, &n) in needle_lower.iter().enumerate() {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != n {
+            return false;
+        }
+    }
+    true
+}
+
+/// AVX2 only has a **signed** byte compare (`cmpgt`), but we need an
+/// **unsigned** range check (`'A' <= byte <= 'Z'`). XOR-ing every byte with
+/// `0x80` maps the unsigned range `[0, 255]` into the signed range
+/// `[-128, 127]` preserving order, so signed `cmpgt` becomes correct
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn eq_lowered_avx2(h: *const u8, needle_lower: &[u8]) -> bool {
+    use core::arch::x86_64::*;
+
+    let len = needle_lower.len();
+    let mut i = 0usize;
+
+    let flip = _mm256_set1_epi8(0x80u8 as i8);
+    let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8);
+    let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8);
+    let bit20 = _mm256_set1_epi8(0x20u8 as i8);
+
+    while i + 32 <= len {
+        let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) };
+        let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) };
+
+        // Signed-domain range check selects uppercase lanes, OR bit 5 folds them.
+        let x = _mm256_xor_si256(hv, flip);
+        let ge_a = _mm256_cmpgt_epi8(x, a_minus_1);
+        let le_z = _mm256_cmpgt_epi8(z_plus_1, x);
+        let upper = _mm256_and_si256(ge_a, le_z);
+        let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20));
+
+        let eq = _mm256_cmpeq_epi8(folded, nv);
+        if _mm256_movemask_epi8(eq) != -1i32 {
+            return false;
+        }
+
+        i += 32;
+    }
+
+    while i < len {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+/// Unsigned range checks (`vcge`/`vcle`) detect uppercase ASCII, bit 5 folds
+/// to lowercase, then equality is checked via udot: xors the folded haystack
+/// with the pre-lowered needle and dot-product the difference with itself
+/// any non-zero byte produces a non-zero u32 lane. udot is emitted via inline
+/// asm because `vdotq_u32` is still behind an unstable feature gate.
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon,dotprod")]
+unsafe fn eq_lowered_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool {
+    use core::arch::aarch64::*;
+
+    let len = needle_lower.len();
+    let mut i = 0usize;
+
+    let a_val = vdupq_n_u8(b'A');
+    let z_val = vdupq_n_u8(b'Z');
+    let bit20 = vdupq_n_u8(0x20);
+
+    while i + 16 <= len {
+        let hv = unsafe { vld1q_u8(h.add(i)) };
+        let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) };
+
+        let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val));
+        let folded = vorrq_u8(hv, vandq_u8(upper, bit20));
+        let xored = veorq_u8(folded, nv);
+
+        let dots: uint32x4_t;
+        let zero = vdupq_n_u32(0);
+        unsafe {
+            core::arch::asm!(
+                "udot {d:v}.4s, {a:v}.16b, {b:v}.16b",
+                d = inlateout(vreg) zero => dots,
+                a = in(vreg) xored,
+                b = in(vreg) xored,
+            );
+        }
+
+        if vmaxvq_u32(dots) != 0 {
+            return false;
+        }
+
+        i += 16;
+    }
+
+    while i < len {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+/// Case-insensitive equality of `needle_lower` against the haystack bytes
+/// starting at `h`. `needle_lower` must be pre-lowercased (ASCII).
+///
+/// # Safety
+/// `h` must be valid for reads of `needle_lower.len()` bytes.
+#[inline]
+pub(crate) unsafe fn eq_lowered_case(haystack: *const u8, needle_lower: &[u8]) -> bool {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") {
+            return unsafe { eq_lowered_avx2(haystack, needle_lower) };
+        }
+    }
+    #[cfg(target_arch = "aarch64")]
+    {
+        if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") {
+            return unsafe { eq_lowered_neon_dotprod(haystack, needle_lower) };
+        }
+    }
+
+    eq_lowered_scalar(haystack, needle_lower)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn eq_lowered(haystack: &[u8], needle_lower: &[u8]) -> bool {
+        assert!(haystack.len() >= needle_lower.len());
+        unsafe { eq_lowered_case(haystack.as_ptr(), needle_lower) }
+    }
+
+    #[test]
+    fn swap_case_toggles_letters() {
+        assert_eq!(ascii_swap_case(b'n'), b'N');
+        assert_eq!(ascii_swap_case(b'N'), b'n');
+        assert_eq!(ascii_swap_case(b'z'), b'Z');
+    }
+
+    #[test]
+    fn eq_matches_std_semantics() {
+        assert!(eq_lowered(b"Hello", b"hello"));
+        assert!(eq_lowered(b"HELLO WORLD", b"hello"));
+        assert!(!eq_lowered(b"Hellp", b"hello"));
+        // Non-letters must not fold: '[' (0x5B) vs '{' (0x7B) differ only in bit 5.
+        assert!(!eq_lowered(b"A[", b"a{"));
+        assert!(eq_lowered(b"A{", b"a{"));
+        // Long inputs exercise the SIMD kernels.
+        let hay = b"INT STRUCT MUTEX *LOCK(STRUCT MUTEX *LOCK) { RETURN 0; }";
+        let needle: Vec<u8> = hay.iter().map(|b| b.to_ascii_lowercase()).collect();
+        assert!(eq_lowered(hay, &needle));
+        let mut bad = needle.clone();
+        *bad.last_mut().unwrap() = b'!';
+        assert!(!eq_lowered(hay, &bad));
+    }
+}

crates/fff-core/src/simd_string_utils/memmem.rs

@@ -0,0 +1,494 @@
+use super::case::{ascii_swap_case, eq_lowered_case};
+use smallvec::SmallVec;
+
+// Byte frequency table stolen from memchr
+const BYTE_FREQUENCIES: [u8; 256] = [
+    55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00
+    42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10
+    255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20
+    208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30
+    120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185,
+    167, // 0x40 A-O
+    186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114,
+    223, // 0x50 P-_
+    151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246,
+    244, // 0x60 a-o
+    231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127,
+    27, // 0x70 p-DEL
+    212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80
+    207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90
+    118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0
+    166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0
+];
+
+#[inline]
+fn rank(lower: u8) -> u8 {
+    if lower.is_ascii_lowercase() {
+        BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[ascii_swap_case(lower) as usize])
+    } else {
+        BYTE_FREQUENCIES[lower as usize]
+    }
+}
+
+/// Pick two needle positions with the rarest bytes (case-insensitive)
+fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) {
+    debug_assert!(needle_lower.len() >= 2);
+
+    let mut best1 = (u8::MAX, 0usize); // (rank, position)
+    let mut best2 = (u8::MAX, 1usize);
+
+    for (i, &b) in needle_lower.iter().enumerate() {
+        let r = rank(b);
+        if r < best1.0 {
+            best2 = best1;
+            best1 = (r, i);
+        } else if r < best2.0 && i != best1.1 {
+            best2 = (r, i);
+        }
+    }
+
+    let i1 = best1.1.min(best2.1);
+    let i2 = best1.1.max(best2.1);
+    (i1, i2)
+}
+
+/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF)
+/// Bit *i* of the result corresponds to byte *i* of the input vector
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+#[inline]
+unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 {
+    use core::arch::aarch64::*;
+
+    // AND each byte with its bit-position mask, then horizontally sum each half.
+    static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
+    let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) };
+    let masked = vandq_u8(v, bit_mask);
+    let lo = vaddv_u8(vget_low_u8(masked));
+    let hi = vaddv_u8(vget_high_u8(masked));
+    (lo as u16) | ((hi as u16) << 8)
+}
+
+/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration,
+/// checking two rare bytes (case-insensitive) simultaneously
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn find_packed_pair_avx2(
+    haystack: &[u8],
+    needle_lower: &[u8],
+    i1: usize,
+    i2: usize,
+) -> Option<usize> {
+    use core::arch::x86_64::*;
+
+    let n = needle_lower.len();
+    let hlen = haystack.len();
+    let ptr = haystack.as_ptr();
+    let last_start = hlen - n; // last valid match-start position
+
+    let b1 = needle_lower[i1];
+    let b1_alt = if b1.is_ascii_lowercase() {
+        ascii_swap_case(b1)
+    } else {
+        b1
+    };
+    let b2 = needle_lower[i2];
+    let b2_alt = if b2.is_ascii_lowercase() {
+        ascii_swap_case(b2)
+    } else {
+        b2
+    };
+
+    let v1_lo = _mm256_set1_epi8(b1 as i8);
+    let v1_hi = _mm256_set1_epi8(b1_alt as i8);
+    let v2_lo = _mm256_set1_epi8(b2 as i8);
+    let v2_hi = _mm256_set1_epi8(b2_alt as i8);
+
+    // Loads come from ptr+offset+i1 and ptr+offset+i2, so we need offset + max(i1,i2) + 31 < hlen.
+    let max_idx = i1.max(i2);
+    let max_offset = hlen.saturating_sub(max_idx + 32);
+    let mut offset = 0usize;
+
+    while offset <= max_offset {
+        let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) };
+        let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) };
+
+        // Case-insensitive match: OR both case variants, then AND the two positions.
+        let eq1 = _mm256_or_si256(
+            _mm256_cmpeq_epi8(chunk1, v1_lo),
+            _mm256_cmpeq_epi8(chunk1, v1_hi),
+        );
+        let eq2 = _mm256_or_si256(
+            _mm256_cmpeq_epi8(chunk2, v2_lo),
+            _mm256_cmpeq_epi8(chunk2, v2_hi),
+        );
+
+        let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32;
+
+        // Candidates are visited in increasing position order, so the first
+        // verified candidate is the leftmost match
+        while mask != 0 {
+            let bit = mask.trailing_zeros() as usize;
+            let candidate = offset + bit;
+            if candidate > last_start {
+                return None;
+            }
+            if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                return Some(candidate);
+            }
+            mask &= mask - 1;
+        }
+
+        offset += 32;
+    }
+
+    // handle remaining characters
+    if offset <= last_start {
+        let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) {
+            i1
+        } else {
+            i2
+        };
+        let rare_byte = needle_lower[rare_pos];
+        let tail_start = offset + rare_pos;
+        let tail_end = last_start + rare_pos + 1;
+        if tail_start < tail_end {
+            let tail_space = &haystack[tail_start..tail_end];
+            if rare_byte.is_ascii_lowercase() {
+                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            } else {
+                for pos in memchr::memchr_iter(rare_byte, tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+unsafe fn find_packed_pair_neon(
+    haystack: &[u8],
+    needle_lower: &[u8],
+    i1: usize,
+    i2: usize,
+) -> Option<usize> {
+    use core::arch::aarch64::*;
+
+    let n = needle_lower.len();
+    let hlen = haystack.len();
+    let ptr = haystack.as_ptr();
+    let last_start = hlen - n;
+
+    let b1 = needle_lower[i1];
+    let b1_alt = if b1.is_ascii_lowercase() {
+        ascii_swap_case(b1)
+    } else {
+        b1
+    };
+    let b2 = needle_lower[i2];
+    let b2_alt = if b2.is_ascii_lowercase() {
+        ascii_swap_case(b2)
+    } else {
+        b2
+    };
+
+    let v1_lo = vdupq_n_u8(b1);
+    let v1_hi = vdupq_n_u8(b1_alt);
+    let v2_lo = vdupq_n_u8(b2);
+    let v2_hi = vdupq_n_u8(b2_alt);
+
+    let max_idx = i1.max(i2);
+    let max_offset = hlen.saturating_sub(max_idx + 16);
+    let mut offset = 0usize;
+
+    while offset <= max_offset {
+        let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) };
+        let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) };
+
+        // Case-insensitive match: OR both case variants, then AND the two positions.
+        let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi));
+        let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi));
+
+        let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) };
+
+        while mask != 0 {
+            let bit = mask.trailing_zeros() as usize;
+            let candidate = offset + bit;
+            if candidate > last_start {
+                return None;
+            }
+            if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                return Some(candidate);
+            }
+            mask &= mask - 1;
+        }
+
+        offset += 16;
+    }
+
+    // Tail: remaining positions that couldn't fill a full vector.
+    if offset <= last_start {
+        let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) {
+            i1
+        } else {
+            i2
+        };
+        let rare_byte = needle_lower[rare_pos];
+        let tail_start = offset + rare_pos;
+        let tail_end = last_start + rare_pos + 1;
+        if tail_start < tail_end {
+            let tail_space = &haystack[tail_start..tail_end];
+            if rare_byte.is_ascii_lowercase() {
+                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            } else {
+                for pos in memchr::memchr_iter(rare_byte, tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+fn find_first_byte_with_memchr(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+    let n = needle_lower.len();
+    debug_assert!(n >= 1 && n <= haystack.len());
+
+    let search_space = &haystack[..=haystack.len() - n];
+    let first = needle_lower[0];
+
+    if first.is_ascii_lowercase() {
+        let alt = ascii_swap_case(first);
+        for pos in memchr::memchr2_iter(first, alt, search_space) {
+            if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } {
+                return Some(pos);
+            }
+        }
+    } else {
+        for pos in memchr::memchr_iter(first, search_space) {
+            if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } {
+                return Some(pos);
+            }
+        }
+    }
+    None
+}
+
+/// ASCII case-insensitive substring search returning the leftmost match
+/// position. `needle_lower` must be pre-lowercased (ASCII).
+// pub because it is used in out of the crate benchmarks
+#[doc(hidden)] // it's pub only for benches
+pub fn find(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+    let n = needle_lower.len();
+    if n == 0 {
+        return Some(0);
+    }
+    if n > haystack.len() {
+        return None;
+    }
+
+    if n == 1 {
+        let first = needle_lower[0];
+        return if first.is_ascii_lowercase() {
+            memchr::memchr2(first, ascii_swap_case(first), haystack)
+        } else {
+            memchr::memchr(first, haystack)
+        };
+    }
+
+    #[cfg_attr(
+        not(any(target_arch = "x86_64", target_arch = "aarch64")),
+        allow(unused_variables)
+    )]
+    let (i1, i2) = select_rare_pair(needle_lower);
+
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::is_x86_feature_detected!("avx2") {
+            // Need enough haystack for at least one vector load.
+            let max_idx = i1.max(i2);
+            if haystack.len() >= max_idx + 32 {
+                return unsafe { find_packed_pair_avx2(haystack, needle_lower, i1, i2) };
+            }
+        }
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    {
+        // Packed-pair wins when the first byte is common (memchr2 drowns in
+        // false positives), but a rare first byte (z, q, x, ...) makes
+        // memchr2's raw throughput dominate. Threshold 200 on the frequency
+        // table splits common letters (s=243, e=253) from rare ones (z=152).
+        let first_byte_rank = rank(needle_lower[0]);
+        let max_idx = i1.max(i2);
+        if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 {
+            return unsafe { find_packed_pair_neon(haystack, needle_lower, i1, i2) };
+        }
+    }
+
+    // fallbacks to memchr based implementation cause we still have it and it supports more SIMD backends
+    // TODO convert all the supported backend by memchr and get rid of the fallback
+    find_first_byte_with_memchr(haystack, needle_lower)
+}
+
+/// A case insensitive find that works better with smaller strings, doesn't unwrap a complicated
+/// AVX backend we use for grep because only cpu flags check takes usually more time than find itself
+pub fn find_case_insensitive_short(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+    debug_assert!(haystack.len() < 1024);
+    let mut needle_lower: SmallVec<[u8; 64]> = SmallVec::from_slice(needle);
+    needle_lower.make_ascii_lowercase();
+
+    find(haystack, &needle_lower)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn reference_find(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+        if needle_lower.is_empty() {
+            return Some(0);
+        }
+        if needle_lower.len() > haystack.len() {
+            return None;
+        }
+        haystack
+            .windows(needle_lower.len())
+            .position(|w| w.eq_ignore_ascii_case(needle_lower))
+    }
+
+    #[test]
+    fn basic_case_insensitive() {
+        assert_eq!(find(b"Hello World", b"hello"), Some(0));
+        assert_eq!(find(b"Hello World", b"world"), Some(6));
+        assert_eq!(find(b"NOMORE bugs", b"nomore"), Some(0));
+        assert_eq!(find(b"Hello World", b"xyz"), None);
+        assert!(find(b"Hello World", b"o w").is_some());
+    }
+
+    #[test]
+    fn edge_cases() {
+        assert_eq!(find(b"ab", b"ab"), Some(0));
+        assert_eq!(find(b"AB", b"ab"), Some(0));
+        assert_eq!(find(b"a", b"ab"), None);
+        assert_eq!(find(b"anything", b""), Some(0));
+        assert_eq!(find(b"", b"x"), None);
+        assert_eq!(find(b"xxA", b"a"), Some(2));
+        assert_eq!(find(b"xx:", b":"), Some(2));
+    }
+
+    #[test]
+    fn returns_leftmost_match() {
+        assert_eq!(find(b"foo FOO foo", b"foo"), Some(0));
+        let mut big = vec![b'.'; 300];
+        big[100..103].copy_from_slice(b"FoO");
+        big[200..203].copy_from_slice(b"foo");
+        assert_eq!(find(&big, b"foo"), Some(100));
+    }
+
+    #[test]
+    fn non_letter_bytes_do_not_case_fold() {
+        // '[' (0x5B) and '{' (0x7B) differ only in bit 5 but are not letters.
+        // A fold implemented as a bare `| 0x20` would falsely match these.
+        assert_eq!(find(b"A[", b"a{"), None);
+        assert_eq!(find(b"x@y", b"x`y"), None);
+        assert_eq!(find(b"a]b", b"a}b"), None);
+        assert_eq!(find(b"A{", b"a{"), Some(0));
+    }
+
+    #[test]
+    fn matches_reference_on_random_inputs() {
+        // Deterministic xorshift PRNG — no external deps.
+        let mut state = 0x9E3779B97F4A7C15u64;
+        let mut next = move || {
+            state ^= state << 13;
+            state ^= state >> 7;
+            state ^= state << 17;
+            state
+        };
+
+        // Alphabet with letters, both-case pairs, and 0x20-differing symbols.
+        let alphabet = b"aAbBzZ [{@`]}^~_0.\n";
+        for _ in 0..2000 {
+            let hlen = (next() % 200) as usize;
+            let nlen = (next() % 8) as usize;
+            let haystack: Vec<u8> = (0..hlen)
+                .map(|_| alphabet[(next() % alphabet.len() as u64) as usize])
+                .collect();
+            let needle: Vec<u8> = (0..nlen)
+                .map(|_| alphabet[(next() % alphabet.len() as u64) as usize].to_ascii_lowercase())
+                .collect();
+
+            assert_eq!(
+                find(&haystack, &needle),
+                reference_find(&haystack, &needle),
+                "mismatch for haystack={:?} needle={:?}",
+                haystack,
+                needle,
+            );
+        }
+    }
+
+    #[test]
+    fn long_haystack_simd_paths() {
+        let haystack =
+            b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb";
+        assert_eq!(find(haystack, b"this_is_a_long_needle_test"), Some(32));
+        assert_eq!(find(haystack, b"this_is_a_long_needle_testz"), None);
+
+        // Needle >= 16 bytes exercises SIMD verify.
+        let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }";
+        assert_eq!(find(haystack2, b"struct mutex *lock"), Some(4));
+
+        let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        assert_eq!(find(upper_hay, b"qrstuvwxyz0123456789a"), Some(16));
+        assert_eq!(find(upper_hay, b"qrstuvwxyz01234567899"), None);
+
+        // Needle at very end / very start.
+        let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me";
+        assert_eq!(find(end_hay, b"find_me"), Some(end_hay.len() - 7));
+        assert_eq!(find(end_hay, b"xx"), Some(0));
+
+        // 1KB haystack with needle near the end.
+        let mut big = vec![b'z'; 1024];
+        big[1000..1010].copy_from_slice(b"hElLo_WoRl");
+        assert_eq!(find(&big, b"hello_wo"), Some(1000));
+        assert_eq!(find(&big, b"hello_world"), None);
+    }
+
+    #[test]
+    fn rare_pair_selection() {
+        let (i1, i2) = select_rare_pair(b"nomore");
+        let ranks: Vec<u8> = b"nomore".iter().map(|&b| rank(b)).collect();
+        let (r1, r2) = (ranks[i1], ranks[i2]);
+        for (i, &r) in ranks.iter().enumerate() {
+            if i != i1 && i != i2 {
+                assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal");
+            }
+        }
+    }
+}

crates/fff-core/src/simd_string_utils/mod.rs

@@ -0,0 +1,5 @@
+//! SIMD-accelerated string utilities: case flipping/folding and
+//! case-insensitive substring search.
+
+pub mod case;
+pub mod memmem;

crates/fff-core/src/stable_vec.rs

@@ -0,0 +1,177 @@
+use std::alloc::{self, Layout};
+use std::ptr::NonNull;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// Vector that guarantees no re-alloc happening at runtime
+pub(crate) struct StableVec<T> {
+    inner: Arc<StableBuf<T>>,
+}
+
+struct StableBuf<T> {
+    ptr: NonNull<T>,
+    cap: usize,
+    /// Atomic because:
+    ///   1. `push(&self)` must mutate this through a shared `&StableBuf`,
+    ///      which requires interior mutability.
+    ///   2. Arc clones (e.g. post-scan snapshots) read `len` outside the
+    ///      picker lock, concurrent with an appending writer. Acquire/Release
+    ///      on len is what makes "observed len ⇒ element bytes initialized"
+    ///      actually hold.
+    ///
+    /// Arc wrapping only shares ownership of the buffer; it does NOT
+    /// synchronize access to fields inside the shared buffer.
+    len: AtomicUsize,
+}
+
+// SAFETY: StableBuf is a thread-safe container when T is send + sync
+// There is another application level constraint: mutations are safe
+// when they are atomic updates, not read + update.
+unsafe impl<T: Send> Send for StableBuf<T> {}
+unsafe impl<T: Sync> Sync for StableBuf<T> {}
+
+impl<T> Drop for StableBuf<T> {
+    fn drop(&mut self) {
+        let len = *self.len.get_mut();
+        unsafe {
+            std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), len));
+            if self.cap > 0 {
+                let layout = Layout::array::<T>(self.cap).expect("layout");
+                alloc::dealloc(self.ptr.as_ptr().cast(), layout);
+            }
+        }
+    }
+}
+
+impl<T> StableVec<T> {
+    pub fn from_vec_with_reserve(mut vec: Vec<T>, extra: usize) -> Self {
+        vec.reserve(extra);
+        let cap = vec.capacity();
+        let len = vec.len();
+
+        let inner = if cap == 0 {
+            StableBuf {
+                ptr: NonNull::dangling(),
+                cap: 0,
+                len: AtomicUsize::new(0),
+            }
+        } else {
+            // Take ownership of the Vec's buffer without running element
+            // drops; we hand them off to the StableBuf.
+            let mut vec = std::mem::ManuallyDrop::new(vec);
+            let ptr = NonNull::new(vec.as_mut_ptr()).expect("non-null");
+            StableBuf {
+                ptr,
+                cap,
+                len: AtomicUsize::new(len),
+            }
+        };
+
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+
+    /// Append. Returns `false` if capacity is exhausted (item dropped).
+    ///
+    /// Safe to call via `&self` as long as the caller holds the outer
+    /// picker write lock (single-writer invariant).
+    #[inline]
+    pub fn push(&self, item: T) -> bool {
+        let cap = self.inner.cap;
+        let len = self.inner.len.load(Ordering::Acquire);
+        if len >= cap {
+            debug_assert!(
+                false,
+                "StableVec: push would exceed capacity ({len} at capacity {cap})"
+            );
+            tracing::error!(
+                len,
+                capacity = cap,
+                "StableVec: capacity exhausted — dropping item to prevent reallocation"
+            );
+            return false;
+        }
+        unsafe {
+            std::ptr::write(self.inner.ptr.as_ptr().add(len), item);
+        }
+        self.inner.len.store(len + 1, Ordering::Release);
+        true
+    }
+
+    // this method is specifically private because you probably need to use
+    // live_count if you are trying to access this method
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.inner.len.load(Ordering::Acquire)
+    }
+
+    /// Mutable element access for in-place field updates. Never shifts.
+    ///
+    /// LATENT UB: produces `&mut T` aliasing Arc-shared memory; the
+    /// `&mut self` on StableVec does NOT imply unique access to the
+    /// `StableBuf` when sibling Arc clones exist. Safe in practice
+    /// because callers hold the picker write lock and writes target
+    /// disjoint fields, but strictly forbidden by the aliasing model.
+    #[inline]
+    pub fn get_mut(&mut self, index: usize) -> Option<&mut T> {
+        let len = self.inner.len.load(Ordering::Acquire);
+        if index >= len {
+            return None;
+        }
+        unsafe { Some(&mut *self.inner.ptr.as_ptr().add(index)) }
+    }
+
+    #[inline]
+    pub fn last(&self) -> Option<&T> {
+        let len = self.len();
+        if len == 0 {
+            None
+        } else {
+            unsafe { Some(&*self.inner.ptr.as_ptr().add(len - 1)) }
+        }
+    }
+
+    /// Iterate mutably for in-place field updates. Never shifts storage.
+    /// Same latent-UB caveat as [`get_mut`]: `&mut T` into Arc-shared memory.
+    #[inline]
+    pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> {
+        let len = self.inner.len.load(Ordering::Acquire);
+        unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len).iter_mut() }
+    }
+}
+
+impl<T> Clone for StableVec<T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        Self {
+            inner: Arc::clone(&self.inner),
+        }
+    }
+}
+
+impl<T: std::fmt::Debug> std::fmt::Debug for StableVec<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("StableVec").field(&self.len()).finish()
+    }
+}
+
+impl<T> std::ops::Deref for StableVec<T> {
+    type Target = [T];
+    #[inline]
+    fn deref(&self) -> &[T] {
+        let len = self.len();
+        unsafe { std::slice::from_raw_parts(self.inner.ptr.as_ptr(), len) }
+    }
+}
+
+impl<T> std::ops::DerefMut for StableVec<T> {
+    /// LATENT UB: `&mut [T]` aliases Arc-shared memory. Kept for
+    /// Index/IndexMut ergonomics at call sites that write disjoint
+    /// fields under the picker write lock. See module-level doc.
+    #[inline]
+    fn deref_mut(&mut self) -> &mut [T] {
+        let len = self.inner.len.load(Ordering::Acquire);
+        unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len) }
+    }
+}

crates/fff-core/src/types.rs

@@ -1,11 +1,15 @@
 use std::io::Read;
 use std::path::{Path, PathBuf};
+#[cfg(not(target_os = "windows"))]
 use std::sync::OnceLock;
-use std::sync::atomic::{AtomicI32, AtomicU64, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicI32, AtomicU8, AtomicU64, AtomicUsize, Ordering};
 
-use crate::constraints::Constrainable;
+#[cfg(not(target_os = "windows"))]
+use crate::constants::{FRESH_MMAP_THRESHOLD, MMAP_THRESHOLD};
+use crate::constants::{MAX_CACHED_CONTENT_BYTES, MAX_FFFILE_SIZE, PATH_BUF_SIZE};
+use crate::index::constraints::Constrainable;
 use crate::query_tracker::QueryMatchEntry;
-use crate::simd_path::{ArenaPtr, PATH_BUF_SIZE};
+use crate::simd_path::ArenaPtr;
 use fff_query_parser::{FFFQuery, FuzzyQuery, Location};
 
 /// Different sources of the string storage used by FFF
@@ -38,30 +42,14 @@ impl FFFStringStorage for ArenaPtr {
     }
 }
 
-/// Cached file contents — mmap on Unix, heap buffer on Windows.
-///
-/// On Windows, memory-mapped files hold the file handle open and prevent
-/// editors from saving (writing/replacing) those files. Reading into a
-/// `Vec<u8>` releases the handle immediately after the read completes.
-///
-/// The `Buffer` variant is also used on Unix for temporary (uncached) reads
-/// where the mmap/munmap syscall overhead exceeds the cost of a heap copy.
-#[derive(Debug)]
-#[allow(dead_code)] // variants are conditionally used per platform
-pub enum FileContent {
-    #[cfg(not(target_os = "windows"))]
-    Mmap(memmap2::Mmap),
-    Buffer(Vec<u8>),
+pub trait FileSliceExt {
+    fn live_count(&self) -> usize;
 }
 
-impl std::ops::Deref for FileContent {
-    type Target = [u8];
-    fn deref(&self) -> &[u8] {
-        match self {
-            #[cfg(not(target_os = "windows"))]
-            FileContent::Mmap(m) => m,
-            FileContent::Buffer(b) => b,
-        }
+impl FileSliceExt for [FileItem] {
+    #[inline]
+    fn live_count(&self) -> usize {
+        self.iter().filter(|f| !f.is_deleted()).count()
     }
 }
 
@@ -81,6 +69,7 @@ pub struct DirFlags;
 
 impl DirFlags {
     pub const OVERFLOW: u8 = 1 << 0;
+    pub const DELETED: u8 = 1 << 1;
 }
 
 /// A directory in the file index. Shares chunk arena with file paths.
@@ -110,7 +99,25 @@ impl Clone for DirItem {
 impl DirItem {
     #[inline(always)]
     pub fn is_overflow(&self) -> bool {
-        self.flags & DirFlags::OVERFLOW == 0
+        self.flags & DirFlags::OVERFLOW != 0
+    }
+
+    #[inline(always)]
+    pub fn is_deleted(&self) -> bool {
+        self.flags & DirFlags::DELETED != 0
+    }
+
+    /// Marks the dir deleted/restored. Returns `true` when the state changed.
+    pub(crate) fn set_deleted(&mut self, deleted: bool) -> bool {
+        if self.is_deleted() == deleted {
+            return false;
+        }
+        if deleted {
+            self.flags |= DirFlags::DELETED;
+        } else {
+            self.flags &= !DirFlags::DELETED;
+        }
+        true
     }
 
     pub(crate) fn new(path: crate::simd_path::ChunkedString, last_segment_offset: u16) -> Self {
@@ -122,6 +129,19 @@ impl DirItem {
         }
     }
 
+    /// A dir appended after the initial scan; its path lives in the overflow arena.
+    pub(crate) fn new_overflow(
+        path: crate::simd_path::ChunkedString,
+        last_segment_offset: u16,
+    ) -> Self {
+        Self {
+            path,
+            flags: DirFlags::OVERFLOW,
+            last_segment_offset,
+            max_access_frecency: AtomicI32::new(0),
+        }
+    }
+
     /// Byte offset of the last path segment within the directory path.
     #[inline]
     pub fn last_segment_offset(&self) -> u16 {
@@ -217,6 +237,11 @@ impl Constrainable for DirItem {
     fn git_status(&self) -> Option<git2::Status> {
         None
     }
+
+    #[inline]
+    fn is_overflow(&self) -> bool {
+        DirItem::is_overflow(self)
+    }
 }
 
 #[derive(Debug)]
@@ -227,28 +252,43 @@ pub struct FileItem {
     pub modification_frecency_score: i16,
     pub git_status: Option<git2::Status>,
     pub(crate) path: crate::simd_path::ChunkedString,
-    parent_dir: u32,
-    flags: u8,
-    content: OnceLock<FileContent>,
+    pub(crate) parent_dir_index: u32,
+    flags: AtomicU8,
+    /// Lazy mmap cache. Only populated by the actual file read, controlled by the budget.
+    #[cfg(not(target_os = "windows"))]
+    content: OnceLock<memmap2::Mmap>,
 }
 
 impl Clone for FileItem {
     fn clone(&self) -> Self {
         Self {
             path: self.path.clone(),
-            parent_dir: self.parent_dir,
+            parent_dir_index: self.parent_dir_index,
             size: self.size,
             modified: self.modified,
             access_frecency_score: self.access_frecency_score,
             modification_frecency_score: self.modification_frecency_score,
             git_status: self.git_status,
-            flags: self.flags,
+            flags: AtomicU8::new(self.flags.load(Ordering::Relaxed)),
             // on clone we have to reset the content lock
+            #[cfg(not(target_os = "windows"))]
             content: OnceLock::new(),
         }
     }
 }
 
+/// Single-block read used by the binary classifier. Most binaries reveal a
+/// NUL byte within the first filesystem block, so 16 KB lets one read settle
+/// the classification for typical files while keeping the scratch buffer
+/// small enough to live on the stack.
+pub const BINARY_CLASSIFICATION_CHUNK_SIZE: usize = 16 * 1024;
+
+/// A file is treated as binary if any NUL byte appears in the scanned prefix.
+#[inline]
+pub(crate) fn detect_binary_content(content: &[u8]) -> bool {
+    memchr::memchr(0, content).is_some()
+}
+
 impl FileItem {
     pub fn new_raw(
         filename_start: u16,
@@ -267,13 +307,14 @@ impl FileItem {
 
         Self {
             path,
-            parent_dir: u32::MAX,
+            parent_dir_index: u32::MAX,
             size,
             modified,
             access_frecency_score: 0,
             modification_frecency_score: 0,
             git_status,
-            flags,
+            flags: AtomicU8::new(flags),
+            #[cfg(not(target_os = "windows"))]
             content: OnceLock::new(),
         }
     }
@@ -289,14 +330,6 @@ impl FileItem {
         self.path = path;
     }
 
-    pub(crate) fn parent_dir_index(&self) -> u32 {
-        self.parent_dir
-    }
-
-    pub(crate) fn set_parent_dir(&mut self, idx: u32) {
-        self.parent_dir = idx;
-    }
-
     pub fn dir_str(&self, arena: impl FFFStringStorage) -> String {
         let mut s = String::with_capacity(64);
         self.path.write_dir_to(arena.arena_for(self), &mut s);
@@ -371,7 +404,12 @@ impl FileItem {
 
         let base_end_idx = base_len + sep_len;
         let relative_portion_str = self.path.read_to_buf(arena, &mut buf[base_end_idx..]);
-        let total = base_end_idx + relative_portion_str.len();
+        let rel_len = relative_portion_str.len();
+        let total = base_end_idx + rel_len;
+        // Stored relative paths are '/'-canonical; rewrite to the OS-native
+        // separator so the result matches git-cache keys, the frecency DB, and
+        // Win32 file APIs. No-op off Windows.
+        crate::path_utils::nativize_slashes_in_place(&mut buf[base_end_idx..total]);
         Path::new(unsafe { std::str::from_utf8_unchecked(&buf[..total]) })
     }
 
@@ -401,56 +439,198 @@ impl FileItem {
         self.access_frecency_score as i32 + self.modification_frecency_score as i32
     }
 
+    #[allow(dead_code)]
     #[inline]
-    pub fn is_binary(&self) -> bool {
-        self.flags & FileItemFlags::BINARY != 0
+    pub(crate) fn is_likely_hot(&self) -> bool {
+        self.access_frecency_score > 0 || self.git_status.is_some()
+    }
+
+    /// Reads a fixed bytes count from the file optimized for quick speed of opening
+    #[inline]
+    pub(crate) fn read_trimmed_into_buf(
+        &self,
+        base_fd: i32,
+        base_path: &Path,
+        arena: ArenaPtr,
+        path_buf: &mut [u8; PATH_BUF_SIZE],
+        buf: &mut [u8],
+    ) -> usize {
+        #[cfg(unix)]
+        {
+            self.read_into_buf_unix(base_fd, base_path, arena, path_buf, buf)
+        }
+        #[cfg(not(unix))]
+        {
+            let _ = base_fd;
+            self.read_into_buf_std(base_path, arena, path_buf, buf)
+        }
+    }
+
+    #[cfg(unix)]
+    fn read_into_buf_unix(
+        &self,
+        base_fd: libc::c_int,
+        base_path: &Path,
+        arena: ArenaPtr,
+        path_buf: &mut [u8; PATH_BUF_SIZE],
+        buf: &mut [u8],
+    ) -> usize {
+        let fd = if base_fd >= 0 {
+            let relative_path = self.write_relative_cstr(arena, path_buf);
+            // SAFETY: `relative_path` is NUL-terminated, `base_fd` is a
+            // valid directory descriptor owned by the caller.
+            unsafe { libc::openat(base_fd, relative_path.as_ptr(), libc::O_RDONLY) }
+        } else {
+            use std::os::unix::io::IntoRawFd;
+            let abs = self.write_absolute_path(arena, base_path, path_buf);
+            match std::fs::File::open(abs) {
+                Ok(f) => f.into_raw_fd(),
+                Err(e) => {
+                    tracing::error!(?e, "Failed to fopen file");
+                    return 0;
+                }
+            }
+        };
+        if fd < 0 {
+            return 0;
+        }
+
+        let mut filled = 0usize;
+        while filled < buf.len() {
+            // SAFETY: `fd` is an owned descriptor, `buf[filled..]` is a
+            // valid writable slice for `buf.len() - filled` bytes.
+            let n = unsafe {
+                libc::read(
+                    fd,
+                    buf[filled..].as_mut_ptr() as *mut libc::c_void,
+                    (buf.len() - filled) as libc::size_t,
+                )
+            };
+            if n <= 0 {
+                break;
+            }
+            filled += n as usize;
+        }
+
+        // SAFETY: matching close for the owned descriptor.
+        unsafe { libc::close(fd) };
+        filled
+    }
+
+    #[cfg(not(unix))]
+    fn read_into_buf_std(
+        &self,
+        base_path: &Path,
+        arena: ArenaPtr,
+        path_buf: &mut [u8; PATH_BUF_SIZE],
+        buf: &mut [u8],
+    ) -> usize {
+        let abs = self.write_absolute_path(arena, base_path, path_buf);
+        let Ok(mut f) = std::fs::File::open(abs) else {
+            return 0;
+        };
+        let mut filled = 0usize;
+        while filled < buf.len() {
+            match f.read(&mut buf[filled..]) {
+                Ok(0) => break,
+                Ok(n) => filled += n,
+                Err(_) => return 0,
+            }
+        }
+        filled
     }
 
     #[inline]
-    pub fn set_binary(&mut self, val: bool) {
+    pub fn is_binary(&self) -> bool {
+        self.flags.load(Ordering::Relaxed) & FileItemFlags::BINARY != 0
+    }
+
+    #[inline]
+    pub fn set_binary(&self, val: bool) {
         if val {
-            self.flags |= FileItemFlags::BINARY;
+            self.flags
+                .fetch_or(FileItemFlags::BINARY, Ordering::Relaxed);
         } else {
-            self.flags &= !FileItemFlags::BINARY;
+            self.flags
+                .fetch_and(!FileItemFlags::BINARY, Ordering::Relaxed);
+        }
+    }
+
+    /// Chunked classifier of the binary content of the file chunk by chunk
+    /// accepts path which to reuse the allocated buffer for absolute path read
+    pub(crate) fn detect_binary_per_byte(&self, path: &Path, chunk: &mut [u8]) {
+        if self.size == 0 {
+            return;
+        }
+
+        let Ok(mut file) = std::fs::OpenOptions::new()
+            .write(false)
+            .read(true)
+            .open(path)
+        else {
+            tracing::error!(path = ?path.display(), "Failed to open indexed file");
+            return;
+        };
+
+        loop {
+            match file.read(chunk) {
+                Ok(0) => break,
+                Err(e) => {
+                    tracing::error!(?e, "Failed to read file chunk");
+                    break;
+                }
+                Ok(n) => {
+                    if detect_binary_content(&chunk[..n]) {
+                        self.set_binary(true);
+                    }
+                }
+            }
         }
     }
 
     #[inline]
     pub fn is_deleted(&self) -> bool {
-        self.flags & FileItemFlags::DELETED != 0
+        self.flags.load(Ordering::Relaxed) & FileItemFlags::DELETED != 0
     }
 
     #[inline]
-    pub fn set_deleted(&mut self, val: bool) {
+    #[doc(hidden)]
+    /// Don't use it, use FilePicker::delete_file
+    pub fn set_deleted(&self, val: bool) {
         if val {
-            self.flags |= FileItemFlags::DELETED;
+            self.flags
+                .fetch_or(FileItemFlags::DELETED, Ordering::Relaxed);
         } else {
-            self.flags &= !FileItemFlags::DELETED;
+            self.flags
+                .fetch_and(!FileItemFlags::DELETED, Ordering::Relaxed);
         }
     }
 
     #[inline]
     pub fn is_overflow(&self) -> bool {
-        self.flags & FileItemFlags::OVERFLOW != 0
+        self.flags.load(Ordering::Relaxed) & FileItemFlags::OVERFLOW != 0
     }
 
     #[inline]
-    pub fn set_overflow(&mut self, val: bool) {
+    pub fn set_overflow(&self, val: bool) {
         if val {
-            self.flags |= FileItemFlags::OVERFLOW;
+            self.flags
+                .fetch_or(FileItemFlags::OVERFLOW, Ordering::Relaxed);
         } else {
-            self.flags &= !FileItemFlags::OVERFLOW;
+            self.flags
+                .fetch_and(!FileItemFlags::OVERFLOW, Ordering::Relaxed);
         }
     }
 }
 
 impl FileItem {
-    /// Invalidate the cached content so the next `get_content()` call creates a fresh one.
+    /// Invalidate the cached mmap content, has to be called every time the file is updated.
     ///
     /// Call this when the background watcher detects that the file has been modified.
     /// On Unix, a file that is truncated while mapped can cause SIGBUS. On Windows,
     /// the stale buffer simply won't reflect the new contents. In both cases,
     /// invalidating ensures a fresh read on the next access.
+    #[cfg(not(target_os = "windows"))]
     pub fn invalidate_mmap(&mut self, budget: &ContentCacheBudget) {
         if self.content.get().is_some() {
             budget.cached_count.fetch_sub(1, Ordering::Relaxed);
@@ -460,6 +640,9 @@ impl FileItem {
         self.content = OnceLock::new();
     }
 
+    #[cfg(target_os = "windows")]
+    pub fn invalidate_mmap(&mut self, _: &ContentCacheBudget) {}
+
     pub fn update_metadata(
         &mut self,
         budget: &ContentCacheBudget,
@@ -486,7 +669,26 @@ impl FileItem {
     /// of the budget should use [`get_content_for_search`].
     ///
     /// After the first call, this is lock-free (just an atomic load + pointer deref).
-    pub(crate) fn get_content(
+    ///
+    /// On Windows we never back this cache — `memmap2` would require a full
+    /// `std::fs::read` heap copy and the OS page cache already absorbs repeat
+    /// reopens. Returning `None` keeps callers on the scratch-read slow path
+    /// and avoids duplicating every indexed file on the heap.
+    #[cfg(target_os = "windows")]
+    pub(crate) fn get_cached_content(
+        &self,
+        _arena: ArenaPtr,
+        _base_path: &Path,
+        _budget: &ContentCacheBudget,
+    ) -> Option<&[u8]> {
+        None
+    }
+
+    /// Returns a reference to a cached mmap of the file's contents.
+    ///
+    /// SAFETY-CRITICAL: callers must hold the picker read lock for as long as the returned slice is in use.
+    #[cfg(not(target_os = "windows"))]
+    pub(crate) fn get_cached_content(
         &self,
         arena: ArenaPtr,
         base_path: &Path,
@@ -496,8 +698,7 @@ impl FileItem {
             return Some(content);
         }
 
-        let max_file_size = budget.max_file_size;
-        if self.size == 0 || self.size > max_file_size {
+        if self.size < MMAP_THRESHOLD || self.size > budget.max_file_size {
             return None;
         }
 
@@ -510,11 +711,14 @@ impl FileItem {
             return None;
         }
 
-        let content = load_file_content(&self.absolute_path(arena, base_path), self.size)?;
-        let result = self.content.get_or_init(|| content);
+        let path = self.absolute_path(arena, base_path);
+        let file = std::fs::File::open(&path).ok()?;
+        // SAFETY: the mmap is backed by the kernel page cache and reflects
+        // file updates; the only risk is SIGBUS on a concurrent truncate,
+        // which the watcher mitigates by invalidating on modification.
+        let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
+        let result = self.content.get_or_init(|| mmap);
 
-        // Bump counters. Slight over-count under races is fine — the budget
-        // is a soft limit and the overshoot is bounded by rayon thread count.
         budget.cached_count.fetch_add(1, Ordering::Relaxed);
         budget.cached_bytes.fetch_add(self.size, Ordering::Relaxed);
 
@@ -529,14 +733,20 @@ impl FileItem {
     #[inline]
     pub(crate) fn get_content_for_search<'a>(
         &'a self,
-        buf: &'a mut Vec<u8>, // we allow it to grow
+        buf: &'a mut Vec<u8>,
+        #[cfg_attr(target_os = "windows", allow(unused_variables))] mmap_slot: &'a mut MmapSlot,
         arena: ArenaPtr,
         base_path: &Path,
         budget: &ContentCacheBudget,
     ) -> Option<&'a [u8]> {
-        // Fast path: persistent cache hit (zero-copy).
-        if let Some(cached) = self.get_content(arena, base_path, budget) {
-            return Some(cached);
+        #[cfg(not(target_os = "windows"))]
+        {
+            // Fast path: persistent cache hit (zero-copy). Safe here because
+            // grep callers hold the picker read lock for the lifetime of the
+            // returned slice — see [`Self::get_cached_content`] safety note.
+            if let Some(cached) = self.get_cached_content(arena, base_path, budget) {
+                return Some(cached);
+            }
         }
 
         let max_file_size = budget.max_file_size;
@@ -544,49 +754,34 @@ impl FileItem {
             return None;
         }
 
-        // Slow path: read into the reusable buffer — open() + read_exact() + close().
-        // No mmap()/munmap() syscalls, no page table setup/teardown.
-        // We know the exact size so we use read_exact (1 read syscall) instead of
-        // read_to_end (2 read syscalls — one for data, one for EOF confirmation).
         let abs = self.absolute_path(arena, base_path);
+
+        #[cfg(not(target_os = "windows"))]
+        if self.size >= FRESH_MMAP_THRESHOLD {
+            let file = std::fs::File::open(&abs).ok()?;
+            let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
+            let stored = mmap_slot.insert(mmap);
+            return Some(&stored[..]);
+        } else {
+            let _ = (mmap_slot, arena);
+        }
+
         let len = self.size as usize;
         buf.resize(len, 0);
+
         let mut file = std::fs::File::open(&abs).ok()?;
         file.read_exact(buf).ok()?;
         Some(buf.as_slice())
     }
 }
 
-/// Files smaller than one page waste the remainder when mmapped.
-/// Unused on Windows where `load_file_content` does not mmap.
-#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
-const MMAP_THRESHOLD: u64 = 16 * 1024;
-#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
-const MMAP_THRESHOLD: u64 = 4 * 1024;
-
-fn load_file_content(path: &Path, size: u64) -> Option<FileContent> {
-    #[cfg(not(target_os = "windows"))]
-    {
-        if size < MMAP_THRESHOLD {
-            let data = std::fs::read(path).ok()?;
-            Some(FileContent::Buffer(data))
-        } else {
-            let file = std::fs::File::open(path).ok()?;
-            // SAFETY: The mmap is backed by the kernel page cache and automatically
-            // reflects file modifications. The only risk is SIGBUS if the file is
-            // truncated while mapped.
-            let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
-            Some(FileContent::Mmap(mmap))
-        }
-    }
-
-    #[cfg(target_os = "windows")]
-    {
-        let _ = size;
-        let data = std::fs::read(path).ok()?;
-        Some(FileContent::Buffer(data))
-    }
-}
+/// Per-thread scratch slot owning a transient mmap returned from
+/// [`FileItem::get_content_for_search`]. `Option<Mmap>` on Unix,
+/// unit on Windows where mmap is unused.
+#[cfg(not(target_os = "windows"))]
+pub type MmapSlot = Option<memmap2::Mmap>;
+#[cfg(target_os = "windows")]
+pub type MmapSlot = ();
 
 impl Constrainable for FileItem {
     #[inline]
@@ -603,6 +798,11 @@ impl Constrainable for FileItem {
     fn git_status(&self) -> Option<git2::Status> {
         self.git_status
     }
+
+    #[inline]
+    fn is_overflow(&self) -> bool {
+        FileItem::is_overflow(self)
+    }
 }
 
 #[derive(Debug, Clone, Default)]
@@ -663,6 +863,7 @@ impl ScoringContext<'_> {
 pub struct SearchResult<'a> {
     pub items: Vec<&'a FileItem>,
     pub scores: Vec<Score>,
+    pub match_byte_offsets: Vec<smallvec::SmallVec<[(u32, u32); 4]>>,
     pub total_matched: usize,
     pub total_files: usize,
     pub location: Option<Location>,
@@ -703,10 +904,6 @@ impl Default for MixedItemRef<'_> {
     }
 }
 
-const MAX_MMAP_FILE_SIZE: u64 = 10 * 1024 * 1024;
-
-const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
-
 #[derive(Debug)]
 pub struct ContentCacheBudget {
     pub max_files: usize,
@@ -721,7 +918,7 @@ impl ContentCacheBudget {
         Self {
             max_files: usize::MAX,
             max_bytes: u64::MAX,
-            max_file_size: MAX_MMAP_FILE_SIZE,
+            max_file_size: MAX_FFFILE_SIZE,
             cached_count: AtomicUsize::new(0),
             cached_bytes: AtomicU64::new(0),
         }
@@ -737,6 +934,12 @@ impl ContentCacheBudget {
         }
     }
 
+    // Byte budget
+    pub fn is_exhausted(&self) -> bool {
+        self.cached_count.load(Ordering::Relaxed) >= self.max_files
+            || self.cached_bytes.load(Ordering::Relaxed) >= self.max_bytes
+    }
+
     pub fn new_for_repo(file_count: usize) -> Self {
         let max_files = if file_count > 50_000 {
             5_000
@@ -757,7 +960,7 @@ impl ContentCacheBudget {
         Self {
             max_files,
             max_bytes,
-            max_file_size: MAX_MMAP_FILE_SIZE,
+            max_file_size: MAX_FFFILE_SIZE,
             cached_count: AtomicUsize::new(0),
             cached_bytes: AtomicU64::new(0),
         }
@@ -799,43 +1002,3 @@ impl Default for ContentCacheBudget {
         Self::new_for_repo(30_000)
     }
 }
-
-#[cfg(test)]
-impl FileItem {
-    /// Leaks a single-file arena so the pointer stays valid forever.
-    pub fn new_for_test(
-        rel_path: &str,
-        size: u64,
-        modified: u64,
-        git_status: Option<git2::Status>,
-        is_binary: bool,
-    ) -> Self {
-        let (item, _arena) =
-            Self::new_for_test_with_arena(rel_path, size, modified, git_status, is_binary);
-        item
-    }
-
-    pub(crate) fn new_for_test_with_arena(
-        rel_path: &str,
-        size: u64,
-        modified: u64,
-        git_status: Option<git2::Status>,
-        is_binary: bool,
-    ) -> (Self, ArenaPtr) {
-        let filename_start = rel_path
-            .rfind(std::path::is_separator)
-            .map(|i| i + 1)
-            .unwrap_or(0) as u16;
-        let mut item = Self::new_raw(filename_start, size, modified, git_status, is_binary);
-        let paths = [rel_path.to_string()];
-        let (store, strings) = crate::simd_path::build_chunked_path_store_from_strings(
-            &paths,
-            std::slice::from_ref(&item),
-        );
-        let cs = strings.into_iter().next().unwrap();
-        let arena = store.as_arena_ptr();
-        item.set_path(cs);
-        std::mem::forget(store);
-        (item, arena)
-    }
-}

crates/fff-core/src/walk/mod.rs

@@ -0,0 +1,148 @@
+//! Filesystem traversal backend. Selects one implementation at compile time:
+//! - `zlob`: zlob's native parallel walker (requires the Zig toolchain).
+//! - `ripgrep`: the `ignore` crate (ripgrep's walker), used by default.
+//!
+//! Both expose [`walk_collect_files`] with identical semantics so the rest of
+//! the crate stays backend-agnostic.
+
+use crate::types::FileItem;
+use std::path::Path;
+
+#[cfg(feature = "zlob")]
+mod zlob;
+#[cfg(feature = "zlob")]
+pub(crate) use zlob::walk_collect_files;
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+mod ripgrep;
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) use ripgrep::walk_collect_files;
+
+pub(crate) struct WalkOutput {
+    pub(crate) pairs: Vec<(FileItem, String)>,
+    /// Every non-ignored directory the walk visited, relative, ending with /
+    pub(crate) dirs: Vec<String>,
+    pub(crate) ignore_rules: Option<WalkIgnoreRules>,
+}
+
+pub(crate) struct WalkIgnoreRules {
+    #[cfg(feature = "zlob")]
+    inner: ::zlob::walk::WalkerOutcomeRules,
+    #[cfg(not(feature = "zlob"))]
+    _never: std::convert::Infallible,
+}
+
+// SAFETY: the underlying storage is immutable, heap-owned, and thread-safe to
+// read from concurrently (mirrors zlob's `IgnoreRules: Send + Sync`).
+unsafe impl Send for WalkIgnoreRules {}
+unsafe impl Sync for WalkIgnoreRules {}
+
+impl std::fmt::Debug for WalkIgnoreRules {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("WalkIgnoreRules")
+    }
+}
+
+// In ripgrep builds `WalkIgnoreRules` is never constructed (the `_never`
+// field is uninhabited), so its methods are legitimately dead there.
+#[cfg_attr(not(feature = "zlob"), allow(dead_code))]
+impl WalkIgnoreRules {
+    /// Returns `true` if the provided path is ignored by the collected rule set
+    ///
+    /// `relative_path` has to be relative to the walker's provided base path
+    pub(crate) fn is_ignored(&self, relative_path: &Path) -> bool {
+        #[cfg(feature = "zlob")]
+        {
+            self.inner
+                .rules()
+                .is_some_and(|rules| rules.is_ignored(relative_path))
+        }
+        #[cfg(not(feature = "zlob"))]
+        {
+            let _ = relative_path;
+            match self._never {}
+        }
+    }
+
+    // The old `is_ignored_untrusted` variant was folded away when zlob's
+    // ignore matcher moved to full ancestor enumeration — trailing-slash
+    // sniffing on the input is now sufficient for external queries.
+}
+
+#[cfg(test)]
+mod tests {
+    use super::walk_collect_files;
+    use std::fs;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    // Backend-agnostic parity check: both the zlob and ripgrep walkers must
+    // respect .gitignore, skip hidden files in a git repo, and surface the
+    // expected file set with a correct synced count.
+    #[test]
+    fn collects_files_respecting_gitignore() {
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join(".git")).unwrap();
+        fs::create_dir(root.join("src")).unwrap();
+        fs::create_dir(root.join("target")).unwrap();
+        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();
+        fs::write(root.join("Cargo.toml"), "x").unwrap();
+        fs::write(root.join("debug.log"), "").unwrap();
+        fs::write(root.join("src/main.rs"), "fn main() {}").unwrap();
+        fs::write(root.join("target/out.bin"), "bin").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, true, false, 1, &counter).unwrap();
+
+        let mut names: Vec<String> = out.pairs.into_iter().map(|(_, rel)| rel).collect();
+        names.sort();
+
+        assert!(names.contains(&"Cargo.toml".to_string()));
+        assert!(names.iter().any(|n| n.ends_with("main.rs")));
+        // target/ and *.log are gitignored; .git/ is skipped.
+        assert!(!names.iter().any(|n| n.contains("target")));
+        assert!(!names.iter().any(|n| n.ends_with(".log")));
+        assert!(!names.iter().any(|n| n.contains(".git/")));
+        assert_eq!(counter.load(Ordering::Relaxed), names.len());
+    }
+
+    // Non-git roots prune known non-code directories (node_modules).
+    #[test]
+    fn prunes_non_code_dirs_for_non_git_root() {
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join("node_modules")).unwrap();
+        fs::write(root.join("node_modules/lib.js"), "x").unwrap();
+        fs::write(root.join("index.js"), "x").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, false, false, 1, &counter).unwrap();
+        let names: Vec<String> = out.pairs.into_iter().map(|(_, rel)| rel).collect();
+
+        assert!(names.iter().any(|n| n.ends_with("index.js")));
+        assert!(!names.iter().any(|n| n.contains("node_modules")));
+    }
+
+    // Only the zlob backend surfaces reusable ignore rules; they must match
+    // the same tree the walk respected.
+    #[cfg(feature = "zlob")]
+    #[test]
+    fn surfaces_reusable_ignore_rules() {
+        use std::path::Path;
+
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join(".git")).unwrap();
+        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();
+        fs::write(root.join("Cargo.toml"), "x").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, true, false, 1, &counter).unwrap();
+
+        let rules = out.ignore_rules.expect("zlob surfaces ignore rules");
+        assert!(rules.is_ignored(Path::new("target/")));
+        assert!(rules.is_ignored(Path::new("debug.log")));
+        assert!(!rules.is_ignored(Path::new("Cargo.toml")));
+    }
+}

crates/fff-core/src/walk/ripgrep.rs

@@ -0,0 +1,87 @@
+use crate::ignore::non_git_repo_overrides;
+use crate::types::FileItem;
+use crate::walk::WalkOutput;
+use crate::watch::is_git_file;
+use ignore::WalkBuilder;
+use std::path::Path;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+#[tracing::instrument(skip_all, name = "ripgrep walker", level = "info")]
+pub(crate) fn walk_collect_files(
+    base_path: &Path,
+    is_git_repo: bool,
+    follow_symlinks: bool,
+    threads: usize,
+    synced_files_count: &Arc<AtomicUsize>,
+) -> crate::Result<WalkOutput> {
+    let mut walk_builder = WalkBuilder::new(base_path);
+    walk_builder
+        // this is a very important guard for the user opening ~/ or other root non-git dir
+        .hidden(!is_git_repo)
+        .git_ignore(true)
+        .git_exclude(true)
+        .git_global(true)
+        .ignore(true)
+        .follow_links(follow_symlinks)
+        .threads(threads);
+
+    if !is_git_repo && let Some(overrides) = non_git_repo_overrides(base_path) {
+        walk_builder.overrides(overrides);
+    }
+
+    let walker = walk_builder.build_parallel();
+
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected =
+        parking_lot::Mutex::new((Vec::<(FileItem, String)>::new(), Vec::<String>::new()));
+    walker.run(|| {
+        let collected = &collected;
+        let counter = Arc::clone(synced_files_count);
+        let base_path = base_path.to_path_buf();
+
+        Box::new(move |result| {
+            let Ok(entry) = result else {
+                return ignore::WalkState::Continue;
+            };
+
+            if entry.file_type().is_some_and(|ft| ft.is_file()) {
+                let path = entry.path();
+
+                // Ignore walkers sometimes surface files inside `.git/`
+                // when the base is itself a git repo — skip them.
+                if is_git_file(path) {
+                    return ignore::WalkState::Continue;
+                }
+
+                let metadata = entry.metadata().ok();
+                let (file_item, rel_path) =
+                    FileItem::new_from_walk(path, &base_path, None, metadata.as_ref());
+
+                collected.lock().0.push((file_item, rel_path));
+                counter.fetch_add(1, Ordering::Relaxed);
+            } else if entry.depth() > 0 && entry.file_type().is_some_and(|ft| ft.is_dir()) {
+                let path = entry.path();
+                if !is_git_file(path)
+                    && let Ok(rel) = path.strip_prefix(&base_path)
+                {
+                    let mut rel = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy())
+                        .into_owned();
+                    rel.push('/');
+                    collected.lock().1.push(rel);
+                }
+            }
+            ignore::WalkState::Continue
+        })
+    });
+
+    let (pairs, dirs) = collected.into_inner();
+    Ok(WalkOutput {
+        pairs,
+        dirs,
+        ignore_rules: None,
+    })
+}

crates/fff-core/src/walk/zlob.rs

@@ -0,0 +1,123 @@
+use crate::file_picker::is_known_binary_extension_basename;
+use crate::ignore::IGNORED_DIRS;
+use crate::types::FileItem;
+use crate::walk::{WalkIgnoreRules, WalkOutput};
+use parking_lot::Mutex;
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use zlob::walk::{WalkBuilder, WalkFlags, WalkMetadata, WalkState};
+
+const PROGRESS_STEP: usize = 13;
+
+#[tracing::instrument(skip_all, name = "zlob walker", level = "info")]
+pub(crate) fn walk_collect_files(
+    base_path: &Path,
+    is_git_repo: bool,
+    follow_symlinks: bool,
+    threads: usize,
+    synced_files_count: &Arc<AtomicUsize>,
+) -> crate::Result<WalkOutput> {
+    // gitignore on; skip hidden on non-git roots (so `~/` doesn't recurse into
+    // ~/.cache, ~/.config, etc.); optionally follow symlinks.
+    let mut flags = WalkFlags::GITIGNORE;
+    if !is_git_repo {
+        flags |= WalkFlags::SKIP_HIDDEN;
+    }
+    if follow_symlinks {
+        flags |= WalkFlags::FOLLOW_SYMLINKS;
+    }
+
+    let mut builder = WalkBuilder::new(base_path)
+        .map_err(|e| crate::Error::WalkFailed(format!("WalkBuilder::new: {e:?}")))?;
+    builder
+        .options(flags)
+        .threads(threads)
+        // Bulk-fetch the only metadata FileItem needs; zlob never stats more.
+        .metadata(WalkMetadata::SIZE | WalkMetadata::MTIME);
+
+    if !is_git_repo
+        && !IGNORED_DIRS.is_empty()
+        && let Err(e) = builder.extra_ignore(IGNORED_DIRS)
+    {
+        // Interior NUL in one of the extra_ignore patterns would fail
+        // here — treat as if no extras were supplied rather than
+        // aborting the whole walk.
+        tracing::warn!(?e, "zlob extra_ignore rejected; walking without it");
+    }
+
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected = Mutex::new((Vec::new(), Vec::new()));
+
+    let outcome = match builder.run(|entry| {
+        if !entry.is_file() {
+            // unlike ripgrep walker zlob doesnt show .git files
+            if entry.is_dir() {
+                let rel_bytes = entry.relative_path_bytes();
+                if !rel_bytes.is_empty() {
+                    let mut rel = String::from_utf8_lossy(rel_bytes).into_owned();
+                    rel.push('/');
+                    collected.lock().1.push(rel);
+                }
+            }
+
+            return WalkState::Continue;
+        }
+
+        let rel_bytes = entry.relative_path_bytes();
+
+        // `basename()` returns `&str` for files only.
+        let basename = entry.basename().unwrap_or("");
+        let is_binary = is_known_binary_extension_basename(basename);
+
+        let size = entry.size().unwrap_or(0);
+        // zlob reports mtime in ns since the Unix epoch; FileItem wants secs.
+        let modified = entry
+            .modified_ns()
+            .map(|ns| (ns / 1_000_000_000).max(0) as u64)
+            .unwrap_or(0);
+
+        let basename_offset = entry.basename_offset_in_relative();
+        // zlob emits '/'-separated relative paths, which is fff's canonical
+        // internal form on every platform — store them verbatim.
+        let rel_str = String::from_utf8_lossy(rel_bytes).into_owned();
+        let item = FileItem::new_raw(basename_offset, size, modified, None, is_binary);
+
+        let mut guard = collected.lock();
+        guard.0.push((item, rel_str));
+        let n = guard.0.len();
+        drop(guard);
+
+        if n % PROGRESS_STEP == 0 {
+            synced_files_count.store(n, Ordering::Relaxed);
+        }
+
+        WalkState::Continue
+    }) {
+        Ok(outcome) => outcome,
+        Err(e) => {
+            // Preserve whatever we collected before the failure so the caller
+            // can still surface a partial index instead of nothing.
+            tracing::error!(?e, "zlob walk failed");
+            return Err(crate::Error::WalkFailed(format!("{e:?}")));
+        }
+    };
+
+    let (pairs, dirs) = collected.into_inner();
+    // Always report the exact final total regardless of the last step.
+    synced_files_count.store(pairs.len(), Ordering::Relaxed);
+
+    // Retain the ignore rules only when the walk actually gathered some
+    // (git roots with .gitignore/.ignore). Otherwise callers fall back.
+    let ignore_rules = outcome
+        .rules()
+        .is_some()
+        .then(|| WalkIgnoreRules { inner: outcome });
+
+    Ok(WalkOutput {
+        pairs,
+        dirs,
+        ignore_rules,
+    })
+}

crates/fff-core/src/watcher/mod.rs

@@ -0,0 +1,9 @@
+mod background_watcher;
+pub use background_watcher::*;
+
+mod watch;
+pub use watch::*;
+
+// The harness reads rescan counters, which release builds compile out.
+#[cfg(all(test, rescan_stats))]
+mod rescan_tests;

crates/fff-core/src/watcher/rescan_tests.rs

@@ -0,0 +1,621 @@
+use std::path::PathBuf;
+use std::process::Command;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use notify::Event;
+use notify::EventKind;
+use notify::event::{
+    AccessKind, AccessMode, CreateKind, DataChange, Flag, ModifyKind, RemoveKind, RenameMode,
+};
+use notify_debouncer_full::DebouncedEvent;
+use tempfile::TempDir;
+
+use super::handle_debounced_events;
+use crate::constants::MAX_OVERFLOW_FILES;
+use crate::file_picker::{FFFMode, FilePicker, FilePickerOptions};
+use crate::git_status_worker::GitStatusWorker;
+use crate::rescan_stats::{RescanReason, RescanStats};
+use crate::shared::{SharedFilePicker, SharedFrecency};
+
+#[test]
+fn saving_an_indexed_file_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/main.rs", "fn main() { println!(); }");
+    let delta = f.feed([modify(f.path("src/main.rs"))]);
+
+    f.assert_no_rescan(&delta, "saving a tracked file");
+}
+
+#[test]
+fn editor_atomic_save_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // write-to-temp + rename-over-target, the way vim/VSCode/IntelliJ save.
+    f.write("src/main.rs", "fn main() { println!(); }");
+    let target = f.path("src/main.rs");
+    let temp = f.path("src/.main.rs.swp");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Create(CreateKind::File)).add_path(temp.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::From)))
+                .add_path(temp.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::To)))
+                .add_path(target.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Remove(RemoveKind::File)).add_path(temp),
+            Instant::now(),
+        ),
+    ]);
+
+    f.assert_no_rescan(&delta, "an atomic editor save");
+    assert!(f.is_indexed("src/main.rs"), "target must stay indexed");
+}
+
+#[test]
+fn creating_and_deleting_files_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/added.rs", "pub fn added() {}");
+    let created = f.feed([create(f.path("src/added.rs"))]);
+    f.assert_no_rescan(&created, "creating a file");
+    assert!(f.is_indexed("src/added.rs"));
+
+    f.remove("src/added.rs");
+    let removed = f.feed([remove_file(f.path("src/added.rs"))]);
+    f.assert_no_rescan(&removed, "deleting a file");
+    assert!(!f.is_indexed("src/added.rs"));
+}
+
+#[test]
+fn deleting_a_directory_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.write("src/nested/a.rs", "");
+    f.write("src/nested/b.rs", "");
+    f.index();
+
+    std::fs::remove_dir_all(f.path("src/nested")).unwrap();
+    let delta = f.feed([DebouncedEvent::new(
+        Event::new(EventKind::Remove(RemoveKind::Folder)).add_path(f.path("src/nested")),
+        Instant::now(),
+    )]);
+
+    f.assert_no_rescan(&delta, "deleting a directory");
+    assert!(!f.is_indexed("src/nested/a.rs"));
+    assert!(f.is_indexed("src/main.rs"));
+}
+
+#[test]
+fn read_only_access_events_are_ignored() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // fff's own preview + grep reads generate these; reacting to them would
+    // make the picker rescan whenever the user scrolls the result list.
+    let path = f.path("src/main.rs");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Read)).add_path(path.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Open(AccessMode::Read)))
+                .add_path(path.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Close(AccessMode::Read))).add_path(path),
+            Instant::now(),
+        ),
+    ]);
+
+    f.assert_no_rescan(&delta, "read-only access events");
+}
+
+#[test]
+fn recreating_the_same_paths_does_not_consume_overflow_capacity() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Recreated paths must reuse their overflow slots.
+    for _ in 0..8 {
+        for i in 0..200 {
+            let rel = format!("gen/out{i}.rs");
+            f.write(&rel, "generated");
+            f.feed([create(f.path(&rel))]);
+        }
+        for i in 0..200 {
+            let rel = format!("gen/out{i}.rs");
+            f.remove(&rel);
+            f.feed([remove_file(f.path(&rel))]);
+        }
+    }
+
+    let delta = f.all_rescans();
+    f.assert_no_rescan(&delta, "1600 create/delete cycles over 200 stable paths");
+    assert!(
+        f.overflow_len() <= 200,
+        "each path must claim one overflow slot at most, got {}",
+        f.overflow_len()
+    );
+}
+
+#[test]
+fn writes_inside_a_gitignored_directory_stay_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\nnode_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let mut events = Vec::new();
+    for i in 0..64 {
+        let rel = format!("target/debug/artifact{i}.o");
+        f.write(&rel, "binary");
+        events.push(create(f.path(&rel)));
+    }
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "build output written into an ignored directory");
+}
+
+#[test]
+fn ignored_event_batch_above_index_capacity_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "node_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|i| {
+            let rel = format!("node_modules/pkg/file{i}.js");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "ignored events above the index capacity");
+    assert_eq!(f.overflow_len(), 0);
+}
+
+#[test]
+fn repeated_edits_above_index_capacity_stay_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "repeated edits above the index capacity");
+    assert_eq!(f.overflow_len(), 0);
+}
+
+#[test]
+fn ignore_file_inside_an_ignored_directory_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "node_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let ignore_files =
+        ["left-pad", "lodash", "typescript"].map(|pkg| format!("node_modules/{pkg}/.gitignore"));
+
+    for rel in &ignore_files {
+        f.write(rel, "dist\n");
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| create(f.path(rel))));
+    f.assert_no_rescan(&delta, "creating ignored .gitignore files");
+
+    for rel in &ignore_files {
+        f.write(rel, "build\n");
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| modify(f.path(rel))));
+    f.assert_no_rescan(&delta, "modifying ignored .gitignore files");
+
+    for rel in &ignore_files {
+        f.remove(rel);
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| remove_file(f.path(rel))));
+    f.assert_no_rescan(&delta, "removing ignored .gitignore files");
+}
+
+#[test]
+fn ignore_file_inside_an_indexed_directory_triggers_a_rescan() {
+    let f = Fixture::with_git();
+    f.write("src/.gitignore", ".gitignore\ngenerated/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/.gitignore", ".gitignore\ngenerated/\nbuild/\n");
+    let delta = f.feed([modify(f.path("src/.gitignore"))]);
+
+    assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
+}
+
+#[test]
+fn git_internal_churn_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let git_dir = f.path(".git");
+    let delta = f.feed([
+        create(git_dir.join("index.lock")),
+        modify(git_dir.join("index")),
+        remove_file(git_dir.join("index.lock")),
+        modify(git_dir.join("HEAD")),
+        modify(git_dir.join("logs/HEAD")),
+        modify(git_dir.join("COMMIT_EDITMSG")),
+        modify(git_dir.join("refs/heads/main")),
+    ]);
+
+    f.assert_no_rescan(&delta, "git writing its own metadata");
+}
+
+#[test]
+fn changing_the_root_ignore_file_triggers_a_rescan() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write(".gitignore", "target/\nsrc/\n");
+    let delta = f.feed([modify(f.path(".gitignore"))]);
+
+    assert_eq!(
+        delta.count(RescanReason::IgnoreFileChanged),
+        1,
+        "the indexed set depends on the root ignore rules, got {delta}"
+    );
+}
+
+#[test]
+fn kernel_event_loss_on_a_directory_triggers_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let delta = f.feed([DebouncedEvent::new(
+        Event::new(EventKind::Modify(ModifyKind::Any))
+            .add_path(f.path("src"))
+            .set_flag(Flag::Rescan),
+        Instant::now(),
+    )]);
+
+    assert_eq!(
+        delta.count(RescanReason::KernelEventLoss),
+        1,
+        "a dropped-events flag over a directory means unknown subtree state, got {delta}"
+    );
+}
+
+#[test]
+fn new_files_above_index_capacity_trigger_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|i| {
+            let rel = format!("src/bulk{i}.rs");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    assert_eq!(
+        delta.count(RescanReason::IndexUpdateRejected),
+        1,
+        "new files above the overflow region cannot be applied incrementally, got {delta}"
+    );
+}
+
+#[test]
+fn batch_at_the_overflow_boundary_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES)
+        .map(|i| {
+            let rel = format!("src/bulk{i}.rs");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "a batch exactly at the overflow limit");
+}
+
+#[test]
+fn event_batch_at_four_times_index_capacity_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES * 4)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "an event batch exactly at the event limit");
+}
+
+#[test]
+fn event_batch_above_four_times_index_capacity_triggers_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES * 4 + 1)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    assert_eq!(
+        delta.count(RescanReason::EventBatchOverflow),
+        1,
+        "an event batch above four times the index capacity must rescan, got {delta}"
+    );
+}
+
+#[test]
+fn repeated_triggers_inside_the_cooldown_collapse_to_one_rescan() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Repeated batches during the cooldown must share one walk.
+    for round in 0..50 {
+        f.write(".gitignore", &format!("target/\n# round {round}\n"));
+        f.feed([modify(f.path(".gitignore"))]);
+    }
+
+    let stats = f.all_rescans();
+    assert_eq!(
+        stats.total, 1,
+        "50 triggers inside the cooldown must collapse to a single walk, got {stats}"
+    );
+    assert_eq!(
+        stats.throttled, 49,
+        "every suppressed request must be accounted for, got {stats}"
+    );
+}
+
+#[test]
+fn an_explicit_request_is_never_throttled() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Burn the cooldown with a watcher trigger, then confirm a user-initiated
+    // refresh still goes through.
+    f.write(".gitignore", "target/\nsrc/\n");
+    f.feed([modify(f.path(".gitignore"))]);
+
+    for _ in 0..3 {
+        f.picker.trigger_full_rescan_async(&f.frecency).unwrap();
+    }
+
+    let stats = f.all_rescans();
+    assert_eq!(
+        stats.count(RescanReason::Explicit),
+        3,
+        "explicit refreshes must bypass the throttle, got {stats}"
+    );
+    assert_eq!(stats.count_throttled(RescanReason::Explicit), 0);
+}
+
+#[test]
+fn events_after_a_suppressed_kernel_rescan_are_still_applied() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/added.rs", "pub fn added() {}");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content)))
+                .add_path(f.path("src/main.rs"))
+                .set_flag(Flag::Rescan),
+            Instant::now(),
+        ),
+        create(f.path("src/added.rs")),
+    ]);
+
+    f.assert_no_rescan(&delta, "a dropped-events flag over a single tracked file");
+    assert!(
+        f.is_indexed("src/added.rs"),
+        "suppressing the rescan must not drop the rest of the batch"
+    );
+}
+
+#[test]
+fn a_throttled_ignore_file_event_is_still_applied_incrementally() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Burn the cooldown: deleting .gitignore admits a full rescan.
+    f.remove(".gitignore");
+    let delta = f.feed([remove_file(f.path(".gitignore"))]);
+    assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
+    f.picker.wait_for_indexing_complete(Duration::from_secs(10));
+
+    // Recreating it inside the cooldown throttles the rescan, but the file
+    // itself must re-enter the index via the incremental fallback.
+    f.write(".gitignore", "target/\n__ignored_x/\n");
+    let delta = f.feed([create(f.path(".gitignore"))]);
+    assert_eq!(delta.total, 0, "the rescan must be throttled, got {delta}");
+    assert_eq!(delta.count_throttled(RescanReason::IgnoreFileChanged), 1);
+    assert!(
+        f.is_indexed(".gitignore"),
+        "a throttled ignore-file event must still index the file itself"
+    );
+}
+
+struct Fixture {
+    base: PathBuf,
+    picker: SharedFilePicker,
+    frecency: SharedFrecency,
+    git_workdir: Option<PathBuf>,
+    git_worker: Arc<GitStatusWorker>,
+    // Dropped last so background work started by a triggered rescan still
+    // sees the tree it was asked to walk.
+    _tmp: TempDir,
+}
+
+impl Fixture {
+    fn new() -> Self {
+        Self::build(false)
+    }
+
+    fn with_git() -> Self {
+        Self::build(true)
+    }
+
+    fn build(git: bool) -> Self {
+        let tmp = tempfile::tempdir().unwrap();
+        let base = crate::path_utils::canonicalize(tmp.path()).unwrap();
+        let git_workdir = git.then(|| {
+            let status = Command::new("git")
+                .args(["init", "-b", "main"])
+                .current_dir(&base)
+                .output()
+                .expect("git init");
+            assert!(status.status.success(), "git init failed");
+            base.clone()
+        });
+
+        Self {
+            base,
+            picker: SharedFilePicker::default(),
+            frecency: SharedFrecency::noop(),
+            git_workdir,
+            git_worker: GitStatusWorker::new(),
+            _tmp: tmp,
+        }
+    }
+
+    fn index(&self) {
+        let mut picker = FilePicker::new(FilePickerOptions {
+            base_path: self.base.to_string_lossy().into_owned(),
+            watch: false,
+            ..Default::default()
+        })
+        .unwrap();
+        picker.collect_files().unwrap();
+        self.picker.rebase_watches(&self.base);
+        *self.picker.write().unwrap() = Some(picker);
+    }
+
+    fn feed(&self, events: impl IntoIterator<Item = DebouncedEvent>) -> RescanStats {
+        let before = self.picker.rescan_stats();
+        handle_debounced_events(
+            FFFMode::Neovim,
+            events.into_iter().collect(),
+            &self.base,
+            &self.git_workdir,
+            &self.picker,
+            &self.frecency,
+            &self.git_worker,
+        );
+
+        self.picker.rescan_stats().since(&before)
+    }
+
+    fn assert_no_rescan(&self, delta: &RescanStats, what: &str) {
+        assert_eq!(delta.total, 0, "{what} must not trigger a rescan: {delta}");
+    }
+
+    fn path(&self, rel: &str) -> PathBuf {
+        self.base.join(rel)
+    }
+
+    fn write(&self, rel: &str, contents: &str) {
+        let path = self.path(rel);
+        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+        std::fs::write(path, contents).unwrap();
+    }
+
+    fn remove(&self, rel: &str) {
+        std::fs::remove_file(self.path(rel)).unwrap();
+    }
+
+    fn is_indexed(&self, rel: &str) -> bool {
+        let guard = self.picker.read().unwrap();
+        guard
+            .as_ref()
+            .and_then(|p| p.get_file_by_path(self.path(rel)))
+            .is_some_and(|file| !file.is_deleted())
+    }
+
+    fn all_rescans(&self) -> RescanStats {
+        self.picker.rescan_stats()
+    }
+
+    fn overflow_len(&self) -> usize {
+        let guard = self.picker.read().unwrap();
+        guard
+            .as_ref()
+            .map(|p| p.get_overflow_files().len())
+            .unwrap_or(0)
+    }
+}
+
+impl Drop for Fixture {
+    fn drop(&mut self) {
+        // A test that intentionally triggers a rescan leaves a walk running on
+        // the background pool; let it finish before the tree disappears.
+        self.picker
+            .wait_for_indexing_complete(Duration::from_secs(10));
+    }
+}
+
+fn event(kind: EventKind, path: PathBuf) -> DebouncedEvent {
+    DebouncedEvent::new(Event::new(kind).add_path(path), Instant::now())
+}
+
+fn create(path: PathBuf) -> DebouncedEvent {
+    event(EventKind::Create(CreateKind::File), path)
+}
+
+fn modify(path: PathBuf) -> DebouncedEvent {
+    event(
+        EventKind::Modify(ModifyKind::Data(DataChange::Content)),
+        path,
+    )
+}
+
+fn remove_file(path: PathBuf) -> DebouncedEvent {
+    event(EventKind::Remove(RemoveKind::File), path)
+}

crates/fff-core/tests/bigram_overlay_coherence_test.rs

@@ -13,6 +13,8 @@
 //!    exists but is dead code — never called from the grep path.
 //!    See: grep.rs lines ~1787-1855.
 
+mod overflow_frecency_segfault;
+
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -84,7 +86,7 @@ fn bigram_overlay_coherence_stress_base_edits_and_deletes() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(name)).is_some(),
+                    picker.handle_create_or_modify(base.join(name)).is_some(),
                     "round {round}: modify({name}) should succeed"
                 );
             }
@@ -148,7 +150,7 @@ fn bigram_overlay_coherence_long_session_incremental_edits() {
             {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
-                picker.on_create_or_modify(base.join(name));
+                picker.handle_create_or_modify(base.join(name));
             }
             latest_tokens[file_idx] = new_token;
         }
@@ -225,7 +227,7 @@ fn bigram_overlay_coherence_resurrect_tombstoned_file() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&target_path).is_some());
+        assert!(picker.handle_create_or_modify(&target_path).is_some());
     }
 
     {
@@ -275,7 +277,7 @@ fn bigram_overlay_coherence_proves_contribution_for_modified_base() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(&target_path);
+        picker.handle_create_or_modify(&target_path);
     }
 
     {
@@ -284,12 +286,6 @@ fn bigram_overlay_coherence_proves_contribution_for_modified_base() {
 
         let with_overlay = grep_count(picker, unique);
         assert_eq!(with_overlay, 1, "overlay should find the new token");
-
-        let without_overlay = grep_without_overlay_count(picker, unique);
-        assert_eq!(
-            without_overlay, 0,
-            "without overlay, bigram should exclude the file (stale bigrams)"
-        );
     }
 
     stop_picker(&shared_picker);
@@ -333,7 +329,7 @@ fn bigram_overlay_coherence_rapid_create_delete_same_base_path() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(&volatile_path);
+            picker.handle_create_or_modify(&volatile_path);
         }
 
         {
@@ -376,7 +372,7 @@ fn bigram_overlay_coherence_rapid_create_delete_same_base_path() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(&volatile_path);
+            picker.handle_create_or_modify(&volatile_path);
         }
     }
 
@@ -434,7 +430,7 @@ fn bigram_overlay_coherence_overflow_files_searchable_via_grep() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&new_path).is_some());
+        assert!(picker.handle_create_or_modify(&new_path).is_some());
     }
 
     // Overflow file is tracked.
@@ -499,7 +495,7 @@ fn bigram_overlay_coherence_mixed_tombstones_and_overflow() {
         write_file_with_token(base, &name, &token);
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join(&name));
+        picker.handle_create_or_modify(base.join(&name));
         new_tokens.push(token);
     }
 
@@ -573,7 +569,7 @@ fn bigram_overlay_coherence_full_stress_loop_with_overflow() {
             write_file_with_token(base, name, &new_token);
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
             dead_tokens.push(old_token.clone());
             *old_token = new_token;
         }
@@ -585,7 +581,7 @@ fn bigram_overlay_coherence_full_stress_loop_with_overflow() {
             write_file_with_token(base, &name, &token);
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(&name));
+            picker.handle_create_or_modify(base.join(&name));
             live_overflow.push((name, token));
         }
 
@@ -648,7 +644,7 @@ fn bigram_overlay_coherence_overflow_file_edit_and_delete() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(&path);
+            picker.handle_create_or_modify(&path);
         }
         overflow_files.push((path, token));
     }
@@ -673,7 +669,7 @@ fn bigram_overlay_coherence_overflow_file_edit_and_delete() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(path);
+            picker.handle_create_or_modify(path);
         }
         edited_tokens.push(new_token);
     }
@@ -689,14 +685,19 @@ fn bigram_overlay_coherence_overflow_file_edit_and_delete() {
         }
     }
 
-    // Verify 5 overflow remain.
+    // Verify 5 overflow remain live (tombstones still occupy slots by design —
+    // StableVec never shifts, so get_overflow_files().len() stays at 10).
     {
         let guard = shared_picker.read().unwrap();
         let picker = guard.as_ref().unwrap();
+        let live = picker
+            .get_overflow_files()
+            .iter()
+            .filter(|f| !f.is_deleted())
+            .count();
         assert_eq!(
-            picker.get_overflow_files().len(),
-            5,
-            "should have 5 overflow files after deleting 5"
+            live, 5,
+            "should have 5 live overflow files after deleting 5"
         );
     }
 
@@ -737,7 +738,7 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         edited_tokens.push(token);
     }
@@ -750,7 +751,7 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(&name));
+            picker.handle_create_or_modify(base.join(&name));
         }
         new_tokens.push(token);
     }
@@ -801,14 +802,6 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
                 with >= 1,
                 "post-rescan: edited token {token} should be findable"
             );
-
-            // The content is now in the base index, so it should be
-            // findable even without the overlay.
-            let without = grep_without_overlay_count(picker, token);
-            assert!(
-                without >= 1,
-                "post-rescan: {token} should be in base index (without overlay: {without})"
-            );
         }
 
         for token in &new_tokens {
@@ -851,7 +844,7 @@ fn bigram_overlay_coherence_full_lifecycle_seed_edit_commit_rescan_edit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         phase1_tokens.push(token);
     }
@@ -930,7 +923,7 @@ fn bigram_overlay_coherence_full_lifecycle_seed_edit_commit_rescan_edit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         phase3_tokens.push(token);
     }
@@ -1009,7 +1002,7 @@ fn bigram_overlay_coherence_nested_directory_edits() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         edited.push(token);
     }
@@ -1127,7 +1120,7 @@ fn bigram_overlay_coherence_fuzzy_search_base_overflow_and_deleted() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join("controller_admin.rs"));
+        picker.handle_create_or_modify(base.join("controller_admin.rs"));
     }
 
     // Fuzzy search should find the new overflow file.
@@ -1174,7 +1167,7 @@ fn bigram_overlay_coherence_fuzzy_search_after_rescan() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join("router_grpc.rs"));
+        picker.handle_create_or_modify(base.join("router_grpc.rs"));
     }
 
     let web_path = base.join("router_web.rs");
@@ -1239,7 +1232,7 @@ fn bigram_overlay_coherence_fuzzy_and_grep_combined() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join(edit_name));
+        picker.handle_create_or_modify(base.join(edit_name));
     }
 
     // Add an overflow file with a distinctive name.
@@ -1251,7 +1244,7 @@ fn bigram_overlay_coherence_fuzzy_and_grep_combined() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join("unique_overflow_widget.rs"));
+        picker.handle_create_or_modify(base.join("unique_overflow_widget.rs"));
     }
 
     // Delete a base file.
@@ -1334,11 +1327,6 @@ fn grep_count(picker: &FilePicker, query: &str) -> usize {
     picker.grep(&parsed, &grep_opts()).matches.len()
 }
 
-fn grep_without_overlay_count(picker: &FilePicker, query: &str) -> usize {
-    let parsed = parse_grep_query(query);
-    picker.grep_original(&parsed, &grep_opts()).matches.len()
-}
-
 /// Wait for scanning to finish (no bigram requirement).
 /// Use after `trigger_rescan` which replaces sync_data but does not
 /// rebuild the bigram index.
@@ -1635,7 +1623,7 @@ fn bigram_overlay_coherence_fuzzy_grep_finds_overflow_files() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&new_path).is_some());
+        assert!(picker.handle_create_or_modify(&new_path).is_some());
         assert_eq!(picker.get_overflow_files().len(), 1);
     }
 

crates/fff-core/tests/bigram_overlay_integration.rs

@@ -113,7 +113,7 @@ fn modified_file_findable_via_overlay() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        let result = picker.on_create_or_modify(&modified_path);
+        let result = picker.handle_create_or_modify(&modified_path);
         assert!(
             result.is_some(),
             "on_create_or_modify should return the file"
@@ -144,21 +144,6 @@ fn modified_file_findable_via_overlay() {
         );
     }
 
-    // Prove the overlay is actually doing something: without it, the bigram
-    // index would filter out beta.txt and the search would miss the needle.
-    {
-        let guard = shared_picker.read().unwrap();
-        let picker = guard.as_ref().unwrap();
-        let parsed = parse_grep_query("UNIQUE_NEEDLE");
-        let opts = grep_opts();
-        let result = picker.grep_original(&parsed, &opts);
-        assert_eq!(
-            result.matches.len(),
-            0,
-            "Without overlay, bigram prefiltering should exclude the modified file"
-        );
-    }
-
     // Cleanup: stop background watcher.
     if let Ok(mut guard) = shared_picker.write() {
         if let Some(ref mut picker) = *guard {
@@ -273,7 +258,7 @@ fn new_file_findable_after_add() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        let result = picker.on_create_or_modify(&new_path);
+        let result = picker.handle_create_or_modify(&new_path);
         assert!(
             result.is_some(),
             "on_create_or_modify should return the new file"
@@ -350,7 +335,7 @@ fn modified_file_findable_via_regex_overlay() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&modified_path).is_some());
+        assert!(picker.handle_create_or_modify(&modified_path).is_some());
     }
 
     // Regex grep should find the modified file through the overlay.

crates/fff-core/tests/dir_index_consistency_test.rs

@@ -0,0 +1,301 @@
+use fff_search::file_picker::{FFFMode, FilePicker};
+use fff_search::{
+    DirSearchConfig, FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser,
+    SharedFilePicker, SharedFrecency,
+};
+use std::fs;
+use std::path::Path;
+use std::time::{Duration, Instant};
+use tempfile::TempDir;
+
+fn make_watched_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) {
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::noop();
+
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().into_owned(),
+            enable_mmap_cache: false,
+            enable_content_indexing: false,
+            mode: FFFMode::Neovim,
+            watch: true,
+            ..Default::default()
+        },
+    )
+    .expect("FilePicker::new_with_shared_state");
+
+    assert!(
+        shared_picker.wait_for_scan(Duration::from_secs(30)),
+        "initial scan did not complete"
+    );
+    assert!(
+        shared_picker.wait_for_watcher(Duration::from_secs(30)),
+        "watcher did not install"
+    );
+    // macOS FSEvents streams need a beat before they deliver reliably
+    std::thread::sleep(Duration::from_millis(300));
+
+    (shared_picker, shared_frecency)
+}
+
+fn search_dirs(picker: &SharedFilePicker, query: &str) -> Vec<String> {
+    let guard = picker.read().expect("picker read lock");
+    let p = guard.as_ref().expect("picker initialized");
+    let parser = QueryParser::new(DirSearchConfig);
+    let parsed = parser.parse(query);
+    let results = p.fuzzy_search_directories(
+        &parsed,
+        FuzzySearchOptions {
+            pagination: PaginationArgs {
+                offset: 0,
+                limit: 100,
+            },
+            ..Default::default()
+        },
+    );
+    results.items.iter().map(|d| d.relative_path(p)).collect()
+}
+
+fn wait_until<F: Fn() -> bool>(cond: F, timeout: Duration) -> bool {
+    let deadline = Instant::now() + timeout;
+    while Instant::now() < deadline {
+        if cond() {
+            return true;
+        }
+        std::thread::sleep(Duration::from_millis(50));
+    }
+    cond()
+}
+
+#[test]
+fn removed_directory_disappears_from_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("doomed/nested")).unwrap();
+    fs::write(base.join("doomed/a.rs"), "x").unwrap();
+    fs::write(base.join("doomed/nested/b.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(
+        search_dirs(&picker, "doomed")
+            .iter()
+            .any(|d| d.starts_with("doomed")),
+        "sanity: dir indexed after scan"
+    );
+
+    fs::remove_dir_all(base.join("doomed")).unwrap();
+
+    assert!(
+        wait_until(
+            || !search_dirs(&picker, "doomed")
+                .iter()
+                .any(|d| d.starts_with("doomed")),
+            Duration::from_secs(10)
+        ),
+        "removed dir must disappear from dir search, got: {:?}",
+        search_dirs(&picker, "doomed")
+    );
+}
+
+#[test]
+fn moved_out_directory_disappears_from_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let trash = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("doomed/nested")).unwrap();
+    fs::write(base.join("doomed/a.rs"), "x").unwrap();
+    fs::write(base.join("doomed/nested/b.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(
+        search_dirs(&picker, "doomed")
+            .iter()
+            .any(|d| d.starts_with("doomed")),
+        "sanity: dir indexed after scan"
+    );
+
+    fs::rename(base.join("doomed"), trash.path().join("doomed")).unwrap();
+
+    assert!(
+        wait_until(
+            || !search_dirs(&picker, "doomed")
+                .iter()
+                .any(|d| d.starts_with("doomed")),
+            Duration::from_secs(10)
+        ),
+        "moved-out dir must disappear from dir search, got: {:?}",
+        search_dirs(&picker, "doomed")
+    );
+}
+
+#[test]
+fn moved_in_directory_appears_in_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let staging = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let incoming = staging.path().join("arrived");
+    fs::create_dir_all(incoming.join("nested")).unwrap();
+    fs::write(incoming.join("a.rs"), "x").unwrap();
+    fs::write(incoming.join("nested/b.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(search_dirs(&picker, "arrived").is_empty(), "sanity");
+
+    fs::rename(&incoming, base.join("arrived")).unwrap();
+
+    assert!(
+        wait_until(
+            || {
+                let dirs = search_dirs(&picker, "arrived");
+                dirs.iter().any(|d| d.starts_with("arrived"))
+            },
+            Duration::from_secs(10)
+        ),
+        "moved-in dir must appear in dir search, got: {:?}",
+        search_dirs(&picker, "arrived")
+    );
+}
+
+#[test]
+fn new_file_in_new_directory_surfaces_the_dir() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(search_dirs(&picker, "brandnew").is_empty(), "sanity");
+
+    fs::create_dir_all(base.join("brandnew")).unwrap();
+    fs::write(base.join("brandnew/file.rs"), "x").unwrap();
+
+    assert!(
+        wait_until(
+            || search_dirs(&picker, "brandnew")
+                .iter()
+                .any(|d| d.starts_with("brandnew")),
+            Duration::from_secs(10)
+        ),
+        "new dir must appear in dir search, got: {:?}",
+        search_dirs(&picker, "brandnew")
+    );
+}
+
+#[test]
+fn deleting_last_file_keeps_directory_visible() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("lonely")).unwrap();
+    fs::write(base.join("lonely/only.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+
+    // the file goes away but the directory itself still exists on disk
+    fs::remove_file(base.join("lonely/only.rs")).unwrap();
+
+    assert!(
+        wait_until(
+            || {
+                let guard = picker.read().unwrap();
+                let p = guard.as_ref().unwrap();
+                p.get_file_by_path(base.join("lonely/only.rs"))
+                    .is_none_or(|f| f.is_deleted())
+            },
+            Duration::from_secs(10)
+        ),
+        "file removal must be applied"
+    );
+    assert!(
+        search_dirs(&picker, "lonely")
+            .iter()
+            .any(|d| d.starts_with("lonely")),
+        "dir still exists on disk and must stay searchable"
+    );
+}
+
+#[test]
+fn recreated_directory_reappears_in_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("phoenix")).unwrap();
+    fs::write(base.join("phoenix/a.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+
+    fs::remove_dir_all(base.join("phoenix")).unwrap();
+    assert!(
+        wait_until(
+            || !search_dirs(&picker, "phoenix")
+                .iter()
+                .any(|d| d.starts_with("phoenix")),
+            Duration::from_secs(10)
+        ),
+        "dir must disappear after removal"
+    );
+
+    fs::create_dir_all(base.join("phoenix")).unwrap();
+    fs::write(base.join("phoenix/a.rs"), "x").unwrap();
+
+    assert!(
+        wait_until(
+            || search_dirs(&picker, "phoenix")
+                .iter()
+                .any(|d| d.starts_with("phoenix")),
+            Duration::from_secs(10)
+        ),
+        "recreated dir must reappear in dir search, got: {:?}",
+        search_dirs(&picker, "phoenix")
+    );
+}
+
+/// Regression for #725: a dir that is EMPTY at scan time must be indexed —
+/// searchable in dir search and watched so later file creations are seen.
+#[test]
+fn empty_directory_at_scan_is_searchable_and_watched() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("commands")).unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+
+    assert!(
+        search_dirs(&picker, "commands")
+            .iter()
+            .any(|d| d.starts_with("commands")),
+        "empty dir must be searchable right after the scan, got: {:?}",
+        search_dirs(&picker, "commands")
+    );
+
+    // The empty dir must reuse its scan-built DirItem when a file lands in it
+    // and the watcher must have registered a watch on it (the #725 repro).
+    fs::write(base.join("commands/review.md"), "# review").unwrap();
+    assert!(
+        wait_until(
+            || {
+                let guard = picker.read().unwrap();
+                let p = guard.as_ref().unwrap();
+                p.get_file_by_path(base.join("commands/review.md"))
+                    .is_some()
+            },
+            Duration::from_secs(10)
+        ),
+        "file created in a scan-time-empty dir must be indexed"
+    );
+
+    let guard = picker.read().unwrap();
+    let p = guard.as_ref().unwrap();
+    let commands_dirs = p
+        .get_dirs()
+        .iter()
+        .filter(|d| d.relative_path(p).starts_with("commands"))
+        .count();
+    assert_eq!(commands_dirs, 1, "no duplicate DirItem for the empty dir");
+}

crates/fff-core/tests/drop_during_post_scan.rs

@@ -0,0 +1,161 @@
+// Regression pinning: dropping a picker during poset scan off-lock time
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+use tempfile::TempDir;
+
+use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions};
+use fff_search::{FilePickerOptions, QueryParser, SharedFilePicker, SharedFrecency};
+
+fn seed_files(dir: &Path, count: usize) {
+    for i in 0..count {
+        let subdir = dir.join(format!("dir_{}", i / 20));
+        fs::create_dir_all(&subdir).unwrap();
+        fs::write(
+            subdir.join(format!("file_{i}.rs")),
+            format!("pub fn func_{i}() {{ /* token_{i} */ }}\n"),
+        )
+        .unwrap();
+    }
+}
+
+fn git_init(dir: &Path) {
+    let run = |args: &[&str]| {
+        Command::new("git")
+            .args(args)
+            .current_dir(dir)
+            .env("GIT_AUTHOR_NAME", "test")
+            .env("GIT_AUTHOR_EMAIL", "t@t")
+            .env("GIT_COMMITTER_NAME", "test")
+            .env("GIT_COMMITTER_EMAIL", "t@t")
+            .output()
+            .unwrap();
+    };
+    run(&["init"]);
+    run(&["add", "-A"]);
+    run(&["commit", "-m", "init"]);
+}
+
+fn make_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) {
+    let sp = SharedFilePicker::default();
+    let sf = SharedFrecency::default();
+    FilePicker::new_with_shared_state(
+        sp.clone(),
+        sf.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: true,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("init");
+    (sp, sf)
+}
+
+/// Drop picker immediately after scan starts — scan thread will find
+/// the picker gone and exit cleanly.
+#[test]
+fn drop_picker_during_walk_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 500);
+    git_init(tmp.path());
+
+    let (sp, _sf) = make_picker(tmp.path());
+    // Don't wait — drop immediately while walk is likely in progress
+    drop(sp);
+
+    // If we get here without SIGSEGV, the test passes.
+    std::thread::sleep(Duration::from_millis(200));
+}
+
+/// Drop picker while post-scan indexing is running. The snapshot holds
+/// Arc clones that keep the buffers alive.
+#[test]
+fn drop_picker_during_post_scan_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 500);
+    git_init(tmp.path());
+
+    let (sp, _sf) = make_picker(tmp.path());
+
+    // Wait for walk to finish (files are searchable) but post-scan is
+    // still running (bigram not yet built).
+    sp.wait_for_scan(Duration::from_secs(10));
+
+    // At this point post_scan_indexing_active is likely true.
+    // Drop the picker — this releases the picker's Arc clones, but the
+    // post-scan snapshot's clones keep the buffers alive.
+    if let Ok(mut guard) = sp.write() {
+        guard.take(); // drop the FilePicker
+    }
+
+    // Give post-scan threads time to run against the "dead" picker.
+    // They must not segfault.
+    std::thread::sleep(Duration::from_secs(2));
+}
+
+/// Drop picker from a second thread while the first thread is doing
+/// fuzzy searches. Verifies no segfault from interleaved access.
+#[test]
+fn drop_picker_concurrent_with_search_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 500);
+    git_init(tmp.path());
+
+    let (sp, _sf) = make_picker(tmp.path());
+    sp.wait_for_scan(Duration::from_secs(10));
+
+    let sp_clone = sp.clone();
+    let running = Arc::new(AtomicBool::new(true));
+    let running_clone = running.clone();
+
+    // Searcher thread: continuously queries while the picker lives
+    let searcher = std::thread::spawn(move || {
+        let parser = QueryParser::default();
+        while running_clone.load(Ordering::Relaxed) {
+            if let Ok(guard) = sp_clone.read() {
+                if let Some(picker) = guard.as_ref() {
+                    let query = parser.parse("func");
+                    let _ = picker.fuzzy_search(&query, None, FuzzySearchOptions::default());
+                }
+            }
+            std::thread::sleep(Duration::from_millis(1));
+        }
+    });
+
+    // Let searches run for a bit, then drop
+    std::thread::sleep(Duration::from_millis(100));
+    if let Ok(mut guard) = sp.write() {
+        guard.take();
+    }
+    std::thread::sleep(Duration::from_millis(100));
+
+    running.store(false, Ordering::Relaxed);
+    searcher.join().unwrap();
+}
+
+/// Repeated init + wait + clean-drop cycle. This is the pattern that
+/// SIGSEGV'd on the pre-refactor code in the benchmark.
+#[test]
+fn repeated_init_and_drop_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 200);
+    git_init(tmp.path());
+
+    for _ in 0..5 {
+        let (sp, _sf) = make_picker(tmp.path());
+        sp.wait_for_scan(Duration::from_secs(10));
+        sp.wait_for_indexing_complete(Duration::from_secs(30));
+        if let Ok(mut guard) = sp.write()
+            && let Some(mut picker) = guard.take()
+        {
+            picker.stop_background_monitor();
+        }
+    }
+}

crates/fff-core/tests/fuzz_file_operations.rs

@@ -1,14 +1,3 @@
-//! Randomized file-system mutation stress test.
-//!
-//! Seeds a directory with ~40 files across diverse content domains, builds the
-//! picker + bigram index, then runs 20 rounds of randomized create / edit /
-//! delete / rename / read-only operations. After every round the test verifies
-//! that plain-text grep, regex grep, and fuzzy file search all return correct
-//! results for every live and dead file.
-//!
-//! Uses a seeded RNG (`SmallRng::seed_from_u64`) for deterministic
-//! reproduction.
-
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -379,7 +368,7 @@ fn fuzz_file_operations_stress() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(name)).is_some(),
+                    picker.handle_create_or_modify(base.join(name)).is_some(),
                     "round {round}: on_create_or_modify({name}) should succeed for edit"
                 );
             }
@@ -399,7 +388,7 @@ fn fuzz_file_operations_stress() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(&name)).is_some(),
+                    picker.handle_create_or_modify(base.join(&name)).is_some(),
                     "round {round}: on_create_or_modify({name}) should succeed for create"
                 );
             }
@@ -453,7 +442,9 @@ fn fuzz_file_operations_stress() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(&new_name)).is_some(),
+                    picker
+                        .handle_create_or_modify(base.join(&new_name))
+                        .is_some(),
                     "round {round}: on_create_or_modify({new_name}) should succeed for rename"
                 );
             }
@@ -766,3 +757,98 @@ fn git_init_and_commit(dir: &Path) {
     git_run(dir, &["add", "-A"]);
     git_run(dir, &["commit", "-m", "initial"]);
 }
+
+/// Proves that dropping the picker while post-scan (warmup + bigram build)
+/// is actively iterating raw pointers does NOT segfault. The Drop impl
+/// sets `cancelled`, waits for `post_scan_indexing_active` to clear, and
+/// only then frees the backing Vec.
+///
+/// Runs 10 iterations to exercise the race window reliably.
+#[test]
+fn drop_during_post_scan_does_not_crash() {
+    let mut caught_active = 0u32;
+
+    for round in 0..10 {
+        let tmp = TempDir::new().unwrap();
+        let base = tmp.path();
+
+        // Create enough files so bigram build takes measurable time
+        for i in 0..2000 {
+            let dir = base.join(format!("d_{:02}", i % 20));
+            fs::create_dir_all(&dir).unwrap();
+            let content = format!(
+                "fn func_{i}() {{ let x = {i}; println!(\"{{x}}\"); }}\n\
+                 const T_{i}: &str = \"TOKEN_{i}\";\n"
+            );
+            fs::write(dir.join(format!("f_{i:04}.rs")), content).unwrap();
+        }
+
+        git_init_and_commit(base);
+
+        let shared_picker = SharedFilePicker::default();
+
+        FilePicker::new_with_shared_state(
+            shared_picker.clone(),
+            SharedFrecency::noop(),
+            FilePickerOptions {
+                base_path: base.to_string_lossy().to_string(),
+                enable_mmap_cache: true,
+                enable_content_indexing: true,
+                watch: false,
+                mode: FFFMode::Neovim,
+                ..Default::default()
+            },
+        )
+        .unwrap();
+
+        // Wait for scan but NOT for bigram — drop while post-scan is active
+        shared_picker.wait_for_scan(Duration::from_secs(10));
+
+        // Poll until post_scan_indexing_active is true (bigram started)
+        let deadline = std::time::Instant::now() + Duration::from_secs(5);
+        let mut was_active = false;
+        loop {
+            if let Ok(guard) = shared_picker.read() {
+                if let Some(picker) = guard.as_ref() {
+                    if picker.is_post_scan_active() {
+                        was_active = true;
+                        break;
+                    }
+                }
+            }
+            if std::time::Instant::now() > deadline {
+                break;
+            }
+            std::thread::sleep(Duration::from_millis(1));
+        }
+
+        if was_active {
+            caught_active += 1;
+        }
+
+        // Drop the picker while post_scan_indexing_active is set.
+        // Take it out of the shared handle first, then drop outside the lock —
+        // Drop spins until post-scan finishes, which needs the write lock for
+        // bigram install, so we can't hold it during Drop.
+        let old_picker = shared_picker.write().unwrap().take();
+        drop(old_picker); // Drop fires here — spins until post-scan exits
+
+        assert!(
+            shared_picker.read().unwrap().is_none(),
+            "round {round}: picker should be None after drop"
+        );
+    }
+
+    // The primary invariant — dropping while post-scan may be active must not
+    // crash — is exercised every round regardless. Catching the active window
+    // is timing-dependent: with a fast walker/scan the post-scan phase can
+    // complete before the poll observes it, especially on loaded CI runners.
+    // So we only warn (not fail) if no round observed it.
+    if caught_active == 0 {
+        eprintln!(
+            "warning: never observed post_scan_indexing_active=true; \
+             drop-safety was still exercised in all rounds ({caught_active}/10)"
+        );
+    }
+    eprintln!("Caught post-scan active in {caught_active}/10 rounds");
+}

crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions

@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 2c9d1ea2efbf6161f84b69598e884dbf1bde6039c70625adde0374817e20e2ea
+cc 1ac0f8f02b160dce13ca3f3630266abd24bd32b4e36d72a6a0a5365139ded3a8

crates/fff-core/tests/fuzz_git_watcher_stress.rs

@@ -192,8 +192,10 @@ fn op_strategy() -> impl Strategy<Value = AbstractOp> {
 }
 
 fn ops_strategy() -> impl Strategy<Value = Vec<AbstractOp>> {
-    let min = stress_min_ops();
-    let max = stress_max_ops();
+    ops_strategy_bounded(stress_min_ops(), stress_max_ops())
+}
+
+fn ops_strategy_bounded(min: usize, max: usize) -> impl Strategy<Value = Vec<AbstractOp>> {
     prop::collection::vec(op_strategy(), min..=max)
 }
 
@@ -285,6 +287,65 @@ fn stress_seeded() {
     }
 }
 
+/// Pinned deterministic regression for the git-status divergence found on
+/// Windows CI (run 28264744320): after a `GitCommit` the picker retained stale
+/// `INDEX_*` bits because a pre-commit per-path status snapshot was applied
+/// after the post-commit full rescan.
+///
+/// The op sequence is regenerated from the proptest seed persisted in the
+/// regressions file (`cc 2c9d...`) using the CI op bounds (30..=60) that were
+/// in effect when the failure was found. The fingerprint assertion fails
+/// loudly if `ops_strategy()` ever changes shape — a changed strategy would
+/// silently decode the same seed into a *different* scenario, turning this
+/// regression guard into a no-op.
+#[test]
+fn stress_regression_stale_index_after_commit() {
+    let ops = ops_from_chacha_seed(REGRESSION_SEED_HEX, 30, 60);
+    assert_eq!(
+        (ops.len(), fingerprint_ops(&ops)),
+        (59, 0xc73f_16ce_b249_78eb),
+        "ops_strategy() changed shape: the pinned seed no longer decodes to \
+         the original Windows-CI scenario. Either revert the strategy change \
+         or re-pin this regression (the original literal op list is in git \
+         history of this file).",
+    );
+    run_stress_scenario(&ops);
+}
+
+/// 32-byte ChaCha seed persisted by proptest for the Windows CI failure
+/// (the `cc 2c9d...` entry in the regressions file).
+const REGRESSION_SEED_HEX: &str =
+    "2c9d1ea2efbf6161f84b69598e884dbf1bde6039c70625adde0374817e20e2ea";
+
+/// Regenerate an op sequence from a persisted proptest ChaCha seed by
+/// replaying `ops_strategy()` the same way proptest does for regressions.
+/// `min`/`max` must match the `FFF_STRESS_{MIN,MAX}_OPS` bounds that were
+/// in effect when the seed was persisted — the strategy's value tree
+/// depends on them.
+fn ops_from_chacha_seed(seed_hex: &str, min: usize, max: usize) -> Vec<AbstractOp> {
+    let seed_bytes: Vec<u8> = (0..seed_hex.len() / 2)
+        .map(|i| u8::from_str_radix(&seed_hex[2 * i..2 * i + 2], 16).expect("valid hex seed"))
+        .collect();
+    let mut config = proptest_config();
+    config.failure_persistence = Some(Box::new(FileFailurePersistence::Off));
+    let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &seed_bytes);
+    let mut runner = TestRunner::new_with_rng(config, rng);
+    ops_strategy_bounded(min, max)
+        .new_tree(&mut runner)
+        .expect("ops_strategy::new_tree")
+        .current()
+}
+
+/// FNV-1a over the debug repr of the ops; stable across platforms and runs.
+fn fingerprint_ops(ops: &[AbstractOp]) -> u64 {
+    let mut h = 0xcbf2_9ce4_8422_2325u64;
+    for b in format!("{ops:?}").bytes() {
+        h ^= b as u64;
+        h = h.wrapping_mul(0x0000_0100_0000_01b3);
+    }
+    h
+}
+
 /// Parse `FFF_STRESS_SEED` as either decimal or `0x`-prefixed hex.
 fn parse_stress_seed() -> u64 {
     match std::env::var("FFF_STRESS_SEED") {
@@ -712,7 +773,7 @@ fn read_truth_status(base: &Path) -> BTreeMap<String, Status> {
 
     let mut out = BTreeMap::new();
     for entry in statuses.iter() {
-        if let Some(p) = entry.path() {
+        if let Ok(p) = entry.path() {
             // git2 returns forward-slash paths; accept as-is.
             out.insert(p.to_string(), entry.status());
         }
@@ -912,6 +973,73 @@ fn grep_plain_matches(shared: &SharedFilePicker, query: &str) -> Vec<String> {
         .collect()
 }
 
+/// Run live grep (fuzzy mode) and return matched file paths.
+/// Exercises the `fuzzy_grep_search` code path which resolves content
+/// via arena pointers — the path that was silently broken for overflow
+/// files before the overflow_arena fix.
+fn grep_fuzzy_matches(shared: &SharedFilePicker, query: &str) -> Vec<String> {
+    let guard = match shared.read() {
+        Ok(g) => g,
+        Err(_) => return Vec::new(),
+    };
+    let Some(picker) = guard.as_ref() else {
+        return Vec::new();
+    };
+    let parsed = parse_grep_query(query);
+    let opts = GrepSearchOptions {
+        max_file_size: 10 * 1024 * 1024,
+        max_matches_per_file: 200,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 500,
+        mode: GrepMode::Fuzzy,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: None,
+    };
+    let result = picker.grep(&parsed, &opts);
+    result
+        .files
+        .iter()
+        .map(|f| normalize(f.relative_path(picker)))
+        .collect()
+}
+
+/// Run live grep (regex mode) and return matched file paths.
+fn grep_regex_matches(shared: &SharedFilePicker, query: &str) -> Vec<String> {
+    let guard = match shared.read() {
+        Ok(g) => g,
+        Err(_) => return Vec::new(),
+    };
+    let Some(picker) = guard.as_ref() else {
+        return Vec::new();
+    };
+    let parsed = parse_grep_query(query);
+    let opts = GrepSearchOptions {
+        max_file_size: 10 * 1024 * 1024,
+        max_matches_per_file: 200,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 500,
+        mode: GrepMode::Regex,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: None,
+    };
+    let result = picker.grep(&parsed, &opts);
+    result
+        .files
+        .iter()
+        .map(|f| normalize(f.relative_path(picker)))
+        .collect()
+}
+
 /// Report from [`probe_real_queries`]. `None` means "nothing to probe this
 /// round" (empty live set). `Some(Err)` means a probe disagreed with truth
 /// — convergence should not treat this as success.
@@ -952,12 +1080,21 @@ fn probe_real_queries(shared: &SharedFilePicker, live: &[Live]) -> ProbeOutcome
         }
     }
 
-    // --- Grep probe: search for the content marker ---
+    // --- Grep probe: search for the content marker using a randomly
+    // rotated grep strategy. Each round picks one of PlainText / Fuzzy /
+    // Regex so over many rounds all three code paths get exercised,
+    // including the overflow-arena resolution that was previously broken
+    // in fuzzy grep.
     if let Some(marker) = extract_marker(&target.abs) {
-        let matches = grep_plain_matches(shared, &marker);
+        let probe_round = PROBE_COUNTER.load(Ordering::Relaxed);
+        let (mode_name, matches) = match probe_round % 3 {
+            0 => ("plain", grep_plain_matches(shared, &marker)),
+            1 => ("fuzzy", grep_fuzzy_matches(shared, &marker)),
+            _ => ("regex", grep_regex_matches(shared, &marker)),
+        };
         if !matches.contains(&target.relative) {
             return Some(Err(format!(
-                "grep({marker:?}) did not return expected live file {:?}\n\
+                "grep[{mode_name}]({marker:?}) did not return expected live file {:?}\n\
                  got {} matched files; first few: {:?}",
                 target.relative,
                 matches.len(),
@@ -1121,7 +1258,7 @@ fn get_baseline_status_from_git(base: &Path) -> Vec<Live> {
         Err(_) => return out,
     };
     for entry in statuses.iter() {
-        if let Some(p) = entry.path() {
+        if let Ok(p) = entry.path() {
             let abs = base.join(p);
             // Must be a real file *right now* — ignore stale WT_DELETED rows.
             if abs.is_file() {