chore(ci): Attempt to optimize CI build time

* fix: apply standalone constraints in multi_grep

* chore: use pub(crate)

---------

Co-authored-by: Dmitriy Kovalenko <dmitriy@iusevimbtw.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,58 @@
+name: Bug report
+description: Report a crash, hang, or incorrect behavior in fff (any frontend — nvim plugin, Node/Bun SDK, MCP server, C SDK).
+title: "[Bug]: "
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: frontend
+    attributes:
+      label: Which fff frontend?
+      options:
+        - Neovim plugin (fff.nvim)
+        - MCP server (fff-mcp)
+        - Node SDK (@ff-labs/fff-node)
+        - Bun SDK
+        - C SDK (libfff)
+        - Other / multiple
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: has logs
+      description: |
+        Attach your fff log file — the single most useful thing for debugging.
+
+        fff writes a fresh log file on every process startup, named `fff+<unix-ts>+<pid>.log`, and keeps the last 20. Find the file matching your crashed/buggy run and drag-and-drop it here (or paste its contents).
+
+        Where the log files live:
+
+        | Frontend | Linux / macOS | Windows |
+        |---|---|---|
+        | Neovim plugin | `~/.local/state/nvim/log/fff+*.log` | `%LOCALAPPDATA%\nvim-data\log\fff+*.log` |
+        | MCP server (`fff-mcp`) | `~/.cache/fff_mcp+*.log` (override with `--log-file`) | `%LOCALAPPDATA%\fff_mcp+*.log` |
+        | Node / Bun SDK | path you passed as `logFilePath` to `FileFinder.create({...})` | same |
+        | C SDK | path you passed as `log_file_path` in `FffCreateOptions` | same |
+
+        Neovim users: run `:FFFOpenLog` to open the current session's log directly.
+
+        Also some useful commands:
+
+        ```sh
+        # Neovim plugin
+        ls -t ~/.local/state/nvim/log/fff+*.log | head -1
+
+        # MCP server
+        ls -t ~/.cache/fff_mcp+*.log | head -1
+        ```
+    validations:
+      required: false
+
+  - type: textarea
+    id: body
+    attributes:
+      label: Description
+      description: Please provide as much helpful information as you can
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Discussion / question
+    url: https://github.com/dmtrKovalenko/fff/discussions
+    about: For usage questions, design discussion, or anything that's not a bug or feature request.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,34 @@
+name: Feature request
+description: Suggest something new for fff
+title: "[Suggestion]: "
+labels: ["enhancement"]
+body:
+  - type: dropdown
+    id: frontend
+    attributes:
+      label: Which fff frontend(s)?
+      multiple: true
+      options:
+        - Neovim plugin (fff.nvim)
+        - MCP server (fff-mcp)
+        - Node SDK (@ff-labs/fff-node)
+        - Bun SDK (@ff-labs/fff-bun)
+        - C lib (libfff)
+        - Core or Rust crate
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem are you trying to solve?
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: If you have an idea of the shape of the API, describe it here.
+    validations:
+      required: false

.github/workflows/external-tests.yml

@@ -3,8 +3,14 @@ name: e2e Tests
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 env:
   CARGO_TERM_COLOR: always
@@ -12,6 +18,10 @@ env:
   # Force Node 24 for all JS-based actions to avoid the libuv
   # process_title assertion crash on Windows (known Node 20 bug).
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  # e2e only needs a working binary, so skip fat LTO (same settings as the `ci`
+  # profile releases ship). Overriding release keeps artifacts in target/release.
+  CARGO_PROFILE_RELEASE_LTO: thin
+  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
 
 jobs:
   lua-tests:
@@ -26,7 +36,6 @@ jobs:
           - os: ubuntu-latest
           - os: macos-latest
           - os: windows-latest
-            target: x86_64-pc-windows-msvc
     steps:
       - uses: actions/checkout@v5
       - uses: oven-sh/setup-bun@v2
@@ -43,20 +52,13 @@ jobs:
           cache-on-failure: false
           cache-key: "v2-lua-e2e"
           rustflags: ""
-          target: ${{ matrix.target || '' }}
 
-      - name: Build Rust binary (Windows)
-        if: matrix.target
-        run: cargo build --release --target ${{ matrix.target }} -p fff-nvim --features zlob
-
-      - name: Copy binary to target/release (Windows)
-        if: matrix.target
+      - name: Build Rust binary
         shell: bash
-        run: |
-          cp target/${{ matrix.target }}/release/fff_nvim.dll target/release/fff_nvim.dll
+        run: make build-e2e
 
       - name: Verify Windows DLL has no unexpected dependencies
-        if: matrix.target
+        if: matrix.os == 'windows-latest'
         shell: pwsh
         run: |
           # Find dumpbin via vswhere (always available on GitHub Actions Windows runners)
@@ -72,10 +74,6 @@ jobs:
             exit 1
           }
 
-      - name: Build Rust binary
-        if: ${{ !matrix.target }}
-        run: cargo build --release -p fff-nvim --features zlob
-
       - name: Install Neovim
         uses: rhysd/action-setup-vim@v1
         with:
@@ -90,14 +88,36 @@ jobs:
         shell: bash
         run: make test-lua
 
+      - name: Dump fff trace log on failure
+        if: failure()
+        shell: bash
+        run: |
+          # init_tracing writes session files named fff-test+<ts>+<pid>.log
+          found=0
+          for f in fff-test*.log; do
+            [ -f "$f" ] || continue
+            found=1
+            echo "=== $f ==="
+            cat "$f"
+          done
+          if [ "$found" = 0 ]; then
+            echo "(no log file produced)"
+          fi
+
       - name: Run version resolution tests
         shell: bash
         run: make test-version
 
-      - name: Run bun tests
+      - name: Run non windows tests
         shell: bash
         if: ${{ matrix.os != 'windows-latest' }}
-        run: make test-bun
+        run: |
+          make test-bun
+          make test-c-api
+
+      - name: Verify bun --compile 
+        shell: bash
+        run: make test-bun-compile
 
       - name: Install Node.js
         if: ${{ matrix.os != 'ubuntu-latest' }}
@@ -112,3 +132,82 @@ jobs:
       - name: Run node tests
         shell: bash
         run: make test-node
+
+  # Regression for https://github.com/dmtrKovalenko/fff/issues/480: build &
+  # run @ff-labs/fff-node end-to-end on real Alpine Linux (musl). Forces
+  # findBinary() through the npm-package resolver so detectLinuxLibc()
+  # actually runs.
+  alpine-musl:
+    name: e2e (alpine-musl)
+    runs-on: ubuntu-latest
+    container: node:22-alpine
+    continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }}
+    defaults:
+      run:
+        shell: sh
+    steps:
+      - name: Install build deps
+        run: apk add --no-cache git rust cargo musl-dev
+
+      - uses: actions/checkout@v5
+
+      # libgit2 refuses repos owned by a different user; checkout in a
+      # container can land at a uid mismatch, so opt every dir in.
+      - name: Mark workspace safe for git
+        run: git config --global --add safe.directory '*'
+
+      - name: Sanity check libc is musl
+        run: |
+          if ! ldd --version 2>&1 | grep -qi musl; then
+            echo "FAIL: container is not running musl libc"
+            exit 1
+          fi
+
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: alpine-musl-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            alpine-musl-cargo-
+
+      - name: Build libfff_c (musl)
+        run: cargo build --release -p fff-c
+
+      - name: Install workspace npm deps
+        run: npm install --no-package-lock
+
+      # Upstream @yuuang/ffi-rs-linux-x64-musl ships with libc:"glibc" in
+      # its package.json (a publishing bug in ffi-rs), so npm filters it
+      # out. Force-install it so the FFI runtime is present on Alpine.
+      - name: Install ffi-rs musl runtime
+        run: |
+          FFI_RS_VERSION=$(node -p "require('ffi-rs/package.json').version")
+          npm install --no-package-lock --no-save --force \
+            "@yuuang/ffi-rs-linux-x64-musl@${FFI_RS_VERSION}"
+
+      # Stage the freshly built libfff_c.so as the platform npm package
+      # so findBinary() resolves through the @ff-labs/fff-bin-* path —
+      # this is what exercises detectLinuxLibc().
+      - name: Stage musl bin package
+        run: |
+          PKG_DIR=node_modules/@ff-labs/fff-bin-linux-x64-musl
+          mkdir -p "$PKG_DIR"
+          cp target/release/libfff_c.so "$PKG_DIR/libfff_c.so"
+          cat >"$PKG_DIR/package.json" <<'JSON'
+          { "name": "@ff-labs/fff-bin-linux-x64-musl", "version": "0.0.0" }
+          JSON
+
+      # fff-node builds with bun; the npm `bun` package ships musl binaries
+      - name: Install bun
+        run: npm install -g bun
+
+      - name: Build fff-node
+        working-directory: packages/fff-node
+        run: npm run build
+
+      - name: Run fff-node e2e suite
+        working-directory: packages/fff-node
+        run: node test/e2e.mjs

.github/workflows/lua.yml

@@ -3,8 +3,14 @@ name: Lua CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 jobs:
   lua-ls:

.github/workflows/nix.yml

@@ -3,8 +3,14 @@ name: Nix CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 jobs:
   check:

.github/workflows/panvimdoc.yaml

@@ -1,7 +1,8 @@
 on:
-  push:
-    branches-ignore:
-      - main
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
 name: docs
 
 jobs:
@@ -9,27 +10,76 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      pull-requests: write
     steps:
       - uses: actions/checkout@v5
         with:
-          # fetch last 2 commits required for auto force push back
+          ref: main
           fetch-depth: 2
 
+      - name: Extract Neovim section from README.md
+        run: |
+          awk '
+            /^<details id="neovim-plugin">/ { capture=1; next }
+            capture && /^<\/details>/        { capture=0; exit }
+            capture && /^<summary>$/         { next }
+            capture && /^<\/summary>$/       { next }
+            capture && /<h2>.*<\/h2>/ {
+              gsub(/<\/?h2>/, "")
+              sub(/^[[:space:]]+/, "")
+              print "# " $0
+              print ""
+              print "The best file search picker for Neovim. Frecency-ranked, typo-resistant, git-award, very fast."
+              print ""
+              next
+            }
+            capture { print }
+          ' README.md > .panvimdoc-input.md
+          test -s .panvimdoc-input.md
+
       - name: panvimdoc
         uses: kdheepak/panvimdoc@main
         with:
           vimdoc: fff.nvim
+          pandoc: .panvimdoc-input.md
           version: "Neovim >= 0.10.0"
           demojify: true
           treesitter: true
 
-      - name: Get last commit message
-        id: last-commit
-        run: |
-          echo "message=$(git log -1 --pretty=%s)" >> $GITHUB_OUTPUT
-          echo "author=$(git log -1 --pretty=\"%an <%ae>\")" >> $GITHUB_OUTPUT
+      - name: Cleanup intermediate file
+        run: rm -f .panvimdoc-input.md
 
-      - uses: stefanzweifel/git-auto-commit-action@v6
+      # panvimdoc stamps "Last change: <today>" every run, so a daily cron always
+      # produces a one-line diff. Skip the PR unless a non-date line changed.
+      - name: Detect real doc changes
+        id: docdiff
+        run: |
+          if git diff --quiet -I 'Last change:' -- doc/fff.nvim.txt; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create pull request
+        id: cpr
+        if: steps.docdiff.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
         with:
-          commit_author: ${{ steps.last-commit.outputs.author }}
-          commit_message: "chore: Update docs for - ${{ steps.last-commit.outputs.message }}"
+          branch: bot/regenerate-vimdoc
+          token: ${{ secrets.GUSTAV_PAT }}
+          delete-branch: true
+          title: "chore: regenerate Neovim vimdoc"
+          commit-message: |
+            chore: regenerate Neovim vimdoc
+
+            Co-authored-by: Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>
+          author: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>"
+          committer: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>"
+          body: Automated vimdoc regeneration from README.md, scribed by Gustav.
+          add-paths: doc/fff.nvim.txt
+
+      - name: Enable auto-merge
+        if: steps.cpr.outputs.pull-request-number
+        env:
+          GH_TOKEN: ${{ secrets.GUSTAV_PAT }}
+        run: gh pr merge --auto --squash "${{ steps.cpr.outputs.pull-request-number }}"

.github/workflows/python.yml

@@ -0,0 +1,56 @@
+name: Python CI
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+env:
+  CARGO_TERM_COLOR: always
+  MACOSX_DEPLOYMENT_TARGET: "13.0"
+  CARGO_PROFILE_RELEASE_LTO: thin
+  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
+
+jobs:
+  test:
+    name: Python bindings (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-python"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.11.14"
+          enable-cache: true
+
+      - name: Build and test Python bindings
+        working-directory: packages/fff-python
+        shell: bash
+        run: |
+          # --no-install-project: uv would otherwise build the wheel via
+          # maturin's PEP 517 backend, then `maturin develop` builds it again.
+          uv sync --all-extras --no-install-project
+          uv run maturin develop --release
+          uv run pytest -v

.github/workflows/release.yaml

@@ -1,11 +1,18 @@
-name: Prebuild
+name: Build & Publish
 
 on:
   push:
-    branches: [main, fix/download-version]
+    branches: [main, fix/use-trusted-publishing]
     tags:
       - "v*"
   pull_request:
+  workflow_dispatch:
+    inputs:
+      publish_pypi:
+        description: "Manually build and publish Python wheels to PyPI"
+        required: false
+        default: false
+        type: boolean
 
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
@@ -16,6 +23,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     permissions:
       contents: read
+      id-token: write
     strategy:
       matrix:
         include:
@@ -76,6 +84,13 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      # Cache the per-target build dir and cargo-zigbuild binary. Keyed by
+      # target so matrix legs don't collide. See issue on slow release CI.
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: nvim-${{ matrix.target }}
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
@@ -88,7 +103,7 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android')
         run: |
-          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for Android (Termux)
@@ -102,13 +117,13 @@ jobs:
           export AR_aarch64_linux_android="$NDK_BIN/llvm-ar"
           export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang"
 
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Ad-hoc sign macOS binary
@@ -119,7 +134,7 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Upload artifacts
@@ -168,6 +183,7 @@ jobs:
           - os: ubuntu-latest
             target: aarch64-linux-android
             artifact_name: target/aarch64-linux-android/ci/libfff_c.so
+            npm_package: fff-bin-android-arm64
             lib_filename: libfff_c.so
             ext: so
 
@@ -207,6 +223,11 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: c-${{ matrix.target }}
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
@@ -219,7 +240,7 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android')
         run: |
-          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for Android (Termux)
@@ -232,13 +253,13 @@ jobs:
           export AR_aarch64_linux_android="$NDK_BIN/llvm-ar"
           export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang"
 
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-c --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Ad-hoc sign macOS binary
@@ -249,11 +270,10 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Prepare npm package
-        if: "!contains(matrix.target, 'android')"
         shell: bash
         run: |
           # Copy the built binary into the platform npm package directory
@@ -266,7 +286,6 @@ jobs:
           path: c-lib-${{ matrix.target }}.*
 
       - name: Upload npm package artifact
-        if: "!contains(matrix.target, 'android')"
         uses: actions/upload-artifact@v4
         with:
           name: npm-${{ matrix.npm_package }}
@@ -320,6 +339,11 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: mcp-${{ matrix.target }}
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
@@ -332,13 +356,13 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu')
         run: |
-          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}"
 
       - name: Ad-hoc sign macOS binary
@@ -349,7 +373,7 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}.exe"
 
       - name: Upload artifact
@@ -358,16 +382,110 @@ jobs:
           name: mcp-${{ matrix.target }}
           path: fff-mcp-${{ matrix.target }}*
 
+  build-python:
+    name: Build Python wheels ${{ matrix.target }} (${{ matrix.os }})
+    # Wheels are release artifacts; PR validation uses the develop build in
+    # python.yml, so skip the cross-compile matrix on pull requests.
+    if: github.event_name != 'pull_request'
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            target: x86_64
+            container: "off"
+          - os: ubuntu-latest
+            target: aarch64
+            container: "off"
+          - os: macos-latest
+            target: x86_64
+          - os: macos-latest
+            target: aarch64
+          - os: windows-latest
+            target: x86_64
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install Zig
+        uses: mlugg/setup-zig@v2
+        with:
+          version: 0.16.0
+
+      - name: Install cargo-zigbuild
+        if: contains(matrix.os, 'ubuntu')
+        run: cargo install cargo-zigbuild
+
+      - name: Install aarch64 cross compiler
+        if: matrix.target == 'aarch64' && contains(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+          CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+          CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+          AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --no-default-features --features zlob
+          sccache: "true"
+          working-directory: packages/fff-python
+          container: ${{ matrix.container || '' }}
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-wheels-${{ matrix.os }}-${{ matrix.target }}
+          path: packages/fff-python/dist/
+
+  build-python-sdist:
+    name: Build Python sdist
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+          working-directory: packages/fff-python
+
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-sdist
+          path: packages/fff-python/dist/
+
   release:
     name: Release
-    needs: [build-nvim, build-c, build-mcp]
+    needs: [build-nvim, build-c, build-mcp, build-python, build-python-sdist]
     runs-on: ubuntu-latest
-    # do not create releases on the forks (no permissions)
-    if: github.event_name != 'pull_request' || github.event.pull_request.user.login == 'dmtrKovalenko'
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
     permissions:
       contents: write
     steps:
+      # we have to make sure that pushing a commit on this workflow triggers the CI for nightly neovim
       - uses: actions/checkout@v5
+        with:
+          token: ${{ secrets.GUSTAV_PAT || github.token }}
 
       - name: Install Lua
         uses: leafo/gh-actions-lua@v12
@@ -420,6 +538,20 @@ jobs:
             rmdir "$dir" 2>/dev/null || true
           done
 
+      - name: Move Python wheels to release directory
+        working-directory: ./binaries
+        run: |
+          mkdir -p python
+          for dir in python-wheels-*/ python-sdist/; do
+            [ -d "$dir" ] || continue
+            for file in "$dir"*; do
+              if [ -f "$file" ]; then
+                mv "$file" "python/$(basename "$file")"
+              fi
+            done
+            rmdir "$dir" 2>/dev/null || true
+          done
+
       - name: Remove npm package artifacts from release binaries
         working-directory: ./binaries
         run: |
@@ -429,7 +561,7 @@ jobs:
         working-directory: ./binaries
         run: |
           ls -la
-          for file in *; do
+          for file in * python/*; do
             if [ -f "$file" ] && [[ ! "$file" == *.sha256 ]]; then
               sha256sum "$file" > "${file}.sha256"
             fi
@@ -439,19 +571,32 @@ jobs:
         id: version
         run: lua scripts/determine-version.lua
 
+      # Nightlies publish to a permanent per-sha tag (release_tag == version) so
+      # pinned/stale installs always fetch the binary built for their own commit.
+      # The rolling `nightly` tag is also moved to HEAD for "give me latest" tooling.
+      - name: Move rolling nightly tag to current commit
+        if: steps.version.outputs.is_release != 'true'
+        run: |
+          git tag -f nightly "${{ github.sha }}"
+          git push -f origin refs/tags/nightly
+
       - name: Upload Release Assets
         uses: softprops/action-gh-release@v2
         with:
           name: "${{ steps.version.outputs.version }}"
-          tag_name: "${{ steps.version.outputs.is_release == 'true' && format('v{0}', steps.version.outputs.version) || steps.version.outputs.version }}"
+          tag_name: "${{ steps.version.outputs.release_tag }}"
           token: ${{ github.token }}
-          files: ./binaries/*
+          files: |
+            ./binaries/*
+            ./binaries/python/*
           draft: false
           prerelease: ${{ steps.version.outputs.is_release != 'true' }}
           generate_release_notes: ${{ steps.version.outputs.is_release == 'true' }}
           body: |
             ${{ steps.version.outputs.is_release == 'true' && format('Release {0}', steps.version.outputs.version) || format('Nightly release from commit: {0}', github.sha) }}
 
+            npm packages, rust crates and python wheels are available under this version ${{ steps.version.outputs.version }}
+
             ## Neovim Plugin
             - `{target}.so` / `.dylib` / `.dll` - Lua module for Neovim
 
@@ -461,21 +606,71 @@ jobs:
             ## MCP Server
             - `fff-mcp-{target}` - MCP server binary
 
-            Install with:
+            ## Python Package
+            - `python/*.whl` / `python/*.tar.gz` - Python wheels and sdist
+            - Install from PyPI: `pip install fff-search` (when published)
+
+            Update mcp via:
             ```sh
-            curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash
+            curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.sh | bash
             ```
 
+      - name: Bump Homebrew formula (uses local checksums)
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        run: make bump-homebrew-formula VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries
+
+      - name: Pin SHAs in install-mcp.sh (uses local checksums)
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        run: make bump-install-mcp-sh VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries
+
+      - name: Commit formula + installer bump to main
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          # Workflow runs on detached HEAD at the v* tag — explicit target needed.
+          branch: main
+          commit_message: "chore: bump fff-mcp release artifacts to v${{ steps.version.outputs.version }}"
+          file_pattern: "Formula/fff-mcp.rb install-mcp.sh"
+          commit_user_name: github-actions[bot]
+          commit_user_email: 41898282+github-actions[bot]@users.noreply.github.com
+
+  pypi-publish:
+    name: Publish Python wheels to PyPI
+    needs: [build-python, build-python-sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
+    environment:
+      name: pypi
+      url: https://pypi.org/p/fff-search
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Download Python wheels and sdist
+        uses: actions/download-artifact@v4
+        with:
+          pattern: python-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          skip-existing: true
+
   crates-publish:
     name: Publish Rust crates
     needs: [build-nvim, build-c, build-mcp]
     runs-on: ubuntu-latest
-    if: >-
-      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v')))
-      || (github.event_name == 'pull_request' && github.head_ref == 'main')
-
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v5
+      - uses: rust-lang/crates-io-auth-action@v1
+        id: auth
 
       - name: Install Lua
         uses: leafo/gh-actions-lua@v12
@@ -484,7 +679,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Install cargo-edit
-        run: cargo install cargo-edit
+        run: cargo install cargo-edit --force --locked
 
       - name: Determine version
         id: version
@@ -492,18 +687,17 @@ jobs:
 
       - name: Publish crates
         env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
         run: make publish-crates V="${{ steps.version.outputs.version }}"
 
   npm-publish:
     name: Publish npm packages
     needs: [build-c]
     runs-on: ubuntu-latest
-    if: >-
-      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v')))
-      || (github.event_name == 'pull_request' && github.head_ref == 'main')
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
     permissions:
       contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v5
 
@@ -516,6 +710,8 @@ jobs:
           node-version: "25"
           registry-url: "https://registry.npmjs.org"
 
+      - uses: oven-sh/setup-bun@v2
+
       - name: Determine version
         id: version
         run: lua scripts/determine-version.lua
@@ -527,8 +723,6 @@ jobs:
           path: ./npm-packages
 
       - name: Publish platform packages
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -541,14 +735,12 @@ jobs:
               make set-npm-version PKG="$pkg_dir" VERSION="$VERSION"
 
               cd "$pkg_dir"
-              npm publish --tag "$TAG" --access public || echo "Failed to publish ${pkg_name} (may already exist)"
+              npm publish --tag "$TAG" --access public --provenance
               cd -
             fi
           done
 
       - name: Publish bun package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -557,11 +749,11 @@ jobs:
           make set-npm-version PKG=packages/fff-bun VERSION="$VERSION"
 
           cd packages/fff-bun
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-bun (may already exist)"
+          bun install --frozen-lockfile
+          bun run build
+          npm publish --tag "$TAG" --access public --provenance
 
       - name: Publish Node.js package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -572,11 +764,9 @@ jobs:
           cd packages/fff-node
           npm install
           npm run build
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-node (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance
 
       - name: Publish pi-fff package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -585,4 +775,4 @@ jobs:
           make set-npm-version PKG=packages/pi-fff VERSION="$VERSION"
 
           cd packages/pi-fff
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/pi-fff (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance

.github/workflows/rust.yml

@@ -3,8 +3,14 @@ name: Rust CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 env:
   CARGO_TERM_COLOR: always
@@ -17,8 +23,12 @@ jobs:
     name: Test
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    # Guard against deadlocks in the shared-picker / watcher teardown
+    # path: a stuck test would otherwise consume a full 6h CI slot.
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v5
       
@@ -37,7 +47,86 @@ jobs:
           components: rustfmt, clippy
 
       - name: Run tests
-        run: cargo test --features zlob --workspace --exclude fff-nvim
+        # fff-python requires full python o3 machinery which is very slow
+        run: cargo test --no-default-features --features zlob --workspace --exclude fff-nvim --exclude fff-python
+
+  stress-test:
+    name: Fuzz Tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # Keep going after one OS fails so we can see whether a bug
+      # reproduces everywhere or is platform-specific.
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    # Long-running; don't let a stuck watcher thread burn a full CI
+    # timeout. Two scenarios should finish well under this limit.
+    timeout-minutes: 20
+    env:
+      FFF_STRESS_CASES: "5"
+      FFF_STRESS_MIN_OPS: "30"
+      FFF_STRESS_MAX_OPS: "60"
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install Zig
+        uses: goto-bus-stop/setup-zig@v2
+        with:
+          version: 0.16.0
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-stress-${{ matrix.os }}"
+          components: rustfmt, clippy
+
+      - name: Stress test seeded
+        shell: bash
+        run: make test-stress-seeded
+
+      - name: Stress test random
+        shell: bash
+        run: make test-stress-random
+
+      - name: Stress test regressions
+        shell: bash
+        run: make test-stress-regressions
+
+      - name: Upload proptest regressions on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: proptest-regressions-${{ matrix.os }}
+          path: crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions
+          if-no-files-found: ignore
+
+  build-i686:
+    name: Build i686-unknown-linux-gnu
+    runs-on: ubuntu-latest
+    # Verifies that fff-search compiles on 32-bit x86, where std::arch::x86_64
+    # is unavailable. SIMD paths are disabled on this target; only the scalar
+    # fallback should build. See issue #656.
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install cross toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-multilib g++-multilib
+
+      - name: Install Rust (i686 target)
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          target: i686-unknown-linux-gnu
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-i686"
+
+      - name: Build fff-search for i686
+        run: cargo build -p fff-search --target i686-unknown-linux-gnu
 
   fmt:
     name: cargo fmt
@@ -71,4 +160,4 @@ jobs:
           components: clippy
           
       - name: Run clippy
-        run: cargo clippy -- -D warnings
+        run: cargo clippy --no-default-features --features zlob -- -D warnings

.gitignore

@@ -16,6 +16,7 @@ result
 *.data
 node_modules/
 crates/fff-notify-debouncer-full/
+packages/fff-bun/glob-bench-bin
 
 dist/
 scripts/benchmark-results/
@@ -24,3 +25,41 @@ scripts/benchmark-results/
 *.dylib
 *.so
 *.dll
+*.pdb
+
+# Instruments traces
+*.trace/
+
+# Test logs
+fff-test.log
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyd
+*.egg-info/
+*.egg
+.eggs/
+build/
+*.whl
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+# uv
+# Testing / linting
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~

.luacheckrc

@@ -1,4 +1,4 @@
--- luacheck configuration for fff.nvim
+-- luacheck configuration for fff
 -- https://luacheck.readthedocs.io/en/stable/config.html
 
 -- Neovim globals

.luarc.json

@@ -5,14 +5,14 @@
   },
   "workspace": {
     "library": [
-      "/opt/homebrew/share/nvim/runtime/lua/vim/_meta",
-      "/opt/homebrew/share/nvim/runtime/lua/vim/shared.lua",
+      "$VIMRUNTIME/lua",
       "${3rd}/luv/library",
       "${3rd}/busted/library"
     ],
     "checkThirdParty": false
   },
   "diagnostics": {
+    "globals": ["vim"],
     "severity": {
       "undefined-global": "Error",
       "undefined-field": "Warning",

AGENTS.md

@@ -0,0 +1,118 @@
+# To Clankers
+
+This repository contains **FFF.nvim (Fast File Finder)**, a high-performance file picker for Neovim inspired by blink.cmp's fuzzy matching technology. It's NOT a completion plugin, but rather a standalone file finder with advanced fuzzy search and frecency scoring. The project aims to be the drop-in replacement for telescope, fzf-lua, snacks.picker and similar plugins, focusing on speed, accuracy search and usability features.
+
+## Development Commands
+
+Always prefer Makefile commands listed to the cargo/bun/node if possible.
+
+### Building
+
+- `make build` - build everything
+
+### Testing and Development Tools
+
+This project does not have a traditional test suite. Testing is done through:
+
+- Create e2e local test file for Neovim: Load any Lua test file with `nvim -l <test_file>`
+- Write inline rust unit tests for any functionality that is standalone and scoped within a single function
+
+### Code Quality
+
+- `make lint` - Rust linting and code analysis
+- `make format` - Format all code 
+- `make test` - Run unit tests (limited coverage, primarily integration testing)
+
+When doing code make sure to REDUCE SIZE OF COMMENTS. This is very important. Every comment should be concise 1-2 liner maximum 4 lines if describes really extensive and unnatural concept.
+
+### Important coding rules
+
+- Do not add doc comments to the private structs and functions.
+- Do not make public structs if something can be private
+
+
+## Style guide
+
+- NO MODULES COMMENTS
+- NO TOP FILE COMMENTS
+- NO COMMENT LONGER THAN 2 LINES UNLESS ASKED EXPLICITLY
+- UTILITY FUNCTIONS GO INTO THE END OF FILE
+
+## Architecture
+
+Everything that is performance critical happens in rust world, everything that is neovim specific happens in the lua code.
+
+There are 3 main components:
+
+- Rust binary with the global file picker state containing index of all files
+- Background thread with the file system watcher that updates the index in real time
+- Lua UI layer that renders the picker, handles user input, and calls the rust functions via FFI
+
+There are 2 databases:
+
+- Frecency database (LMDB) that tracks file access patterns for scoring
+- Query history database used to track the user's previous search queries
+
+### Key Files
+
+- `lua/fff.lua` - Entry point, delegates to main.lua
+- `lua/fff/main.lua` - Public API (find_files, search, change_directory)
+- `lua/fff/core.lua` - Initialization, autocmds, global state management
+- `lua/fff/picker_ui.lua` - UI rendering, layout calculation, keymaps
+- `lua/fff/file_picker/preview.lua` - File preview with syntax highlighting
+- `lua/fff/file_picker/image.lua` - Image preview (snacks.nvim integration)
+- `lua/fff/conf.lua` - Default config
+- `lua/fff/rust/init.lua` - Loads compiled Rust shared library
+
+**Rust Side:**
+
+- `lua/fff/rust/lib.rs` - FFI bindings, global state (FILE_PICKER, FRECENCY)
+- `lua/fff/rust/file_picker.rs` - Core FilePicker struct, indexing, background watcher
+- `lua/fff/rust/frecency.rs` - Frecency database (LMDB) and scoring
+- `lua/fff/rust/query_tracker.rs` - Search query history tracking
+- `lua/fff/rust/score.rs` - Fuzzy match scoring with frizbee integration
+- `lua/fff/rust/git.rs` - Git status caching and repository detection
+- `lua/fff/rust/background_watcher.rs` - File system watcher thread
+
+### Scoring Algorithm
+
+Located at the score.rs file
+
+### Build System
+
+- `Cargo.toml` - Rust dependencies and build configuration (package name: `fff_nvim`)
+- `rust-toolchain.toml` - Specifies Rust nightly toolchain with required components
+- `Cross.toml` - Cross-compilation settings using Zig for Linux targets
+- **CI/CD Workflows**:
+  - `.github/workflows/rust.yml` - Rust testing, formatting, and clippy checks
+  - `.github/workflows/release.yaml` - Automated multi-platform builds
+  - `.github/workflows/stylua.yaml` - Lua code formatting validation
+  - `.github/workflows/nix.yml` - Nix build validation
+- **Cross-compilation Support**: Uses `cross` tool with Zig backend for efficient cross-compilation
+
+## Development Notes
+
+### Working with Rust Code
+
+- Prefer struct methods over functions
+- If there is more than 2 impls in the file - create new file
+- Smaller concise comments over giant comment blocks
+- Do not add doc comments to the private functions/structs
+- Be very careful around locking and better double check with the human if something is going to require potentially long lock on a mutex/rwlock
+
+### Working with lua code
+
+- Document the types of public functions in every module
+- Use `vim.validate()` for validating user inputs in public functions
+- Try to reuse as much of existing functions as possible
+- When working on new features for the UI **IT IS EXTREMELY IMPORTANT** to keep the core functionality of navigating between files, selecting, and seeing the preview working as is. NEVER break anything from the core UI functionality, only add new features on top of the current UI.
+- When making a large chunk of code make lua test that opens neovim at `~/dev/lightsource` and opens the picker to test the ui functionality across the actual code.
+- When adding a new highlights or any new shortcuts and configurable UI options add them to the neovim config. AND IMPORTANT: update the README.md with the new configuration options.
+
+### UI rendering
+
+When working on the UI changeds IT IS EXTREMELY important for you to test it for both prompt_position="bottom" and prompt_position="top" as the rendering logic is different for both of them in both rust and lua world. When the prompt is positioed in the bottom everything should work the same way as the top but would be reversed in order. (though navigation is same for both)
+
+## Top level API that can not introduce breaking changes under any circumstance
+
+Top level rust, lua, C, and bun APIs can not be changed under any circumstance

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

Cargo.lock

@@ -91,9 +91,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.102"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
 
 [[package]]
 name = "arrayref"
@@ -139,17 +139,11 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "bindet"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5afee99ef5f7527f3944f2bf4f5d443749fa47d43eb1d4f83a36e839be7900a3"
-
 [[package]]
 name = "bindgen"
-version = "0.70.1"
+version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
  "bitflags 2.11.0",
  "cexpr",
@@ -160,7 +154,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash 1.1.0",
+ "rustc-hash",
  "shlex",
  "syn",
 ]
@@ -446,9 +440,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -474,6 +468,16 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
 
+[[package]]
+name = "ctor"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "ctrlc"
 version = "3.5.2"
@@ -629,20 +633,17 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "fff-c"
-version = "0.6.4"
+version = "0.10.4"
 dependencies = [
  "fff-query-parser",
  "fff-search",
  "git2",
- "mimalloc",
- "serde",
  "serde_json",
- "tracing",
 ]
 
 [[package]]
 name = "fff-grep"
-version = "0.6.4"
+version = "0.10.4"
 dependencies = [
  "bstr",
  "memchr",
@@ -650,7 +651,7 @@ dependencies = [
 
 [[package]]
 name = "fff-mcp"
-version = "0.6.4"
+version = "0.10.4"
 dependencies = [
  "clap",
  "fff-query-parser",
@@ -661,59 +662,60 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
+ "tempfile",
  "tokio",
  "tracing",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "fff-notify-debouncer-full"
-version = "0.9.1"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4594f0ef1ad0bfcd112bbe4b397ef881f50442bc710356b86858d555848e4d09"
+checksum = "29a4ebea7b8a2840cd59358bbf396f6f04313ce8eae84ac79703ce80298b8731"
 dependencies = [
  "file-id",
  "log",
- "notify 9.0.0-rc.3",
+ "notify",
  "notify-types",
+ "rustc-hash",
  "walkdir",
 ]
 
 [[package]]
 name = "fff-nvim"
-version = "0.6.4"
+version = "0.10.4"
 dependencies = [
  "ahash",
- "blake3",
  "chrono",
  "criterion",
  "ctrlc",
- "dirs",
  "fff-query-parser",
  "fff-search",
  "git2",
- "glidesort",
- "heed",
  "ignore",
  "mimalloc",
  "mlua",
- "neo_frizbee",
- "notify 8.2.0",
- "notify-debouncer-full",
  "once_cell",
- "pathdiff",
- "rand",
- "rayon",
- "serde",
- "smallvec",
- "smartstring",
- "tempfile",
- "thiserror 2.0.18",
+ "rand 0.8.5",
  "tracing",
+ "tracing-subscriber",
+ "zlob",
+]
+
+[[package]]
+name = "fff-python"
+version = "0.10.4"
+dependencies = [
+ "fff-query-parser",
+ "fff-search",
+ "git2",
+ "pyo3",
 ]
 
 [[package]]
 name = "fff-query-parser"
-version = "0.6.4"
+version = "0.10.4"
 dependencies = [
  "criterion",
  "zlob",
@@ -721,14 +723,13 @@ dependencies = [
 
 [[package]]
 name = "fff-search"
-version = "0.6.4"
+version = "0.10.4"
 dependencies = [
  "ahash",
  "aho-corasick",
- "bindet",
  "blake3",
- "chrono",
  "criterion",
+ "ctor",
  "dirs",
  "dunce",
  "fff-grep",
@@ -745,21 +746,19 @@ dependencies = [
  "memmap2",
  "mimalloc",
  "neo_frizbee",
- "notify 9.0.0-rc.3",
- "once_cell",
+ "notify",
  "parking_lot",
  "pathdiff",
- "rand",
+ "proptest",
+ "rand 0.8.5",
  "rayon",
  "regex",
  "regex-syntax",
  "serde",
- "serde_json",
+ "signal-hook-registry",
  "smallvec",
- "smartstring",
  "tempfile",
  "thiserror 2.0.18",
- "toml",
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
@@ -781,6 +780,12 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "foldhash"
 version = "0.1.5"
@@ -796,15 +801,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "fsevent-sys"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "futures"
 version = "0.3.32"
@@ -931,15 +927,14 @@ dependencies = [
 
 [[package]]
 name = "git2"
-version = "0.20.4"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
+checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e"
 dependencies = [
  "bitflags 2.11.0",
  "libc",
  "libgit2-sys",
  "log",
- "url",
 ]
 
 [[package]]
@@ -1209,6 +1204,15 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "inotify"
 version = "0.11.1"
@@ -1264,15 +1268,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.17"
@@ -1339,9 +1334,9 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
 
 [[package]]
 name = "libgit2-sys"
-version = "0.18.3+1.9.2"
+version = "0.18.7+1.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
+checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89"
 dependencies = [
  "cc",
  "libc",
@@ -1446,13 +1441,22 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
 
 [[package]]
 name = "memmap2"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
 dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mimalloc"
 version = "0.1.48"
@@ -1493,7 +1497,7 @@ dependencies = [
  "mlua_derive",
  "num-traits",
  "parking_lot",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "rustversion",
 ]
 
@@ -1522,13 +1526,9 @@ dependencies = [
 
 [[package]]
 name = "neo_frizbee"
-version = "0.10.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b216854d8c3752f8dbda56703c708de44bccb0542ca3924976c5977d6fb0724"
-dependencies = [
- "itertools 0.14.0",
- "raw-cpuid",
-]
+checksum = "7a2f6120a8da26bea3587731072111062c5d8c51ca3a3a75a716bd8b735d5882"
 
 [[package]]
 name = "nix"
@@ -1554,27 +1554,9 @@ dependencies = [
 
 [[package]]
 name = "notify"
-version = "8.2.0"
+version = "9.0.0-rc.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3"
-dependencies = [
- "bitflags 2.11.0",
- "fsevent-sys",
- "inotify",
- "kqueue",
- "libc",
- "log",
- "mio",
- "notify-types",
- "walkdir",
- "windows-sys 0.60.2",
-]
-
-[[package]]
-name = "notify"
-version = "9.0.0-rc.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "783683ce6e1059e3747190a6c05688db21e07a846bf90babb351365f9133fe3e"
+checksum = "b44b771d4dd781ef14c84078693e67495da6b47f609f72e8a4da8420a861240e"
 dependencies = [
  "bitflags 2.11.0",
  "inotify",
@@ -1590,19 +1572,6 @@ dependencies = [
  "xxhash-rust",
 ]
 
-[[package]]
-name = "notify-debouncer-full"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "375bd3a138be7bfeff3480e4a623df4cbfb55b79df617c055cd810ba466fa078"
-dependencies = [
- "file-id",
- "log",
- "notify 8.2.0",
- "notify-types",
- "walkdir",
-]
-
 [[package]]
 name = "notify-types"
 version = "2.1.0"
@@ -1762,7 +1731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
 dependencies = [
  "phf_shared",
- "rand",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -1827,6 +1796,12 @@ dependencies = [
  "plotters-backend",
 ]
 
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -1870,6 +1845,92 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "proptest"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
+dependencies = [
+ "bitflags 2.11.0",
+ "num-traits",
+ "rand 0.9.4",
+ "rand_chacha 0.9.0",
+ "rand_xorshift",
+ "regex-syntax",
+ "rusty-fork",
+ "tempfile",
+ "unarray",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
 [[package]]
 name = "quote"
 version = "1.0.45"
@@ -1898,8 +1959,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+dependencies = [
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -1909,7 +1980,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -1922,12 +2003,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "raw-cpuid"
-version = "11.6.0"
+name = "rand_core"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
 dependencies = [
- "bitflags 2.11.0",
+ "getrandom 0.3.4",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
+dependencies = [
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -2021,9 +2111,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
 
 [[package]]
 name = "rmcp"
-version = "1.2.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba6b9d2f0efe2258b23767f1f9e0054cfbcac9c2d6f81a031214143096d7864f"
+checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e"
 dependencies = [
  "async-trait",
  "base64",
@@ -2043,9 +2133,9 @@ dependencies = [
 
 [[package]]
 name = "rmcp-macros"
-version = "1.2.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab9d95d7ed26ad8306352b0d5f05b593222b272790564589790d210aa15caa9e"
+checksum = "6aefac48c364756e97f04c0401ba3231e8607882c7c1d92da0437dc16307904d"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -2054,12 +2144,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.2"
@@ -2085,6 +2169,17 @@ version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
+[[package]]
+name = "rusty-fork"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2"
+dependencies = [
+ "fnv",
+ "quick-error",
+ "tempfile",
+]
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -2186,15 +2281,6 @@ dependencies = [
  "zmij",
 ]
 
-[[package]]
-name = "serde_spanned"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -2238,18 +2324,6 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
-[[package]]
-name = "smartstring"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
-dependencies = [
- "autocfg",
- "serde",
- "static_assertions",
- "version_check",
-]
-
 [[package]]
 name = "socket2"
 version = "0.6.3"
@@ -2266,12 +2340,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -2309,6 +2377,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
 [[package]]
 name = "tempfile"
 version = "3.27.0"
@@ -2463,47 +2537,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "toml"
-version = "0.8.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
-dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_write",
- "winnow",
-]
-
-[[package]]
-name = "toml_write"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
-
 [[package]]
 name = "tracing"
 version = "0.1.44"
@@ -2577,6 +2610,12 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -2589,6 +2628,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
 [[package]]
 name = "url"
 version = "2.5.8"
@@ -2993,15 +3038,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
-[[package]]
-name = "winnow"
-version = "0.7.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "wit-bindgen"
 version = "0.51.0"
@@ -3201,9 +3237,9 @@ dependencies = [
 
 [[package]]
 name = "zlob"
-version = "1.3.3"
+version = "1.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f3522fa9701b74ec72758aedb96da278f6e0533bc16b6a79090bfb465d4661"
+checksum = "f0f010d9b10495cc0037cb47cedc143c39c7fa2d59f752dca19036fd0234a210"
 dependencies = [
  "bindgen",
  "bitflags 2.11.0",

Cargo.toml

@@ -4,26 +4,29 @@ members = [
   "crates/fff-core",
   "crates/fff-mcp",
   "crates/fff-nvim",
+  "crates/fff-python",
   "crates/fff-query-parser",
   "crates/fff-grep",
 ]
-exclude = ["crates/fff-notify-debouncer-full"]
+
 resolver = "2"
 
+[workspace.lints.clippy]
+module_inception = "allow"
+
 [workspace.dependencies]
-fff-grep = { version = "0.6.4", path = "crates/fff-grep" }
-fff-query-parser = { version = "0.6.4", path = "crates/fff-query-parser", default-features = false }
+fff-grep = { version = "0.10.4", path = "crates/fff-grep" }
+fff-query-parser = { version = "0.10.4", path = "crates/fff-query-parser", default-features = false }
 
 # Shared dependencies
 ahash = "0.8"
-bindet = "0.3"
 blake3 = "1.8.2"
 chrono = { version = "0.4", features = ["serde"] }
 ctrlc = "3.4.2"
 dirs = "5.0"
 dunce = "1.0"
 # git2 - base config without TLS (each crate adds platform-specific TLS)
-git2 = { version = "0.20.2", default-features = false, features = [
+git2 = { version = "0.21.0", default-features = false, features = [
   "vendored-libgit2",
 ] }
 glidesort = "0.1"
@@ -32,12 +35,13 @@ heed = "0.22.0"
 ignore = "0.4.22"
 memmap2 = "0.9"
 mimalloc = "0.1.47"
-zlob = "1.3.3"
+signal-hook-registry = "1.4"
+zlob = { version = "=1.6.3" }
 
 mlua = { version = "0.11.1", features = ["module", "luajit"] }
-neo_frizbee = { version = "0.10.1", features = ["match_end_col"] }
+neo_frizbee = { version = "0.11.0", features = ["match_end_col"] }
 notify = { version = "9.0.0-rc.3" }
-notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.1" }
+notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.4" }
 once_cell = "1.20.2"
 parking_lot = "0.12"
 pathdiff = "0.2.1"
@@ -52,7 +56,7 @@ tracing = "0.1"
 opt-level = 3
 lto = "fat"
 codegen-units = 1
-strip = true
+strip = "debuginfo"
 
 [profile.ci]
 inherits = "release"
@@ -62,3 +66,11 @@ lto = "thin"
 
 [profile.bench]
 inherits = "release"
+
+# For Instruments / xctrace: release-level optimization but keep debuginfo
+# and symbols so sampled frames resolve to real Rust names.
+[profile.prof]
+inherits = "release"
+debug = "full"
+strip = false
+lto = "thin"

Formula/fff-mcp.rb

@@ -0,0 +1,60 @@
+# Originally authored by @jellydn (https://github.com/jellydn/homebrew-tap).
+# Maintained in-repo; auto-bumped by .github/workflows/release.yaml on stable releases.
+class FffMcp < Formula
+  desc "Fast file search toolkit for AI agents (MCP server)"
+  homepage "https://github.com/dmtrKovalenko/fff"
+  license "MIT"
+  version "0.10.4"
+
+  LIVECHECK_REPO = "dmtrKovalenko/fff".freeze
+  RELEASE_BASE = "https://github.com/dmtrKovalenko/fff/releases/download".freeze
+
+  on_macos do
+    on_arm do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-apple-darwin"
+      sha256 "f55ce2945cd30fd252a4b1e7d8674ff7ce41b104cc009a8d3d52d08bfccca2c0"
+    end
+
+    on_intel do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-apple-darwin"
+      sha256 "821aedd5873292841f32d7584eb68fea085b248e5c970ff5d4f6dca790e9b18c"
+    end
+  end
+
+  on_linux do
+    on_arm do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-unknown-linux-gnu"
+      sha256 "3f266c0eded51a106badf1d47448a7f963762ccf3a12c5a4f06ee2b7adaf82d1"
+    end
+
+    on_intel do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-unknown-linux-gnu"
+      sha256 "b5dd3182028c8a7bc34ee964cf2d30694f8121d2837a8516cf4a99bcfa822127"
+    end
+  end
+
+  livecheck do
+    url "https://github.com/#{LIVECHECK_REPO}/releases/latest"
+    strategy :github_latest
+  end
+
+  def install
+    if OS.mac?
+      if Hardware::CPU.arm?
+        bin.install "fff-mcp-aarch64-apple-darwin" => "fff-mcp"
+      elsif Hardware::CPU.intel?
+        bin.install "fff-mcp-x86_64-apple-darwin" => "fff-mcp"
+      end
+    elsif OS.linux?
+      if Hardware::CPU.arm?
+        bin.install "fff-mcp-aarch64-unknown-linux-gnu" => "fff-mcp"
+      elsif Hardware::CPU.intel?
+        bin.install "fff-mcp-x86_64-unknown-linux-gnu" => "fff-mcp"
+      end
+    end
+  end
+
+  test do
+    system bin/"fff-mcp", "--healthcheck"
+  end
+end

Makefile

@@ -1,18 +1,57 @@
 PLENARY_DIR ?= ../plenary.nvim
+MINI_DIR ?= ../mini.nvim
 
 PREFIX ?= /usr/local
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR ?= $(PREFIX)/include
 
-.PHONY: build build-c-lib install uninstall test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header
+# Compile-time cfg that gates the watcher + git-status fuzz stress test.
+STRESS_RUSTFLAGS := --cfg stress
+FFF_STRESS_DEFAULT_SEED ?= 0xDEADBEEFCAFEBABE
+
+SHELL := bash
+# Order matters: `-c` must be last so bash treats the recipe as the script
+# string rather than the literal `-o` / `pipefail` tokens.
+.SHELLFLAGS := -o pipefail -euc
+
+.PHONY: build build-c-lib install uninstall test test-rust test-rescan test-rescan-known-defects rescan-probe test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
 
 all: format test lint
 
+SYNC_API_SRC := packages/shared/fff-api.ts
+SYNC_API_TARGETS := packages/fff-node/src/fff-api.ts packages/fff-bun/src/fff-api.ts
+SYNC_API_BANNER := // ----------------------------------------------------------------------------\n// GENERATED FILE - DO NOT EDIT.\n// Copied from: ${SYNC_API_SRC}\n// Run make sync-js-api from the repo root to regenerate.\n// ----------------------------------------------------------------------------\n\n
+
+sync-js-api:
+	@for target in $(SYNC_API_TARGETS); do \
+		printf '$(SYNC_API_BANNER)' > "$$target"; \
+		cat $(SYNC_API_SRC) >> "$$target"; \
+		echo "synced: $$target"; \
+	done
+
+sync-js-api-check:
+	@status=0; \
+	for target in $(SYNC_API_TARGETS); do \
+		tmp=$$(mktemp); \
+		printf '$(SYNC_API_BANNER)' > "$$tmp"; \
+		cat $(SYNC_API_SRC) >> "$$tmp"; \
+		if ! cmp -s "$$tmp" "$$target"; then \
+			echo "out of date: $$target (run make sync-js-api)"; status=1; \
+		fi; \
+		rm -f "$$tmp"; \
+	done; \
+	exit $$status
+
 build:
-	cargo build --release --features zlob
+	cargo build --release --no-default-features --features zlob
+
+# Only the crates the e2e suites load (nvim lua tests + C/bun/node FFI tests),
+# skipping fff-python (pyo3) and fff-mcp (tokio/rmcp) which e2e never touches.
+build-e2e:
+	cargo build --release -p fff-nvim -p fff-c --no-default-features --features zlob
 
 build-c-lib:
-	cargo build --release -p fff-c --features zlob
+	cargo build --release -p fff-c --no-default-features --features zlob
 
 header:
 	cbindgen --config crates/fff-c/cbindgen.toml --crate fff-c --output crates/fff-c/include/fff.h
@@ -50,39 +89,185 @@ test-setup:
 		echo "Cloning plenary.nvim..."; \
 		git clone --depth 1 https://github.com/nvim-lua/plenary.nvim $(PLENARY_DIR); \
 	fi
+	@if [ ! -d "$(MINI_DIR)" ]; then \
+		echo "Cloning mini.nvim..."; \
+		git clone --depth 1 https://github.com/echasnovski/mini.nvim $(MINI_DIR); \
+	fi
 
 test-rust:
-	cargo test --workspace --features zlob --exclude fff-nvim
+	cargo test --workspace --no-default-features --features zlob --exclude fff-nvim --exclude fff-python
 
+# Watcher rescan harness: asserts that editing, build output, git activity and
+# preview reads all stay on the incremental path instead of re-walking the tree.
+test-rescan:
+	cargo test -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- rescan
+
+# Live probe for watcher rescan requests and their causes.
+# Usage: make rescan-probe DIR=~/some/repo [SECONDS=120]
+rescan-probe:
+	cargo run --release -p fff-nvim --bin rescan_probe \
+		--no-default-features --features zlob,rescan-stats -- \
+		$(or $(DIR),.) $(if $(SECONDS),--seconds $(SECONDS),)
+
+# The same harness, restricted to cases that currently fail on purpose. Each
+# `#[ignore]` reason names the code that causes the unnecessary rescan.
+test-rescan-known-defects:
+	cargo test --no-fail-fast -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- --ignored --nocapture
+
+CC ?= cc
+CFLAGS ?= -O0 -g -Wall -Wextra -std=c99
+TARGET_DIR ?= target/release
+SMOKE_BIN := $(TARGET_DIR)/fff_c_smoke
+SMOKE_SRC := crates/fff-c/tests/smoke.c
+SMOKE_INCLUDE := crates/fff-c/include
+
+test-c-smoke: build
+	$(CC) $(CFLAGS) -I $(SMOKE_INCLUDE) -L $(TARGET_DIR) \
+		-Wl,-rpath,@loader_path/../target/release \
+		-Wl,-rpath,$$(pwd)/$(TARGET_DIR) \
+		$(SMOKE_SRC) -lfff_c -o $(SMOKE_BIN)
+	$(SMOKE_BIN) .
+
+# Alias kept for the `external-tests.yml` workflow naming.
+test-c-api: test-c-smoke
+
+# neovim instance swallows internal crashes and doesn't rise the the error exiting silently
+# so check the stdout in case the sigsegv coming out of fff was printed (actual regression).
+# Output is streamed live via `tee`; pipefail (set above) propagates nvim's exit.
 test-lua: test-setup build
+	@logfile=$$(mktemp); \
+	trap 'rm -f "$$logfile"' EXIT; \
 	nvim --headless -u tests/minimal_init.lua \
-		-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1
+		-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1 \
+		| tee "$$logfile"; \
+	if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \
+		echo ""; \
+		echo "FAIL: native crash detected during lua tests"; \
+		exit 1; \
+	fi
+
+test-lua-snap: test-setup build
+	@logfile=$$(mktemp); \
+	trap 'rm -f "$$logfile"' EXIT; \
+	nvim --headless -u tests/minimal_init.lua \
+		-c "lua local ok,err=pcall(require('mini.test').run_file,'tests/picker_ui_snap.lua'); if not ok then io.stderr:write('mini.test failed to load: '..tostring(err)..'\\n'); vim.cmd('cquit 2') end" 2>&1 \
+		| tee "$$logfile"; \
+	if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \
+		echo ""; \
+		echo "FAIL: native crash detected during snapshot tests"; \
+		exit 1; \
+	fi
 
 test-version: test-setup
 	nvim --headless -u tests/minimal_init.lua \
 		-c "PlenaryBustedFile tests/version_spec.lua" 2>&1
 
-prepare-bun: build
+prepare-bun: build sync-js-api
 	mkdir -p packages/fff-bun/bin
-	cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null; \
-	cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null; \
-	cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null; \
-	true
+	cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null || true; \
+	cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null || true; \
+	cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null || true
 
-prepare-node: build
+prepare-node: build sync-js-api
 	mkdir -p packages/fff-node/bin
-	cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null; \
-	cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null; \
-	cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null; \
-	true
+	cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null || true; \
+	cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null || true; \
+	cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null || true
 
 test-bun: prepare-bun
-	cd packages/fff-bun && bun test src/
+	cd packages/fff-bun && bun test test/
+	cd packages/pi-fff && bun test test/
+
+# Same as prepare-bun but puts the compiled binary into the actual npm package location
+prepare-bun-packaged: prepare-bun
+	@machine=$$(uname -m); \
+	case "$$machine" in \
+	  x86_64|amd64) arch=x64 ;; \
+	  aarch64|arm64) arch=arm64 ;; \
+	  *) echo "unsupported arch: $$machine" >&2; exit 1 ;; \
+	esac; \
+	case "$$(uname -s)" in \
+	  Darwin) lib=libfff_c.dylib; pkg=fff-bin-darwin-$$arch ;; \
+	  Linux) lib=libfff_c.so; \
+	    if ldd --version 2>&1 | grep -qi musl; then libc=musl; else libc=gnu; fi; \
+	    pkg=fff-bin-linux-$$arch-$$libc ;; \
+	  MINGW*|MSYS*|CYGWIN*|Windows_NT) lib=fff_c.dll; pkg=fff-bin-win32-$$arch ;; \
+	  *) echo "unsupported OS: $$(uname -s)" >&2; exit 1 ;; \
+	esac; \
+	src=target/release/$$lib; \
+	[ -f "$$src" ] || { echo "missing built library: $$src" >&2; exit 1; }; \
+	dest=packages/fff-bun/node_modules/@ff-labs/$$pkg; \
+	rm -rf "$$dest"; mkdir -p "$$dest"; \
+	cp "$$src" "$$dest/$$lib"; \
+	printf '{ "name": "@ff-labs/%s", "version": "0.0.0", "main": "%s" }\n' "$$pkg" "$$lib" > "$$dest/package.json"
+
+# Compile a bun example to a standalone executable and run it. Verifies the
+# native libfff_c is embedded + loaded from a `bun build --compile` binary.
+# The staged bin package is removed before running so success proves the lib
+# was embedded, not resolved from disk.
+test-bun-compile: prepare-bun-packaged
+	cd packages/fff-bun && \
+		if [ "$$(uname -s)" = "Linux" ]; then \
+		  if ldd --version 2>&1 | grep -qi musl; then DEFINE='--define FFF_LIBC="musl"'; \
+		  else DEFINE='--define FFF_LIBC="gnu"'; fi; \
+		else DEFINE=""; fi; \
+		bun build --compile $$DEFINE ./examples/glob-bench.ts --outfile ./glob-bench-bin && \
+		EXE=./glob-bench-bin; [ -f "$$EXE.exe" ] && EXE="$$EXE.exe"; \
+		rm -rf bin node_modules/@ff-labs; \
+		"$$EXE" . '**/*.ts' 1 | tee /tmp/fff-compile-e2e.log && \
+		grep -q 'fff.glob' /tmp/fff-compile-e2e.log
+	rm -f packages/fff-bun/glob-bench-bin packages/fff-bun/glob-bench-bin.exe
 
 test-node: prepare-node
-	cd packages/fff-node && npm run build && node test/e2e.mjs
+	cd packages/fff-node && npm run build && node test/e2e.mjs && node test/watch.mjs
 
-test: test-rust test-lua test-version test-bun test-node
+test-js: test-bun test-node
+
+# Bug pinning stress test script over fff-node for issue #515
+# Just keep it untouched because it's good enough + some stress for SDK
+FFF_STRESS_ITERS ?= 50
+test-node-stress: prepare-node
+	cd packages/fff-node && npm run build && \
+		FFF_STRESS_ITERS=$(FFF_STRESS_ITERS) node test/stress-515.mjs
+
+test: test-rust test-lua test-lua-snap test-version test-bun test-node test-node-stress
+
+test-stress-seeded:
+	FFF_STRESS_SEED="$${FFF_STRESS_SEED:-$(FFF_STRESS_DEFAULT_SEED)}" \
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_seeded
+
+test-stress-random:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_random
+
+test-stress-regressions:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_regression stress_merge_conflict_convergence
+
+test-stress-repos:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_real_repos \
+		--no-default-features --features zlob \
+		-- --nocapture
+
+test-stress: test-stress-seeded test-stress-random test-stress-regressions test-stress-repos
 
 # Update version in a package.json, including optionalDependencies.
 # Usage: make set-npm-version PKG=packages/fff-bun VERSION=1.0.0-nightly.abc1234
@@ -98,6 +283,9 @@ set-npm-version:
 				pkg.optionalDependencies[dep] = '$(VERSION)'; \
 			} \
 		} \
+		for (const dep of ['@ff-labs/fff-bun', '@ff-labs/fff-node']) { \
+			if (pkg.dependencies?.[dep]) pkg.dependencies[dep] = '$(VERSION)'; \
+		} \
 		fs.writeFileSync('$(PKG)/package.json', JSON.stringify(pkg, null, 2) + '\n'); \
 	"
 	@echo "Set $(PKG) to $(VERSION)"
@@ -112,7 +300,7 @@ format-ts:
 format: format-rust format-lua format-ts
 
 lint-rust:
-	cargo clippy --workspace --features zlob -- -D warnings
+	cargo clippy --workspace --no-default-features --features zlob -- -D warnings
 lint-lua:
 	 ~/.luarocks/bin/luacheck .
 lint-ts:
@@ -122,11 +310,65 @@ lint: lint-rust lint-lua lint-ts
 
 check: format lint
 
+FFF_RELEASE_REPO ?= dmtrKovalenko/fff
+FFF_FORMULA_PATH ?= Formula/fff-mcp.rb
+FFF_INSTALL_SCRIPT_PATH ?= install-mcp.sh
+
+# Read the sha256 for $1 (filename, no .sha256 suffix). Reads from
+# BINARIES_DIR/$1.sha256 when set; otherwise curls the GitHub release.
+define fff_fetch_sha
+	if [ -n "$$BINARIES_DIR" ]; then \
+		awk '{print $$1}' "$$BINARIES_DIR/$$1.sha256" \
+			|| { echo "Missing checksum file: $$BINARIES_DIR/$$1.sha256" >&2; exit 1; }; \
+	else \
+		curl -fsSL "https://github.com/$(FFF_RELEASE_REPO)/releases/download/v$(VERSION)/$$1.sha256" \
+			| awk '{print $$1}'; \
+	fi
+endef
+
+bump-homebrew-formula:
+	@test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-homebrew-formula VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1)
+	@export BINARIES_DIR="$(BINARIES_DIR)"; \
+	fetch_sha() { $(fff_fetch_sha); }; \
+	sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \
+	sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \
+	sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-gnu)"; \
+	sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-gnu)"; \
+	sed -i.bak \
+		-e 's/^  version "[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*"$$/  version "$(VERSION)"/' \
+		-e '/fff-mcp-aarch64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_arm"'"/;}' \
+		-e '/fff-mcp-x86_64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_intel"'"/;}' \
+		-e '/fff-mcp-aarch64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_arm"'"/;}' \
+		-e '/fff-mcp-x86_64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_intel"'"/;}' \
+		"$(FFF_FORMULA_PATH)" && rm -f "$(FFF_FORMULA_PATH).bak"; \
+	echo "Bumped $(FFF_FORMULA_PATH) to v$(VERSION)"
+
+bump-install-mcp-sh:
+	@test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-install-mcp-sh VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1)
+	@export BINARIES_DIR="$(BINARIES_DIR)"; \
+	fetch_sha() { $(fff_fetch_sha); }; \
+	sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-musl)"; \
+	sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-musl)"; \
+	sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \
+	sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \
+	sha_win_intel="$$(fetch_sha fff-mcp-x86_64-pc-windows-msvc.exe)"; \
+	sha_win_arm="$$(fetch_sha fff-mcp-aarch64-pc-windows-msvc.exe)"; \
+	sed -i.bak \
+		-e 's|^PINNED_RELEASE_TAG=".*"|PINNED_RELEASE_TAG="v$(VERSION)"|' \
+		-e 's|^SHA256_X86_64_UNKNOWN_LINUX_MUSL=".*"|SHA256_X86_64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_intel"'"|' \
+		-e 's|^SHA256_AARCH64_UNKNOWN_LINUX_MUSL=".*"|SHA256_AARCH64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_arm"'"|' \
+		-e 's|^SHA256_X86_64_APPLE_DARWIN=".*"|SHA256_X86_64_APPLE_DARWIN="'"$$sha_darwin_intel"'"|' \
+		-e 's|^SHA256_AARCH64_APPLE_DARWIN=".*"|SHA256_AARCH64_APPLE_DARWIN="'"$$sha_darwin_arm"'"|' \
+		-e 's|^SHA256_X86_64_PC_WINDOWS_MSVC=".*"|SHA256_X86_64_PC_WINDOWS_MSVC="'"$$sha_win_intel"'"|' \
+		-e 's|^SHA256_AARCH64_PC_WINDOWS_MSVC=".*"|SHA256_AARCH64_PC_WINDOWS_MSVC="'"$$sha_win_arm"'"|' \
+		"$(FFF_INSTALL_SCRIPT_PATH)" && rm -f "$(FFF_INSTALL_SCRIPT_PATH).bak"; \
+	echo "Bumped $(FFF_INSTALL_SCRIPT_PATH) tag + checksums to v$(VERSION)"
+
 CRATES_TO_PUBLISH= fff-grep fff-query-parser fff-search
 
 publish-crates:
 	@test -n "$(V)" || (echo "V is required. Usage: make publish-crates V=0.2.0" && exit 1)
-	cargo install cargo-edit
+	cargo install cargo-edit --force --locked
 	cargo set-version $(V) || exit 1;
 	@for crate in $(CRATES_TO_PUBLISH); do \
 		cargo publish -p $$crate --allow-dirty $$(if [ -n "$$CI" ]; then echo "--no-verify"; fi) || exit 1; \

README.md

@@ -1,12 +1,17 @@
-<img alt="FFF" src="./assets/logo-orange.png" width="300">
+<a href="./assets/logo-orange.png"><img alt="FFF" src="./assets/logo-orange.png" width="300"></a>
 
 <p>
   <i>A file search toolkit for humans and AI agents. Really fast.</i>
 </p>
 
-Typo-resistant path and content search, frecency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once.
+Typo-resistant path and content search, frequency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once.
+
+Powers file search in [opencode](http://github.com/anomalyco/opencode/), [nushell](https://github.com/nushell/nushell), and many more amazing projects!
 
 Originally started as [Neovim plugin](#neovim-plugin) people loved, but it turned out that plenty of AI harnesses and code editors need the same thing: accurate, fast file search as a library. That is what fff is.
+<p>
+<a href="https://trendshift.io/repositories/26711?utm_source=repository-badge&amp;utm_medium=badge&amp;utm_campaign=badge-repository-26711" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26711" alt="dmtrKovalenko%2Ffff | Trendshift" width="250" height="55"/></a>
+</p>
 
 ---
 
@@ -23,13 +28,55 @@ Works with Claude Code, Codex, OpenCode, Cursor, Cline, and any MCP-capable clie
 
 ### One-line install
 
+Linux / macOS:
+
 ```bash
 curl -L https://dmtrkovalenko.dev/install-fff-mcp.sh | bash
 ```
 
-The script lives at [`install-mcp.sh`](./install-mcp.sh) if you want to read it first.
+Windows (PowerShell):
 
-It prints the exact wiring instructions for your client. Once the server is connected, ask the agent to "use fff" and it picks up the `ffgrep`, `fffind`, and `fff-multi-grep` tools.
+```powershell
+irm https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.ps1 | iex
+```
+
+The scripts live at [`install-mcp.sh`](./install-mcp.sh) and [`install-mcp.ps1`](./install-mcp.ps1) if you want to read them first. They print the exact wiring instructions for your client.
+
+### Homebrew (macOS / Linux)
+
+```bash
+brew install dmtrKovalenko/fff/fff-mcp
+brew upgrade fff-mcp   # after new stable releases
+```
+
+Formula lives in [`Formula/fff-mcp.rb`](./Formula/fff-mcp.rb) in this repo and is **auto-bumped on every stable release** (see `bump-homebrew-formula` in [`.github/workflows/release.yaml`](./.github/workflows/release.yaml)). Installs the prebuilt `fff-mcp` binary from [GitHub releases](https://github.com/dmtrKovalenko/fff/releases).
+
+### Codex setup
+
+Register the installed binary using its absolute path, since Codex desktop sessions may not inherit your interactive shell's `PATH`.
+
+Homebrew:
+
+```bash
+codex mcp add fff -- "$(brew --prefix)/bin/fff-mcp"
+```
+
+One-line installer:
+
+```bash
+codex mcp add fff -- "$HOME/.local/bin/fff-mcp"
+```
+
+This creates an entry in `~/.codex/config.toml` similar to:
+
+```toml
+[mcp_servers.fff]
+command = "/opt/homebrew/bin/fff-mcp"
+```
+
+Use the actual installed path for your system, then restart Codex or start a new task so it loads the server.
+
+Once the server is connected, ask the agent to "use fff" and it picks up the `ffgrep`, `fffind`, and `fff-multi-grep` tools.
 
 ### Recommended agent prompt
 
@@ -94,7 +141,7 @@ The Pi extension swaps pi's native tools for FFF implementations and feeds the i
 
 <details id="neovim-plugin">
 <summary>
-<h2>Neovim plugin</h2>
+<h2>fff.nvim</h2>
 </summary>
 
 Demo on the Linux kernel repo (100k files, 8GB):
@@ -106,8 +153,9 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
 #### lazy.nvim
 
 ```lua
+-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:Lazy clean`
 {
-  'dmtrKovalenko/fff.nvim',
+  'dmtrKovalenko/fff',
   build = function()
     -- downloads a prebuilt binary or falls back to cargo build
     require("fff.download").download_or_build_binary()
@@ -128,9 +176,10 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
       function() require('fff').live_grep({ grep = { modes = { 'fuzzy', 'plain' } } }) end,
       desc = 'Live fffuzy grep',
     },
-    { "fc",
-      function() require('fff').live_grep({ query = vim.fn.expand("<cword>") }) end,
-      desc = 'Search current word',
+    { "fw",
+      function() require('fff').live_grep_under_cursor() end,
+      mode = { 'n', 'x' },
+      desc = 'Search current word / selection',
     },
   },
 }
@@ -139,13 +188,14 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
 #### vim.pack
 
 ```lua
-vim.pack.add({ 'https://github.com/dmtrKovalenko/fff.nvim' })
+-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:packdel fff.nvim`
+vim.pack.add({ 'https://github.com/dmtrKovalenko/fff' })
 
 vim.api.nvim_create_autocmd('PackChanged', {
   callback = function(ev)
     local name, kind = ev.data.spec.name, ev.data.kind
-    if name == 'fff.nvim' and (kind == 'install' or kind == 'update') then
-      if not ev.data.active then vim.cmd.packadd('fff.nvim') end
+    if name == 'fff' and (kind == 'install' or kind == 'update') then
+      if not ev.data.active then vim.cmd.packadd('fff') end
       require('fff.download').download_or_build_binary()
     end
   end,
@@ -164,12 +214,69 @@ vim.keymap.set('n', 'ff', function() require('fff').find_files() end, { desc = '
 ```lua
 require('fff').find_files()                        -- find files in current repo
 require('fff').live_grep()                         -- live content grep
+require('fff').live_grep_under_cursor()            -- grep <cword> in normal, selection in visual
 require('fff').scan_files()                        -- force rescan
 require('fff').refresh_git_status()                -- refresh git status
 require('fff').find_files_in_dir(path)             -- find in a specific dir
 require('fff').change_indexing_directory(new_path) -- change root
+
+-- Programmatic search (no UI). Useful for plugin integrations.
+require('fff').file_search(query, opts)            -- fuzzy search files / dirs / mixed
+require('fff').content_search(query, opts)         -- programmatic grep
 ```
 
+#### `file_search(query, opts)`
+
+Returns a structured result `{ items, scores, total_matched, total_files?, total_dirs?, location? }`. Each item has a `type` field (`"file"` or `"directory"`) and `name` / `relative_path`. File items also expose `size`, `modified`, `git_status`, `is_binary`, and frecency scores.
+
+```lua
+local r = require('fff').file_search('button', {
+  mode             = 'mixed',  -- 'files' (default) | 'directories' | 'mixed'
+  max_results      = 50,
+  page             = 0,        -- 0-based pagination
+  current_file     = nil,      -- path to deprioritize for distance scoring
+  max_threads      = 4,
+  cwd              = nil,      -- switch indexed root if different (see below)
+  wait_for_index_ms = nil,     -- override the default scan wait timeout
+})
+for _, item in ipairs(r.items) do
+  print(item.type, item.relative_path)
+end
+```
+
+#### `content_search(query, opts)`
+
+Returns a `GrepResult` `{ items, total_matched, total_files_searched, total_files, filtered_file_count, next_file_offset, regex_fallback_error? }`. Each match item has `relative_path`, `name`, `line_number`, `col`, `line_content`, `match_ranges`, plus the same file metadata as `file_search`.
+
+```lua
+local r = require('fff').content_search('TODO', {
+  mode                  = 'plain',  -- 'plain' (default) | 'regex' | 'fuzzy'
+  max_file_size         = 10 * 1024 * 1024,
+  max_matches_per_file  = 100,
+  smart_case            = true,
+  page_size             = 50,
+  file_offset           = 0,
+  time_budget_ms        = 0,
+  trim_whitespace       = false,
+  cwd                   = nil,      -- switch indexed root if different
+  wait_for_index_ms     = nil,      -- override the default scan wait timeout
+})
+for _, m in ipairs(r.items) do
+  print(string.format('%s:%d %s', m.relative_path, m.line_number, m.line_content))
+end
+```
+
+Both functions accept the same constraint syntax as the UI pickers (e.g. `git:modified`, `*.rs`, `!test/`, glob patterns).
+
+#### `cwd` and indexing
+
+Both `file_search` and `content_search` honour an optional `cwd` field. The first call to either function lazily initialises the picker at `config.base_path` (your Neovim cwd by default).
+
+- If `cwd` matches the currently indexed root, the call returns immediately against the existing index.
+- If `cwd` differs, the picker is re-indexed at the new root and the call **blocks** (default up to 10 s) until the new picker is installed and its initial scan completes — so callers always get results from the right tree.
+- If the index is still warming up after a `change_indexing_directory`, you can pass `wait_for_index_ms = N` to block for up to `N` ms regardless of whether `cwd` triggered the swap. Pass `0` to skip waiting entirely (useful for fire-and-forget calls where partial results are acceptable).
+- Invalid or non-existent `cwd` paths return an empty result and emit an error via `vim.notify`.
+
 ### Commands
 
 - `:FFFScan`. Rescan files.
@@ -192,15 +299,30 @@ require('fff').setup({
   max_threads = 4,
   lazy_sync = true,
   prompt_vim_mode = false,
+  follow_symlinks = false,
+  -- Allow indexing the user's $HOME directory. Enabled by default.
+  -- Disable if you strictly sure you don't want this, as it makes whole fff error hard
+  enable_home_dir_scanning = true,
+  -- Allow indexing a filesystem root (e.g. `/`, `C:\`). Disabled by default
+  enable_fs_root_scanning = false,
   layout = {
     height = 0.8,
     width = 0.8,
     prompt_position = 'bottom',   -- or 'top'
     preview_position = 'right',   -- 'left' | 'right' | 'top' | 'bottom'
     preview_size = 0.5,
+    -- Border style for the picker windows. Leave unset (nil) to follow the
+    -- global `vim.o.winborder`; set it to override fff's borders independently.
+    border = nil, -- 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | 'none'
+    -- border = {
+    --   { ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' },
+    --   { ' ', ' ', ' ', ' ', ' ' },
+    -- },
+
     flex = { size = 130, wrap = 'top' },
+    min_list_height = 10, --  do not display anything except the list below this threshold
     show_scrollbar = true,
-    path_shorten_strategy = 'middle_number', -- 'middle_number' | 'middle' | 'end'
+    path_shorten_strategy = 'middle_number', -- 'middle_number' | 'middle' | 'end' | 'start'
     anchor = 'center',
   },
   preview = {
@@ -230,6 +352,10 @@ require('fff').setup({
     preview_scroll_down = '<C-d>',
     toggle_debug = '<F2>',
     cycle_grep_modes = '<S-Tab>',
+    insert_newline_escape = '<C-CR>',
+    -- grep mode only: jump cursor to first match of next/prev file group
+    grep_jump_to_next_file = { '<C-A-n>', '<A-Down>' },
+    grep_jump_to_prev_file = { '<C-A-p>', '<A-Up>' },
     cycle_previous_query = '<C-Up>',
     toggle_select = '<Tab>',
     send_to_quickfix = '<C-q>',
@@ -249,6 +375,13 @@ require('fff').setup({
   git = {
     status_text_color = false, -- true to color filenames by git status
   },
+  file_picker = {
+    fuzzy_query_highlighting = false, -- true to highlight fuzzy query matches in file picker results
+  },
+  select = {
+    -- Return winid to open the chosen file in, or nil to open in the original window
+    select_window = function(current_buf, action) --[[ default impl ]] end,
+  },
   grep = {
     max_file_size = 10 * 1024 * 1024,
     max_matches_per_file = 100,
@@ -256,12 +389,31 @@ require('fff').setup({
     time_budget_ms = 150,
     modes = { 'plain', 'regex', 'fuzzy' },
     trim_whitespace = false,
+    enable_filename_constraint = false, -- treat filename-like tokens (e.g. `score.rs`) in a grep query as a file-path filter scoping the search; off = searched as literal text
+    location_format = ':%d:%d', -- printf format for line:col prefix in grep results, e.g. ':%d' for line-only
+  },
+  debug = {
+    enabled = false, -- show the file info panel next to the preview
+    show_scores = false, -- inline scores in the file list
+    -- Per-section toggles for the file info panel. Accepts a boolean shorthand
+    -- (`show_file_info = true|false`) to flip everything at once. The panel
+    -- adapts to width: narrow renders sections vertically, wide renders them
+    -- as a two-column grid. Disable a section to also shrink the panel.
+    show_file_info = {
+      file_info = true, -- size, type, git status, frecency
+      score_breakdown = true, -- total + match type, bonuses, modifiers, penalty
+      -- modified + accessed timestamps; pass a table to hide individual rows:
+      --   timings = { modified = false, accessed = true }
+      timings = true,
+      full_path = true, -- relative path at the bottom (wraps if too long)
+    },
   },
-  debug = { enabled = false, show_scores = false },
   logging = {
-    enabled = true,
+    -- logs will be written in a parent directory of this file path in files like
+    -- `<stem>+<UTC-timestamp>+<pid>.<ext>`. Run :FFFOpenLog to open current one
     log_file = vim.fn.stdpath('log') .. '/fff.log',
     log_level = 'info',
+    retain_runs = 20,
   },
 })
 ```
@@ -283,7 +435,7 @@ Both find and grep accept these tokens to refine a query:
 
 - `git:modified`. One of `modified`, `staged`, `deleted`, `renamed`, `untracked`, `ignored`.
 - `test/`. Any deeply nested children of `test/`.
-- `!something`, `!test/`, `!git:modified`. Exclusion.
+- `!something`, `!test/`, `!git:modified`. Exclusion. Text exclusions need at least 3 alphanumeric-containing characters, so operators like `!=` or `!==` work.
 - `./**/*.{rs,lua}`. Any valid glob, powered by [zlob](https://github.com/dmtrKovalenko/zlob).
 
 Grep-only:
@@ -293,6 +445,20 @@ Grep-only:
 
 Mix freely: `git:modified src/**/*.rs !src/**/mod.rs user controller`.
 
+### Open in invoking window
+
+By default fff.nvim will try to open a file in the most suitable window, so any non-file buffers are not affected. You can customize or disable this by providing:
+
+```lua
+require('fff').setup({
+  select = {
+    select_window = function(_current_buf, _action) return nil end,
+  },
+})
+```
+
+Caveat: the chosen file replaces the buffer in the invoking window even if it's a non-modifiable / special buftype. `winfixbuf` windows still fall back to `:split` to avoid `E1513`.
+
 ### Multi-select and quickfix
 
 - `<Tab>`. Toggle selection (shows a thick `▊` in the signcolumn).
@@ -302,6 +468,51 @@ Mix freely: `git:modified src/**/*.rs !src/**/mod.rs user controller`.
 
 Sign-column indicators are on by default. To color filename text by git status, set `git.status_text_color = true` and adjust the `hl.git_*` groups. See `:help fff.nvim` for the full list.
 
+### Float colors
+
+The picker maps its float content to `NormalFloat` (via `hl.normal`) and the border to `FloatBorder`. Default `FloatBorder` links to `NormalFloat`, so border and content share a background out of the box and the picker reads as a single popup. Override `hl.normal = 'Normal'` to make the picker blend with the editor instead.
+
+For finer control, set `hl.winhl` to override the per-window `winhighlight`. It accepts either a single string applied to every picker window, or a table with optional `prompt`, `list`, `preview`, and `file_info` keys. Missing keys fall back to the default built from `hl.normal`, `hl.border`, and `hl.title`.
+
+```lua
+-- Apply the same winhighlight to all picker windows
+hl = { winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title' }
+
+-- Or override specific windows only
+hl = {
+  winhl = {
+    prompt  = 'Normal:Pmenu,FloatBorder:FloatBorder',
+    list    = 'Normal:NormalFloat,FloatBorder:FloatBorder',
+    preview = 'Normal:NormalFloat,FloatBorder:FloatBorder',
+  },
+}
+```
+
+### File info panel
+
+Enable with `debug.enabled = true`. The panel sits above the preview and shows
+file metadata, score breakdown, timestamps and the full absolute path. It
+adapts to the panel width: at narrow widths sections stack vertically (B2),
+at wide widths sections render as a two-column grid (H2). Each section can be
+disabled individually via `debug.show_file_info`.
+
+Customise the panel via `hl`:
+
+| key                     | default           | used for                           |
+| ----------------------- | ----------------- | ---------------------------------- |
+| `file_info_section`     | `Title`           | section header label               |
+| `file_info_separator`   | `FloatBorder`     | dashes that act as section borders |
+| `file_info_label`       | `Comment`         | row labels (Size, Type, Git, ...)  |
+| `file_info_value`       | `Normal` fg       | plain values                       |
+| `file_info_value_dim`   | `NonText`         | dim values, separators inside rows |
+| `file_info_size`        | `Number`          | file size value                    |
+| `file_info_type`        | `Type`            | filetype value                     |
+| `file_info_path`        | `Directory`       | full path                          |
+| `file_info_total_score` | bold + `Number`   | total score (bold)                 |
+| `file_info_match_type`  | bold + `Special`  | match type (bold)                  |
+| `file_info_score_pos`   | `DiagnosticOk`    | positive score components          |
+| `file_info_score_neg`   | `DiagnosticError` | negative score components          |
+
 ### File filtering
 
 FFF honours `.gitignore`. For picker-only ignores that do not touch git, add a sibling `.ignore` file:
@@ -316,7 +527,9 @@ Run `:FFFScan` to force a rescan.
 ### Troubleshooting
 
 - `:FFFHealth` verifies picker init, optional dependencies, and DB connectivity.
-- `:FFFOpenLog` opens the log file.
+- `:FFFOpenLog` opens the current session's log file.
+- Historical log files are stored near the main log file `<state>/log/fff+<UTC-timestamp>+<pid>.log` (up to 20 files)
+- For a crash backtrace, run `lldb -- nvim` or `gdb -- nvim` and reproduce
 
 </details>
 
@@ -349,6 +562,9 @@ const hits = finder.value.grep("GetOffTheRecordProfile", {
   classifyDefinitions: true,
 });
 
+// Run extremely fast glob matching which is significantly (10-100 times) faster than Bun's and Node implementation
+const rustFiles = finder.value.glob("**/*.rs", { pageSize: 100 });
+
 finder.value.destroy();
 ```
 
@@ -393,9 +609,14 @@ make build-c-lib
 cargo build --release -p fff-c --features zlob
 ```
 
+> The `zlob` feature (requires the [Zig](https://ziglang.org) toolchain) switches both
+> glob matching **and** filesystem traversal to [zlob](https://github.com/dmtrKovalenko/zlob)'s
+> native parallel walker. Without it, the default build uses the pure-Rust
+> [`ignore`](https://crates.io/crates/ignore) (ripgrep) walker and `globset`.
+
 The output is a `cdylib` (`libfff_c.so` / `libfff_c.dylib` / `fff_c.dll`). The header lives at [`crates/fff-c/include/fff.h`](./crates/fff-c/include/fff.h).
 
-Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff.nvim/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages.
+Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages.
 
 ### Install
 
@@ -454,6 +675,35 @@ int main(void) {
 }
 ```
 
+### Versioned options struct (preferred)
+
+For instance creation use [`FffCreateOptions`](./crates/fff-c/include/fff.h) — a
+versioned struct that evolves without ABI breaks. C99 designated
+initializers keep call sites readable and zero-init unspecified fields:
+
+```c
+FffResult *res = fff_create_instance_with(&(FffCreateOptions){
+    .version = FFF_CREATE_OPTIONS_VERSION,
+    .base_path = "/path/to/repo",
+    .ai_mode = true,
+    .watch = true,
+    .enable_fs_root_scanning = false,   // off by default
+    .enable_home_dir_scanning = false,  // off by default
+});
+```
+
+### Glob-only search
+
+`fff_glob` filters indexed files by a single glob pattern, ranks by frecency,
+paginates — bypasses the regular query parser entirely. Use this when you
+already have a literal glob (`*.rs`, `**/*.test.ts`, `src/**`) and don't want
+fuzzy matching layered on top.
+
+```c
+FffResult *res = fff_glob(handle, "**/*.rs", "", 0, 0, 100);
+// FffSearchResult in res->handle, free with fff_free_search_result.
+```
+
 ### Notes
 
 - Every function returning `FffResult*` allocates with Rust's `Box`. Free with `fff_free_result`, do not use malloc's free
@@ -466,6 +716,75 @@ Source: [`crates/fff-c/`](./crates/fff-c/).
 
 Stable C ABI. Bind from C/C++, Zig, Go via cgo, Python via ctypes, or anything with C FFI.
 
+<details id="python-bindings">
+<summary>
+<h2>Python bindings</h2>
+</summary>
+
+### Install
+
+```bash
+pip install fff-search
+```
+
+Or build and install from source:
+
+```bash
+cd packages/fff-python
+uv sync --all-extras
+uv run maturin develop --release
+```
+
+### Basic usage
+
+```python
+from fff import FileFinder
+
+with FileFinder("/path/to/project", watch=False) as finder:
+    finder.wait_for_scan_blocking(timeout_ms=5000)
+
+    result = finder.search("main")
+    for item, score in zip(result.items, result.scores):
+        print(f"{item.relative_path}: {score.total}")
+
+    hits = finder.grep("class Profile", mode="plain", before_context=1, after_context=1)
+```
+
+### Async usage
+
+`wait_for_scan` is a coroutine that polls scan status and yields to the event
+loop, so it never blocks other tasks. Use `wait_for_scan_blocking` from
+synchronous code.
+
+```python
+import asyncio
+from fff import FileFinder
+
+async def main():
+    with FileFinder("/path/to/project", watch=False) as finder:
+        await finder.wait_for_scan(timeout_ms=5000)
+        result = finder.search("main")
+        print(result)
+
+asyncio.run(main())
+```
+
+### What you get
+
+- `search`, `glob`, `directory_search`, `mixed_search` — frecency-ranked fuzzy file/dir search
+- `grep` / `multi_grep` — plain, regex, or fuzzy content search with context lines and cursor pagination
+- `track_query` / `get_historical_query` — optional frecency and query-history databases
+- `reindex`, `refresh_git_status`, `scan_progress`, `health_check` — lifecycle and diagnostics
+
+Typed result objects (`FileItem`, `Score`, `GrepMatch`, …) with `py.typed`
+stubs included. Ships as an `abi3` wheel compatible with Python 3.10+.
+
+Source: [`packages/fff-python/`](./packages/fff-python/).
+
+</details>
+
+Native Python bindings built with PyO3. Use them for notebooks, agent scripts, or any Python tool that needs fast file search.
+
 ---
 
 ## What is FFF and why use it over ripgrep or fzf?
@@ -474,7 +793,7 @@ FFF is a file search library, not a CLI. Ripgrep and fzf are great tools, but th
 
 FFF keeps the index and the file cache resident in one long-lived process and exposes the same Rust core through four thin layers: a native crate (`fff-search`), a C library (`libfff_c`), a Node/Bun SDK (`@ff-labs/fff-node`), and an MCP server. You call `FileFinder.create()` once, then every subsequent search hits warm memory. On a 500k-file Chromium checkout, that is the difference between 3-9 **SECONDS** per ripgrep spawn and sub-10 ms per FFF query.
 
-Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema".
+Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm. It is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "\*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema".
 
 ### Why a programmatic API matters
 
@@ -487,7 +806,7 @@ Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it
 ### What the core actually does
 
 - **Frecency-ranked fuzzy matching.** Every indexed file carries an access score and a modification score. Searches rank files you have opened recently and frequently above cold results. This is the same idea as VS Code's recently-opened list, but applied to every search result, not just a sidebar.
-- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghm/frizbee)-derived core) that survives dropped characters and reorderings.
+- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghen/frizbee)-derived core) that survives dropped characters and reorderings.
 - **Content grep with three modes.** Plain literal (SIMD memmem), regex (the Rust `regex` crate), and fuzzy (Smith-Waterman per line). Auto-detects which mode to use from the pattern, falls back to fuzzy when a plain search returns zero hits.
 - **Multi-pattern OR search.** SIMD Aho-Corasick for "find any of these 20 identifiers at once", which is faster than regex alternation and a lot faster than 20 separate ripgrep runs.
 - **Background file watcher.** The index updates as files change. You never pay for a rescan on the hot path.
@@ -508,7 +827,6 @@ Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it
 
 Yes, fff fundamentally requires more memory than calling a single child process. That is the primary source of the speedup. In practice, alongside one of the most popular file search pickers for Neovim, [fff ends up using less RAM than a burst of ripgrep invocations](https://x.com/neogoose_btw/status/2041606853155811442).
 
-
 FFF also keeps a content index, around 360 bytes per indexed file, so roughly 36 MB for a 100k-file repo. Not every file is indexed - binaries, oversized files, and anything not eligible for grep are skipped. If even that footprint is too much, the index can be backed by a memory-mapped file instead of anonymous RAM.
 
 ### What this means in practice
@@ -533,7 +851,7 @@ If you are running one grep from a terminal, `rg` is still the right tool. If yo
 - `crates/fff-nvim` - Lua/mlua bindings for the Neovim plugin.
 - `crates/fff-mcp` - MCP server binary.
 - `packages/fff-node` - Node.js SDK (`@ff-labs/fff-node`).
-- `packages/fff-bun` - Bun SDK (`@ff-labs/fff-node`).
+- `packages/fff-bun` - Bun SDK (`@ff-labs/fff-bun`).
 - `packages/pi-fff` - pi extension (`@ff-labs/pi-fff`).
 - `lua/` - Neovim-side plugin code.
 
@@ -544,3 +862,15 @@ Bug reports and pull requests welcome. Agentic coding tools are welcome to be us
 ## License
 
 [MIT](./LICENSE) & open source forever.
+
+## FAQ
+
+### What does FFF stand for?
+
+There is intentionally no single canonical definition. Pick your favourite:
+
+- **F**ast **F**ile **F**inder
+- **F**uzzy **F**ile **F**inder
+- will search **F**iles **F**or **F**ood
+
+The brand hex is `#F87216`, not `#FFF`. Logo variants: [orange](./assets/logo-orange.png) · [dark](./assets/logo-dark.png) · [light](./assets/logo-light.png).

_typos.toml

@@ -6,6 +6,8 @@ noice = "noice"
 fo = "fo"
 ba = "ba"
 ue = "ue"
+# file extensions that look like typos
+thm = "thm"
 # some typos we use for tests
 comparsion = "comparsion"
 modfiers = "modfiers"

bun.lock

@@ -10,35 +10,24 @@
     "packages/fff-bun": {
       "name": "@ff-labs/fff-bun",
       "version": "0.1.37",
-      "bin": {
-        "fff": "./scripts/cli.ts",
-        "fff-demo": "./examples/search.ts",
-        "fff-grep": "./examples/grep.ts",
-      },
       "devDependencies": {
         "@types/bun": "^1.3.8",
         "typescript": "^5.0.0",
       },
       "optionalDependencies": {
-        "@ff-labs/fff-bun-darwin-arm64": "0.0.0",
-        "@ff-labs/fff-bun-darwin-x64": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-musl": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-musl": "0.0.0",
-        "@ff-labs/fff-bun-win32-arm64": "0.0.0",
-        "@ff-labs/fff-bun-win32-x64": "0.0.0",
-      },
-      "peerDependencies": {
-        "bun": ">=1.0.0",
+        "@ff-labs/fff-bin-darwin-arm64": "0.0.0",
+        "@ff-labs/fff-bin-darwin-x64": "0.0.0",
+        "@ff-labs/fff-bin-linux-arm64-gnu": "0.0.0",
+        "@ff-labs/fff-bin-linux-arm64-musl": "0.0.0",
+        "@ff-labs/fff-bin-linux-x64-gnu": "0.0.0",
+        "@ff-labs/fff-bin-linux-x64-musl": "0.0.0",
+        "@ff-labs/fff-bin-win32-arm64": "0.0.0",
+        "@ff-labs/fff-bin-win32-x64": "0.0.0",
       },
     },
     "packages/fff-node": {
       "name": "@ff-labs/fff-node",
       "version": "0.1.37",
-      "bin": {
-        "fff-node": "./dist/scripts/cli.js",
-      },
       "dependencies": {
         "ffi-rs": "^1.0.0",
       },
@@ -47,18 +36,89 @@
         "typescript": "^5.0.0",
       },
       "optionalDependencies": {
-        "@ff-labs/fff-bun-darwin-arm64": "0.0.0",
-        "@ff-labs/fff-bun-darwin-x64": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-musl": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-musl": "0.0.0",
-        "@ff-labs/fff-bun-win32-arm64": "0.0.0",
-        "@ff-labs/fff-bun-win32-x64": "0.0.0",
+        "@ff-labs/fff-bin-darwin-arm64": "0.0.0",
+        "@ff-labs/fff-bin-darwin-x64": "0.0.0",
+        "@ff-labs/fff-bin-linux-arm64-gnu": "0.0.0",
+        "@ff-labs/fff-bin-linux-arm64-musl": "0.0.0",
+        "@ff-labs/fff-bin-linux-x64-gnu": "0.0.0",
+        "@ff-labs/fff-bin-linux-x64-musl": "0.0.0",
+        "@ff-labs/fff-bin-win32-arm64": "0.0.0",
+        "@ff-labs/fff-bin-win32-x64": "0.0.0",
+      },
+    },
+    "packages/pi-fff": {
+      "name": "@ff-labs/pi-fff",
+      "version": "0.6.0",
+      "dependencies": {
+        "@ff-labs/fff-bun": "*",
+        "@ff-labs/fff-node": "*",
+      },
+      "devDependencies": {
+        "@types/node": "^22.0.0",
+        "typescript": "^5.0.0",
+      },
+      "peerDependencies": {
+        "@earendil-works/pi-coding-agent": "*",
+        "@earendil-works/pi-tui": "*",
+        "@sinclair/typebox": "*",
       },
     },
   },
   "packages": {
+    "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.91.1", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw=="],
+
+    "@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="],
+
+    "@aws-crypto/sha256-browser": ["@aws-crypto/sha256-browser@5.2.0", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "@aws-crypto/supports-web-crypto": "^5.2.0", "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw=="],
+
+    "@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="],
+
+    "@aws-crypto/supports-web-crypto": ["@aws-crypto/supports-web-crypto@5.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg=="],
+
+    "@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="],
+
+    "@aws-sdk/client-bedrock-runtime": ["@aws-sdk/client-bedrock-runtime@3.1048.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.11", "@aws-sdk/credential-provider-node": "^3.972.42", "@aws-sdk/eventstream-handler-node": "^3.972.16", "@aws-sdk/middleware-eventstream": "^3.972.12", "@aws-sdk/middleware-websocket": "^3.972.19", "@aws-sdk/token-providers": "3.1048.0", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ=="],
+
+    "@aws-sdk/core": ["@aws-sdk/core@3.974.18", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@aws-sdk/xml-builder": "^3.972.28", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.6", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-JDYCPI0j7zGrzXTDFsLB346cxss7J/AxH7+O0MzWlqppJBEyB9Qe6TQXRL6iwLUo/xZkNv9KFmBL2hqElmwW0g=="],
+
+    "@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.44", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-3hKJVrZ7bqXzDAXCQp+OaQ1ASN+vWstaNuEH418wQVl//cRZhqhfR9Bjk1qIWmgUGe8/D3gdO73PgidRj378EQ=="],
+
+    "@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.46", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/fetch-http-handler": "^5.4.6", "@smithy/node-http-handler": "^4.7.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-VhwC9pGAZHhiQ2xSViyOPDFqvr9aRxGCAXZtADsUhU3R65nad7y//CwynE6mQnWNR+suRlqE79W36IVayL+m1g=="],
+
+    "@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.972.50", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/credential-provider-env": "^3.972.44", "@aws-sdk/credential-provider-http": "^3.972.46", "@aws-sdk/credential-provider-login": "^3.972.49", "@aws-sdk/credential-provider-process": "^3.972.44", "@aws-sdk/credential-provider-sso": "^3.972.49", "@aws-sdk/credential-provider-web-identity": "^3.972.49", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/credential-provider-imds": "^4.3.7", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-09Xi6ovxiK42+De/qBGF71sT5F2bWgYM+1fFyDwSOpy1xpsQ5R/naIu7MVDpH6Dic36QNc8dAv4KADtMGK2JYg=="],
+
+    "@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.49", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-EfJF/1Fh9mI4pZyoheU2RY9xUhTcugIZNkD63+orXMkYj/QXacJNbKVDUK90Yv5hE+aX+rt9J/EZ9Qr3vKOa7g=="],
+
+    "@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.52", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.44", "@aws-sdk/credential-provider-http": "^3.972.46", "@aws-sdk/credential-provider-ini": "^3.972.50", "@aws-sdk/credential-provider-process": "^3.972.44", "@aws-sdk/credential-provider-sso": "^3.972.49", "@aws-sdk/credential-provider-web-identity": "^3.972.49", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/credential-provider-imds": "^4.3.7", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-7QX+PbyiWBEOVipJq8Nke/TqXT6lAPLE7fvTaopa39/IVWuLfS+Fzdy71sZJONf/mLGgmtj6aU17+REw3+aRrw=="],
+
+    "@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.44", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-V+UUhZpRP7QDRhi+qgBDisM9tUBnYmMje8Bk77A6MZsfeGeGdMsQXmaHP1CDYFcept0o/Rz5g2Y0TMeVlG9dzg=="],
+
+    "@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.972.49", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/token-providers": "3.1063.0", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-9QqOYGuh5tZ76OzaT68kwI78AH+5lS/uZGGvkfxb3fc8FzRrIz2jOufNTliEBEeSAwmgK2rWLNsK+IB3zbtNPA=="],
+
+    "@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.49", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-IYx1lN38MnnPXv+NBLpuATu0cZakbZ321TAfjW+aVkw7HIJF38YnEwdeEO55MSl3pl7hIX1IvvnD6EmnAzmAJw=="],
+
+    "@aws-sdk/eventstream-handler-node": ["@aws-sdk/eventstream-handler-node@3.972.20", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-qr/S1iFCDIXlZwlZPaCqjKcHbJFr9scIFUhbh2+SrwPXZvRhyOUWjVDJpp8xoU4qrrMR0PqK1Yw5C2sSj7xAyw=="],
+
+    "@aws-sdk/middleware-eventstream": ["@aws-sdk/middleware-eventstream@3.972.16", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-KR2Gdui/QLbkdG9FxW3vk/vIa8KiDP5vQBNERo7MmlPHjn23GXJ53Cq5P/ok7/ALbTUiYZ78DiBHoDcvzPWvgQ=="],
+
+    "@aws-sdk/middleware-websocket": ["@aws-sdk/middleware-websocket@3.972.26", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/fetch-http-handler": "^5.4.6", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-foM3KvxGBHY9lRIm6C9JJJ5haodtXfJPPgJQcv5/c4A2pN4I7tlnOjh1o2d8Il1Y/j6GWOw3YeIYc2/VYjtGVQ=="],
+
+    "@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.17", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.18", "@aws-sdk/signature-v4-multi-region": "^3.996.32", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/fetch-http-handler": "^5.4.6", "@smithy/node-http-handler": "^4.7.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-lDRgraoTfKRawUyc176Ow93mrNrOho/x+EoK4C+lKU+vKkHWhNhzvSMVAx0WEJUJoeQxxDN5ZdKMfiGEyNejig=="],
+
+    "@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.32", "", { "dependencies": { "@aws-sdk/types": "^3.973.11", "@smithy/signature-v4": "^5.4.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-llvApLcsWtmRFhG2wT3WIp1CmDeRaIYutqty1ZZXoMzK7TiJ6MOLOimk9eXUS8PwgG4ew4pa4QAbt0lfhn++1w=="],
+
+    "@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1048.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.11", "@aws-sdk/nested-clients": "^3.997.9", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA=="],
+
+    "@aws-sdk/types": ["@aws-sdk/types@3.973.11", "", { "dependencies": { "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-YjS0qFuECClRh4qhEyW8XagW0fwEPBeZ1cfsW/gU73Kh/ExFILxbzxOfPCmzF/2DwEvhvsHYt0b0qnvStwKYrg=="],
+
+    "@aws-sdk/util-locate-window": ["@aws-sdk/util-locate-window@3.965.6", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-ZfHjfwSzeXj+Lg9AK5ZNmeDkXev6V+w2tn1t4kgDdRtUaRCthepTQiFwbD06EF9oNGH4LaLg+Mb6U16Ypv5bSw=="],
+
+    "@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.28", "", { "dependencies": { "@smithy/types": "^4.14.3", "fast-xml-parser": "5.7.3", "tslib": "^2.6.2" } }, "sha512-lI/l3c/vPvsxmspzV63NfS3x9q4CkMmdhJy4QiM+NThAufVkDvi/PZZQ6xETnICL0UD7jI808pY83gllf86RFg=="],
+
+    "@aws/lambda-invoke-store": ["@aws/lambda-invoke-store@0.2.4", "", {}, "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ=="],
+
+    "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="],
+
     "@biomejs/biome": ["@biomejs/biome@2.4.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.4", "@biomejs/cli-darwin-x64": "2.4.4", "@biomejs/cli-linux-arm64": "2.4.4", "@biomejs/cli-linux-arm64-musl": "2.4.4", "@biomejs/cli-linux-x64": "2.4.4", "@biomejs/cli-linux-x64-musl": "2.4.4", "@biomejs/cli-win32-arm64": "2.4.4", "@biomejs/cli-win32-x64": "2.4.4" }, "bin": { "biome": "bin/biome" } }, "sha512-tigwWS5KfJf0cABVd52NVaXyAVv4qpUXOWJ1rxFL8xF1RVoeS2q/LK+FHgYoKMclJCuRoCWAPy1IXaN9/mS61Q=="],
 
     "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jZ+Xc6qvD6tTH5jM6eKX44dcbyNqJHssfl2nnwT6vma6B1sj7ZLTGIk6N5QwVBs5xGN52r3trk5fgd3sQ9We9A=="],
@@ -77,38 +137,96 @@
 
     "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.4", "", { "os": "win32", "cpu": "x64" }, "sha512-gnOHKVPFAAPrpoPt2t+Q6FZ7RPry/FDV3GcpU53P3PtLNnQjBmKyN2Vh/JtqXet+H4pme8CC76rScwdjDcT1/A=="],
 
+    "@earendil-works/pi-agent-core": ["@earendil-works/pi-agent-core@0.78.1", "", { "dependencies": { "@earendil-works/pi-ai": "^0.78.1", "ignore": "7.0.5", "typebox": "1.1.38", "yaml": "2.9.0" } }, "sha512-oPwVRkkAvyKPWyM7E4k+EaTNmynbYn7ZLG/LBh9BUnMNb2gvpMp+VQ420R6JCJ20uogSqrHnWTyosSa/rU8lVw=="],
+
+    "@earendil-works/pi-ai": ["@earendil-works/pi-ai@0.78.1", "", { "dependencies": { "@anthropic-ai/sdk": "0.91.1", "@aws-sdk/client-bedrock-runtime": "3.1048.0", "@google/genai": "1.52.0", "@mistralai/mistralai": "2.2.1", "@smithy/node-http-handler": "4.7.3", "http-proxy-agent": "7.0.2", "https-proxy-agent": "7.0.6", "openai": "6.26.0", "partial-json": "0.1.7", "typebox": "1.1.38" }, "bin": { "pi-ai": "dist/cli.js" } }, "sha512-CM2pkTs1iupG/maw381lC9Q/Y/aQaMGK7GILc28ttImD0ci3LDwKroDsGkWbly5JIy3iqxdRxB9JlG7vvzCzTg=="],
+
+    "@earendil-works/pi-coding-agent": ["@earendil-works/pi-coding-agent@0.78.1", "", { "dependencies": { "@earendil-works/pi-agent-core": "^0.78.1", "@earendil-works/pi-ai": "^0.78.1", "@earendil-works/pi-tui": "^0.78.1", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "cross-spawn": "7.0.6", "diff": "8.0.4", "glob": "13.0.6", "highlight.js": "10.7.3", "hosted-git-info": "9.0.3", "ignore": "7.0.5", "jiti": "2.7.0", "minimatch": "10.2.5", "proper-lockfile": "4.1.2", "typebox": "1.1.38", "undici": "8.3.0", "yaml": "2.9.0" }, "optionalDependencies": { "@mariozechner/clipboard": "0.3.9" }, "bin": { "pi": "dist/cli.js" } }, "sha512-Syjf6Ib8UoY5t9ZdKjp0BRrQZuFkFBc8j2KEU9zG/ZnmYPcAxYeioofdv2Q3MEXnHEX2U8sKQptkSnJIdMsd0g=="],
+
+    "@earendil-works/pi-tui": ["@earendil-works/pi-tui@0.78.1", "", { "dependencies": { "get-east-asian-width": "1.6.0", "marked": "15.0.12" } }, "sha512-07GVQo/38a0yvIPlWDr3RJn1B8gk3ZuIX9h2oIQ+Biyu3JN0KppWmgWHfaWRydQgse5JtC++KDw5MWaIRnV0mw=="],
+
     "@ff-labs/fff-bun": ["@ff-labs/fff-bun@workspace:packages/fff-bun"],
 
     "@ff-labs/fff-node": ["@ff-labs/fff-node@workspace:packages/fff-node"],
 
-    "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.3.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-PXgg5gqcS/rHwa1hF0JdM1y5TiyejVrMHoBmWY/DjtfYZoFTXie1RCFOkoG0b5diOOmUcuYarMpH7CSNTqwj+w=="],
+    "@ff-labs/pi-fff": ["@ff-labs/pi-fff@workspace:packages/pi-fff"],
 
-    "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.3.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-Nhssuh7GBpP5PiDSOl3+qnoIG7PJo+ec2oomDevnl9pRY6x6aD2gRt0JE+uf+A8Om2D6gjeHCxjEdrw5ZHE8mA=="],
+    "@google/genai": ["@google/genai@1.52.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q=="],
 
-    "@oven/bun-darwin-x64-baseline": ["@oven/bun-darwin-x64-baseline@1.3.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-w1gaTlqU0IJCmJ1X+PGHkdNU1n8Gemx5YKkjhkJIguvFINXEBB5U1KG82QsT65Tk4KyNMfbLTlmy4giAvUoKfA=="],
+    "@mariozechner/clipboard": ["@mariozechner/clipboard@0.3.9", "", { "optionalDependencies": { "@mariozechner/clipboard-darwin-arm64": "0.3.9", "@mariozechner/clipboard-darwin-universal": "0.3.9", "@mariozechner/clipboard-darwin-x64": "0.3.9", "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", "@mariozechner/clipboard-linux-x64-musl": "0.3.9", "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" } }, "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA=="],
 
-    "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.3.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-OUgPHfL6+PM2Q+tFZjcaycN3D7gdQdYlWnwMI31DXZKY1r4HINWk9aEz9t/rNaHg65edwNrt7dsv9TF7xK8xIA=="],
+    "@mariozechner/clipboard-darwin-arm64": ["@mariozechner/clipboard-darwin-arm64@0.3.9", "", { "os": "darwin", "cpu": "arm64" }, "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ=="],
 
-    "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.3.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-Ui5pAgM7JE9MzHokF0VglRMkbak3lTisY4Mf1AZutPACXWgKJC5aGrgnHBfkl7QS6fEeYb0juy1q4eRznRHOsw=="],
+    "@mariozechner/clipboard-darwin-universal": ["@mariozechner/clipboard-darwin-universal@0.3.9", "", { "os": "darwin" }, "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ=="],
 
-    "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-bzUgYj/PIZziB/ZesIP9HUyfvh6Vlf3od+TrbTTyVEuCSMKzDPQVW/yEbRp0tcHO3alwiEXwJDrWrHAguXlgiQ=="],
+    "@mariozechner/clipboard-darwin-x64": ["@mariozechner/clipboard-darwin-x64@0.3.9", "", { "os": "darwin", "cpu": "x64" }, "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg=="],
 
-    "@oven/bun-linux-x64-baseline": ["@oven/bun-linux-x64-baseline@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-oqvMDYpX6dGJO03HgO5bXuccEsH3qbdO3MaAiAlO4CfkBPLUXz3N0DDElg5hz0L6ktdDVKbQVE5lfe+LAUISQg=="],
+    "@mariozechner/clipboard-linux-arm64-gnu": ["@mariozechner/clipboard-linux-arm64-gnu@0.3.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw=="],
 
-    "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-poVXvOShekbexHq45b4MH/mRjQKwACAC8lHp3Tz/hEDuz0/20oncqScnmKwzhBPEpqJvydXficXfBYuSim8opw=="],
+    "@mariozechner/clipboard-linux-arm64-musl": ["@mariozechner/clipboard-linux-arm64-musl@0.3.9", "", { "os": "linux", "cpu": "arm64" }, "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ=="],
 
-    "@oven/bun-linux-x64-musl-baseline": ["@oven/bun-linux-x64-musl-baseline@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-/hOZ6S1VsTX6vtbhWVL9aAnOrdpuO54mAGUWpTdMz7dFG5UBZ/VUEiK0pBkq9A1rlBk0GeD/6Y4NBFl8Ha7cRA=="],
+    "@mariozechner/clipboard-linux-riscv64-gnu": ["@mariozechner/clipboard-linux-riscv64-gnu@0.3.9", "", { "os": "linux", "cpu": "none" }, "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw=="],
 
-    "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.3.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-GXbz2swvN2DLw2dXZFeedMxSJtI64xQ9xp9Eg7Hjejg6mS2E4dP1xoQ2yAo2aZPi/2OBPAVaGzppI2q20XumHA=="],
+    "@mariozechner/clipboard-linux-x64-gnu": ["@mariozechner/clipboard-linux-x64-gnu@0.3.9", "", { "os": "linux", "cpu": "x64" }, "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw=="],
 
-    "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.3.10", "", { "os": "win32", "cpu": "x64" }, "sha512-qaS1In3yfC/Z/IGQriVmF8GWwKuNqiw7feTSJWaQhH5IbL6ENR+4wGNPniZSJFaM/SKUO0e/YCRdoVBvgU4C1g=="],
+    "@mariozechner/clipboard-linux-x64-musl": ["@mariozechner/clipboard-linux-x64-musl@0.3.9", "", { "os": "linux", "cpu": "x64" }, "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ=="],
 
-    "@oven/bun-windows-x64-baseline": ["@oven/bun-windows-x64-baseline@1.3.10", "", { "os": "win32", "cpu": "x64" }, "sha512-gh3UAHbUdDUG6fhLc1Csa4IGdtghue6U8oAIXWnUqawp6lwb3gOCRvp25IUnLF5vUHtgfMxuEUYV7YA2WxVutw=="],
+    "@mariozechner/clipboard-win32-arm64-msvc": ["@mariozechner/clipboard-win32-arm64-msvc@0.3.9", "", { "os": "win32", "cpu": "arm64" }, "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ=="],
+
+    "@mariozechner/clipboard-win32-x64-msvc": ["@mariozechner/clipboard-win32-x64-msvc@0.3.9", "", { "os": "win32", "cpu": "x64" }, "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA=="],
+
+    "@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="],
+
+    "@nodable/entities": ["@nodable/entities@2.1.1", "", {}, "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg=="],
+
+    "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="],
+
+    "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="],
+
+    "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="],
+
+    "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="],
+
+    "@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="],
+
+    "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="],
+
+    "@protobufjs/inquire": ["@protobufjs/inquire@1.1.2", "", {}, "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw=="],
+
+    "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="],
+
+    "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="],
+
+    "@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="],
+
+    "@silvia-odwyer/photon-node": ["@silvia-odwyer/photon-node@0.3.4", "", {}, "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA=="],
+
+    "@sinclair/typebox": ["@sinclair/typebox@0.34.49", "", {}, "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A=="],
+
+    "@smithy/core": ["@smithy/core@3.24.6", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-wBXDRup6UU97VKyaiRo8AssnfStPtG0oAAfpq/bC0a1YYau8pM86YB4kM6ccoVi1mS8l/UHbn9oDM+7uozr/ug=="],
+
+    "@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.3.8", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-5cAM+KZC02sTqDt6NaLXyu50M/GNMd1eTzDVR8Lb0BBsVtu7RWHo47VPPEEv1vt3Yub6uzr+M5FHC+GtoT0USg=="],
+
+    "@smithy/fetch-http-handler": ["@smithy/fetch-http-handler@5.4.6", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-FEwEYJ1jlBKdhe9TPzfghEi1bP55ZeEImlDkEa62bBBYzUcnB6RUCyuiS2mqKt6ZVjUbBgcNhzfIctH+Hevx9g=="],
+
+    "@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
+
+    "@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.3", "", { "dependencies": { "@smithy/core": "^3.24.3", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA=="],
+
+    "@smithy/signature-v4": ["@smithy/signature-v4@5.4.6", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-Ojg4B6oIDlIr1R86xCDJt1zJWnYa0VINmqdjfe9qxWjdRivHalZ3iSlQgVqYbW0MdpFOC5XfHEWsnbmdnpIILQ=="],
+
+    "@smithy/types": ["@smithy/types@4.14.3", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-YupL0ZWmFtJexUN2cHzkvvF/b9pKrtAIfT1o7/oY/Ppu8IYeZ+lDPM5vZdQJaSeA132dJCqojjGC9NhXeF71VQ=="],
+
+    "@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
+
+    "@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
 
     "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
 
     "@types/node": ["@types/node@22.19.15", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg=="],
 
+    "@types/retry": ["@types/retry@0.12.0", "", {}, "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="],
+
     "@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="],
 
     "@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="],
@@ -131,18 +249,166 @@
 
     "@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="],
 
-    "bun": ["bun@1.3.10", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.3.10", "@oven/bun-darwin-x64": "1.3.10", "@oven/bun-darwin-x64-baseline": "1.3.10", "@oven/bun-linux-aarch64": "1.3.10", "@oven/bun-linux-aarch64-musl": "1.3.10", "@oven/bun-linux-x64": "1.3.10", "@oven/bun-linux-x64-baseline": "1.3.10", "@oven/bun-linux-x64-musl": "1.3.10", "@oven/bun-linux-x64-musl-baseline": "1.3.10", "@oven/bun-windows-aarch64": "1.3.10", "@oven/bun-windows-x64": "1.3.10", "@oven/bun-windows-x64-baseline": "1.3.10" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-S/CXaXXIyA4CMjdMkYQ4T2YMqnAn4s0ysD3mlsY4bUiOCqGlv28zck4Wd4H4kpvbekx15S9mUeLQ7Uxd0tYTLA=="],
+    "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
+
+    "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="],
+
+    "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
+
+    "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="],
+
+    "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
+
+    "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="],
+
+    "buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="],
 
     "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
 
+    "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
+
+    "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
+
+    "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="],
+
+    "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
+
+    "diff": ["diff@8.0.4", "", {}, "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw=="],
+
+    "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="],
+
+    "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
+
+    "fast-xml-builder": ["fast-xml-builder@1.2.0", "", { "dependencies": { "path-expression-matcher": "^1.5.0", "xml-naming": "^0.1.0" } }, "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q=="],
+
+    "fast-xml-parser": ["fast-xml-parser@5.7.3", "", { "dependencies": { "@nodable/entities": "^2.1.0", "fast-xml-builder": "^1.1.7", "path-expression-matcher": "^1.5.0", "strnum": "^2.2.3" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg=="],
+
+    "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
+
     "ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="],
 
+    "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="],
+
+    "gaxios": ["gaxios@7.1.5", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2" } }, "sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg=="],
+
+    "gcp-metadata": ["gcp-metadata@8.1.2", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="],
+
+    "get-east-asian-width": ["get-east-asian-width@1.6.0", "", {}, "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA=="],
+
+    "glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="],
+
+    "google-auth-library": ["google-auth-library@10.7.0", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-QpTAbNJ36TliZLx3TTtahR8HG0hN9RllL1e3FymOvQSIKK8JmgV58H924ub2wa2DsS3ANjjP1Aw1N+Ramc8hqQ=="],
+
+    "google-logging-utils": ["google-logging-utils@1.1.3", "", {}, "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA=="],
+
+    "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
+
+    "highlight.js": ["highlight.js@10.7.3", "", {}, "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A=="],
+
+    "hosted-git-info": ["hosted-git-info@9.0.3", "", { "dependencies": { "lru-cache": "^11.1.0" } }, "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg=="],
+
+    "http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="],
+
+    "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
+
+    "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="],
+
+    "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
+
+    "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="],
+
+    "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],
+
+    "json-schema-to-ts": ["json-schema-to-ts@3.1.1", "", { "dependencies": { "@babel/runtime": "^7.18.3", "ts-algebra": "^2.0.0" } }, "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g=="],
+
+    "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="],
+
+    "jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="],
+
+    "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="],
+
+    "lru-cache": ["lru-cache@11.5.1", "", {}, "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A=="],
+
+    "marked": ["marked@15.0.12", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA=="],
+
+    "minimatch": ["minimatch@10.2.5", "", { "dependencies": { "brace-expansion": "^5.0.5" } }, "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg=="],
+
+    "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="],
+
+    "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
+
+    "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
+
+    "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="],
+
+    "openai": ["openai@6.26.0", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA=="],
+
+    "p-retry": ["p-retry@4.6.2", "", { "dependencies": { "@types/retry": "0.12.0", "retry": "^0.13.1" } }, "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ=="],
+
+    "partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="],
+
+    "path-expression-matcher": ["path-expression-matcher@1.5.0", "", {}, "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ=="],
+
+    "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
+
+    "path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="],
+
+    "proper-lockfile": ["proper-lockfile@4.1.2", "", { "dependencies": { "graceful-fs": "^4.2.4", "retry": "^0.12.0", "signal-exit": "^3.0.2" } }, "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA=="],
+
+    "protobufjs": ["protobufjs@7.6.2", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ=="],
+
+    "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="],
+
+    "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
+
+    "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
+
+    "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
+
+    "signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="],
+
+    "strnum": ["strnum@2.3.0", "", {}, "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q=="],
+
+    "ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="],
+
+    "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
+
+    "typebox": ["typebox@1.1.38", "", {}, "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA=="],
+
     "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
 
+    "undici": ["undici@8.3.0", "", {}, "sha512-TkUDgb6tl7KOGZ+7e8E3d2FYgUQgF6z5YypqjWmixVQSQERFcVrVg0ySADm2LVLRh5ljAaHTCR5Fmz3Q34rB7Q=="],
+
     "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
 
+    "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="],
+
+    "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
+
+    "ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="],
+
+    "xml-naming": ["xml-naming@0.1.0", "", {}, "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw=="],
+
+    "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="],
+
+    "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="],
+
+    "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
+
+    "@aws-sdk/credential-provider-http/@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.7", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-ZAFvHXrEk6K180EVhmZVg8GU5pUH5BSFqRs27JW3j1qEFx9YyYwWFx17x/MHcjALYimGAji7qEOlF1++be+G5A=="],
+
+    "@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1063.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.18", "@aws-sdk/nested-clients": "^3.997.17", "@aws-sdk/types": "^3.973.11", "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-nYDaWWdzjKiDP5xj8k4oUgcYd4WPgzfAOgdU5vJsaqH/07Dfvm7ffisHCFJ+NEl7kUC9JEIUxh0kznvenbo3NQ=="],
+
+    "@aws-sdk/nested-clients/@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.7", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-ZAFvHXrEk6K180EVhmZVg8GU5pUH5BSFqRs27JW3j1qEFx9YyYwWFx17x/MHcjALYimGAji7qEOlF1++be+G5A=="],
+
     "bun-types/@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="],
 
+    "p-retry/retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="],
+
+    "protobufjs/@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="],
+
     "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
+
+    "protobufjs/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
   }
 }

crates/fff-c/Cargo.toml

@@ -1,23 +1,24 @@
 [package]
 name = "fff-c"
-version = "0.6.4"
+version = "0.10.4"
 edition = "2024"
 description = "Raw C api of FFF file finder"
 license = "MIT"
 
+[lints]
+workspace = true
+
 [lib]
 crate-type = ["cdylib"]
 
 [features]
-default = []
-zlob = ["fff/zlob"]
+default = ["ripgrep"] # use ripgrep base crates to avoid requiring zig for rust crate
+ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"]
+zlob = ["fff/zlob", "fff-query-parser/zlob"]
 
 [dependencies]
-mimalloc.workspace = true
-tracing.workspace = true
 git2.workspace = true
 
-fff = { package = "fff-search", path = "../fff-core" , version = "0.6.4" }
-fff-query-parser = { path = "../fff-query-parser" , version = "0.6.4" }
-serde = { version = "1.0", features = ["derive"] }
+fff = { package = "fff-search", path = "../fff-core", version = "0.10.4", default-features = false }
+fff-query-parser = { path = "../fff-query-parser", version = "0.10.4", default-features = false }
 serde_json = "1.0"

crates/fff-c/cbindgen.toml

@@ -26,3 +26,8 @@ include = [
 
 [fn]
 sort_by = "None"
+# Translate `#[deprecated]` on extern "C" fns into a real C compiler
+# attribute so callers get a warning when they use a removed/legacy entry.
+# `{}` is substituted with the Rust deprecation note as a C string literal
+# (already quoted) — do not wrap in extra quotes.
+deprecated_with_note = "__attribute__((deprecated({})))"

crates/fff-c/src/accessors.rs

@@ -1,42 +1,70 @@
-//! Stable accessor functions for `fff-c` FFI struct fields.
-//!
-//! # Why this exists
-//!
-//! `fff-c` exposes its result types as plain `#[repr(C)]` structs. External
-//! consumers (Emacs Lisp via `emacs-ffi`, Python `ctypes`, etc.) that access
-//! fields by hardcoding byte offsets break silently whenever the struct layout
-//! changes — a new field shifts every subsequent offset with no compile-time
-//! warning.
-//!
-//! These functions turn field access into a **stable named API**: callers bind
-//! to a symbol name once and are fully insulated from layout changes.
-//!
-//! # Usage from Emacs Lisp (example)
-//!
-//! ```elisp
-//! (define-ffi-function fff--grep-match-line-content
-//!   "fff_grep_match_get_line_content" :pointer [:pointer] fff--library)
-//!
-//! (ffi-get-c-string (fff--grep-match-line-content match-ptr))
-//! ```
-//!
-//! # Array iteration
-//!
-//! To walk result arrays use `fff_search_result_get_item`,
-//! `fff_grep_result_get_match`, and `fff_search_result_get_score` — these are
-//! defined in the main `lib.rs` FFI surface alongside the search functions.
+//! Stable accessor functions for `fff-c` FFI struct fields: a named API so
+//! FFI callers (Emacs Lisp, Python `ctypes`, etc.) don't hardcode struct byte
+//! offsets that break silently on layout changes. For array iteration use
+//! `fff_search_result_get_item` / `fff_grep_result_get_match` in `lib.rs`.
 
 use std::ffi::c_char;
 use std::ptr;
 
-use crate::ffi_types::{FffFileItem, FffGrepMatch, FffGrepResult, FffMatchRange, FffSearchResult};
+use crate::ffi_types::{
+    FffFileItem, FffGrepMatch, FffGrepResult, FffMatchRange, FffResult, FffSearchResult,
+};
+
+// ── FffResult ────────────────────────────────────────────────────────────────
+
+/// Returns whether the operation completed successfully. Returns `false` if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_success(result: *const FffResult) -> bool {
+    if result.is_null() {
+        return false;
+    }
+    unsafe { (*result).success }
+}
+
+/// Returns the operation error message, or null when there is no error or `result` is null.
+///
+/// Do not free the returned pointer. It remains valid until `fff_free_result` is called.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_error(result: *const FffResult) -> *const c_char {
+    if result.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*result).error }
+}
+
+/// Returns the result payload handle, or null if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_handle(result: *const FffResult) -> *mut std::ffi::c_void {
+    if result.is_null() {
+        return ptr::null_mut();
+    }
+    unsafe { (*result).handle }
+}
+
+/// Returns the result integer payload. Returns `0` if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_int_value(result: *const FffResult) -> i64 {
+    if result.is_null() {
+        return 0;
+    }
+    unsafe { (*result).int_value }
+}
 
 // ── FffFileItem ──────────────────────────────────────────────────────────────
 
-/// Returns the relative path of a file item (e.g. `"src/main.rs"`).
-///
-/// Returns null if `item` is null. The returned pointer is valid for the
-/// lifetime of the owning `FffSearchResult`; do not free it directly.
+/// Relative path of a file item (e.g. `"src/main.rs"`); null if `item` is null. Do not free.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -50,9 +78,7 @@ pub unsafe extern "C" fn fff_file_item_get_relative_path(
     unsafe { (*item).relative_path }
 }
 
-/// Returns the file-name component of a file item (e.g. `"main.rs"`).
-///
-/// Returns null if `item` is null. Do not free the returned pointer.
+/// File-name component of a file item (e.g. `"main.rs"`); null if `item` is null. Do not free.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -64,10 +90,8 @@ pub unsafe extern "C" fn fff_file_item_get_file_name(item: *const FffFileItem) -
     unsafe { (*item).file_name }
 }
 
-/// Returns the git status string for a file item (e.g. `"M "`, `"??"`)
-/// or null if git is unavailable, the file is untracked, or `item` is null.
-///
-/// Do not free the returned pointer.
+/// Git status string of a file item (e.g. `"M "`, `"??"`); null if git is unavailable,
+/// the file is untracked, or `item` is null. Do not free.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -79,7 +103,7 @@ pub unsafe extern "C" fn fff_file_item_get_git_status(item: *const FffFileItem)
     unsafe { (*item).git_status }
 }
 
-/// Returns the file size in bytes. Returns `0` if `item` is null.
+/// File size in bytes; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -91,8 +115,7 @@ pub unsafe extern "C" fn fff_file_item_get_size(item: *const FffFileItem) -> u64
     unsafe { (*item).size }
 }
 
-/// Returns the last-modified time as seconds since the UNIX epoch.
-/// Returns `0` if `item` is null.
+/// Last-modified time as seconds since the UNIX epoch; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -104,7 +127,7 @@ pub unsafe extern "C" fn fff_file_item_get_modified(item: *const FffFileItem) ->
     unsafe { (*item).modified }
 }
 
-/// Returns the combined frecency score. Returns `0` if `item` is null.
+/// Combined frecency score; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -116,7 +139,7 @@ pub unsafe extern "C" fn fff_file_item_get_total_frecency_score(item: *const Fff
     unsafe { (*item).total_frecency_score }
 }
 
-/// Returns the access-based frecency score. Returns `0` if `item` is null.
+/// Access-based frecency score; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -128,7 +151,7 @@ pub unsafe extern "C" fn fff_file_item_get_access_frecency_score(item: *const Ff
     unsafe { (*item).access_frecency_score }
 }
 
-/// Returns the modification-based frecency score. Returns `0` if `item` is null.
+/// Modification-based frecency score; `0` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -142,7 +165,7 @@ pub unsafe extern "C" fn fff_file_item_get_modification_frecency_score(
     unsafe { (*item).modification_frecency_score }
 }
 
-/// Returns `true` if the file was detected as binary. Returns `false` if `item` is null.
+/// `true` if the file was detected as binary; `false` if `item` is null.
 ///
 /// ## Safety
 /// `item` must be a valid `FffFileItem` pointer or null.
@@ -156,9 +179,7 @@ pub unsafe extern "C" fn fff_file_item_get_is_binary(item: *const FffFileItem) -
 
 // ── FffGrepMatch ─────────────────────────────────────────────────────────────
 
-/// Returns the relative path of the file containing this grep match.
-///
-/// Returns null if `m` is null. Do not free the returned pointer.
+/// Relative path of the file containing this grep match; null if `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -170,9 +191,7 @@ pub unsafe extern "C" fn fff_grep_match_get_relative_path(m: *const FffGrepMatch
     unsafe { (*m).relative_path }
 }
 
-/// Returns the file-name component of the file containing this grep match.
-///
-/// Returns null if `m` is null. Do not free the returned pointer.
+/// File-name component of the file containing this grep match; null if `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -184,10 +203,8 @@ pub unsafe extern "C" fn fff_grep_match_get_file_name(m: *const FffGrepMatch) ->
     unsafe { (*m).file_name }
 }
 
-/// Returns the git status string for the matched file (e.g. `"M "`, `"??"`)
-/// or null if git is unavailable, the file is untracked, or `m` is null.
-///
-/// Do not free the returned pointer.
+/// Git status string of the matched file (e.g. `"M "`, `"??"`); null if git is unavailable,
+/// the file is untracked, or `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -199,9 +216,7 @@ pub unsafe extern "C" fn fff_grep_match_get_git_status(m: *const FffGrepMatch) -
     unsafe { (*m).git_status }
 }
 
-/// Returns the full text content of the matched line.
-///
-/// Returns null if `m` is null. Do not free the returned pointer.
+/// Full text content of the matched line; null if `m` is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -213,8 +228,7 @@ pub unsafe extern "C" fn fff_grep_match_get_line_content(m: *const FffGrepMatch)
     unsafe { (*m).line_content }
 }
 
-/// Returns the 1-based line number of the match within its file.
-/// Returns `0` if `m` is null.
+/// 1-based line number of the match within its file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -226,8 +240,7 @@ pub unsafe extern "C" fn fff_grep_match_get_line_number(m: *const FffGrepMatch)
     unsafe { (*m).line_number }
 }
 
-/// Returns the 0-based column of the match start within its line.
-/// Returns `0` if `m` is null.
+/// 0-based column of the match start within its line; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -239,8 +252,7 @@ pub unsafe extern "C" fn fff_grep_match_get_col(m: *const FffGrepMatch) -> u32 {
     unsafe { (*m).col }
 }
 
-/// Returns the byte offset of the match start from the beginning of the file.
-/// Returns `0` if `m` is null.
+/// Byte offset of the match start from the beginning of the file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -252,7 +264,7 @@ pub unsafe extern "C" fn fff_grep_match_get_byte_offset(m: *const FffGrepMatch)
     unsafe { (*m).byte_offset }
 }
 
-/// Returns the file size in bytes for the matched file. Returns `0` if `m` is null.
+/// File size in bytes of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -264,8 +276,7 @@ pub unsafe extern "C" fn fff_grep_match_get_size(m: *const FffGrepMatch) -> u64
     unsafe { (*m).size }
 }
 
-/// Returns the combined frecency score for the matched file.
-/// Returns `0` if `m` is null.
+/// Combined frecency score of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -277,8 +288,7 @@ pub unsafe extern "C" fn fff_grep_match_get_total_frecency_score(m: *const FffGr
     unsafe { (*m).total_frecency_score }
 }
 
-/// Returns the access-based frecency score for the matched file.
-/// Returns `0` if `m` is null.
+/// Access-based frecency score of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -290,8 +300,7 @@ pub unsafe extern "C" fn fff_grep_match_get_access_frecency_score(m: *const FffG
     unsafe { (*m).access_frecency_score }
 }
 
-/// Returns the modification-based frecency score for the matched file.
-/// Returns `0` if `m` is null.
+/// Modification-based frecency score of the matched file; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -305,8 +314,7 @@ pub unsafe extern "C" fn fff_grep_match_get_modification_frecency_score(
     unsafe { (*m).modification_frecency_score }
 }
 
-/// Returns the last-modified time as seconds since the UNIX epoch for the matched file.
-/// Returns `0` if `m` is null.
+/// Last-modified time of the matched file as seconds since the UNIX epoch; `0` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -318,8 +326,7 @@ pub unsafe extern "C" fn fff_grep_match_get_modified(m: *const FffGrepMatch) ->
     unsafe { (*m).modified }
 }
 
-/// Returns the number of highlight ranges in this match. Returns `0` if `m` is null.
-///
+/// Number of highlight ranges in this match; `0` if `m` is null.
 /// Use with [`fff_grep_match_get_match_range`] to iterate the highlight spans.
 ///
 /// ## Safety
@@ -332,11 +339,8 @@ pub unsafe extern "C" fn fff_grep_match_get_match_ranges_count(m: *const FffGrep
     unsafe { (*m).match_ranges_count }
 }
 
-/// Returns a pointer to the `index`-th [`FffMatchRange`] highlight span.
-///
-/// Returns null if `m` is null, `index >= match_ranges_count`, or the
-/// ranges array is null. The returned pointer is valid until the owning
-/// `FffGrepResult` is freed; do not free it directly.
+/// Pointer to the `index`-th [`FffMatchRange`] highlight span; null if `m` is null,
+/// `index >= match_ranges_count`, or the ranges array is null. Valid until the owning `FffGrepResult` is freed; do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -355,9 +359,7 @@ pub unsafe extern "C" fn fff_grep_match_get_match_range(
     unsafe { m.match_ranges.add(index as usize) }
 }
 
-/// Returns the number of context lines captured before the match.
-/// Returns `0` if `m` is null.
-///
+/// Number of context lines captured before the match; `0` if `m` is null.
 /// Use with [`fff_grep_match_get_context_before`] to read each line.
 ///
 /// ## Safety
@@ -370,10 +372,8 @@ pub unsafe extern "C" fn fff_grep_match_get_context_before_count(m: *const FffGr
     unsafe { (*m).context_before_count }
 }
 
-/// Returns the `index`-th context line before the match.
-///
-/// Returns null if `m` is null, `index >= context_before_count`, or the
-/// context array is null. Do not free the returned pointer.
+/// The `index`-th context line before the match; null if `m` is null,
+/// `index >= context_before_count`, or the context array is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -392,9 +392,7 @@ pub unsafe extern "C" fn fff_grep_match_get_context_before(
     unsafe { *m.context_before.add(index as usize) }
 }
 
-/// Returns the number of context lines captured after the match.
-/// Returns `0` if `m` is null.
-///
+/// Number of context lines captured after the match; `0` if `m` is null.
 /// Use with [`fff_grep_match_get_context_after`] to read each line.
 ///
 /// ## Safety
@@ -407,10 +405,8 @@ pub unsafe extern "C" fn fff_grep_match_get_context_after_count(m: *const FffGre
     unsafe { (*m).context_after_count }
 }
 
-/// Returns the `index`-th context line after the match.
-///
-/// Returns null if `m` is null, `index >= context_after_count`, or the
-/// context array is null. Do not free the returned pointer.
+/// The `index`-th context line after the match; null if `m` is null,
+/// `index >= context_after_count`, or the context array is null. Do not free.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -429,11 +425,8 @@ pub unsafe extern "C" fn fff_grep_match_get_context_after(
     unsafe { *m.context_after.add(index as usize) }
 }
 
-/// Returns the fuzzy match score. Returns `0` if `m` is null or no fuzzy
-/// score is present.
-///
-/// Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is
-/// ambiguous without that flag.
+/// Fuzzy match score; `0` if `m` is null or no fuzzy score is present.
+/// Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is ambiguous without that flag.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -445,8 +438,7 @@ pub unsafe extern "C" fn fff_grep_match_get_fuzzy_score(m: *const FffGrepMatch)
     unsafe { (*m).fuzzy_score }
 }
 
-/// Returns `true` if this match carries a valid fuzzy score.
-/// Returns `false` if `m` is null.
+/// `true` if this match carries a valid fuzzy score; `false` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -458,8 +450,7 @@ pub unsafe extern "C" fn fff_grep_match_get_has_fuzzy_score(m: *const FffGrepMat
     unsafe { (*m).has_fuzzy_score }
 }
 
-/// Returns `true` if the match was identified as a symbol definition.
-/// Returns `false` if `m` is null.
+/// `true` if the match was identified as a symbol definition; `false` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -471,8 +462,7 @@ pub unsafe extern "C" fn fff_grep_match_get_is_definition(m: *const FffGrepMatch
     unsafe { (*m).is_definition }
 }
 
-/// Returns `true` if the matched file was detected as binary.
-/// Returns `false` if `m` is null.
+/// `true` if the matched file was detected as binary; `false` if `m` is null.
 ///
 /// ## Safety
 /// `m` must be a valid `FffGrepMatch` pointer or null.
@@ -486,7 +476,7 @@ pub unsafe extern "C" fn fff_grep_match_get_is_binary(m: *const FffGrepMatch) ->
 
 // ── FffSearchResult ──────────────────────────────────────────────────────────
 
-/// Returns the number of items in the result. Returns `0` if `r` is null.
+/// Number of items in the result; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffSearchResult` pointer or null.
@@ -498,8 +488,7 @@ pub unsafe extern "C" fn fff_search_result_get_count(r: *const FffSearchResult)
     unsafe { (*r).count }
 }
 
-/// Returns the total number of files that matched before the result was
-/// truncated to the page size. Returns `0` if `r` is null.
+/// Total number of files that matched before truncation to the page size; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffSearchResult` pointer or null.
@@ -511,8 +500,7 @@ pub unsafe extern "C" fn fff_search_result_get_total_matched(r: *const FffSearch
     unsafe { (*r).total_matched }
 }
 
-/// Returns the total number of indexed files considered during search.
-/// Returns `0` if `r` is null.
+/// Total number of indexed files considered during search; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffSearchResult` pointer or null.
@@ -526,7 +514,7 @@ pub unsafe extern "C" fn fff_search_result_get_total_files(r: *const FffSearchRe
 
 // ── FffGrepResult ─────────────────────────────────────────────────────────────
 
-/// Returns the number of matches in the result. Returns `0` if `r` is null.
+/// Number of matches in the result; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -538,8 +526,7 @@ pub unsafe extern "C" fn fff_grep_result_get_count(r: *const FffGrepResult) -> u
     unsafe { (*r).count }
 }
 
-/// Returns the total number of matches found across all pages.
-/// Returns `0` if `r` is null.
+/// Total number of matches found across all pages; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -551,8 +538,7 @@ pub unsafe extern "C" fn fff_grep_result_get_total_matched(r: *const FffGrepResu
     unsafe { (*r).total_matched }
 }
 
-/// Returns the number of files actually opened and searched in this call.
-/// Returns `0` if `r` is null.
+/// Number of files actually opened and searched in this call; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -564,8 +550,7 @@ pub unsafe extern "C" fn fff_grep_result_get_total_files_searched(r: *const FffG
     unsafe { (*r).total_files_searched }
 }
 
-/// Returns the total number of indexed files before any filtering.
-/// Returns `0` if `r` is null.
+/// Total number of indexed files before any filtering; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -577,8 +562,7 @@ pub unsafe extern "C" fn fff_grep_result_get_total_files(r: *const FffGrepResult
     unsafe { (*r).total_files }
 }
 
-/// Returns the number of files eligible for search after path/type filtering.
-/// Returns `0` if `r` is null.
+/// Number of files eligible for search after path/type filtering; `0` if `r` is null.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -590,9 +574,8 @@ pub unsafe extern "C" fn fff_grep_result_get_filtered_file_count(r: *const FffGr
     unsafe { (*r).filtered_file_count }
 }
 
-/// Returns the file offset for the next page, or `0` if all files have been
-/// searched or `r` is null. Pass this value as `file_offset` to a subsequent
-/// `fff_live_grep` or `fff_multi_grep` call to continue pagination.
+/// File offset for the next page; `0` if all files have been searched or `r` is null.
+/// Pass as `file_offset` to a subsequent `fff_live_grep`/`fff_multi_grep` call to continue pagination.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -604,10 +587,8 @@ pub unsafe extern "C" fn fff_grep_result_get_next_file_offset(r: *const FffGrepR
     unsafe { (*r).next_file_offset }
 }
 
-/// Returns the regex compilation error string if the engine fell back to
-/// literal matching, or null if there was no error or `r` is null.
-///
-/// Do not free the returned pointer.
+/// Regex compilation error string if the engine fell back to literal matching;
+/// null if there was no error or `r` is null. Do not free.
 ///
 /// ## Safety
 /// `r` must be a valid `FffGrepResult` pointer or null.
@@ -801,8 +782,41 @@ mod tests {
         }
     }
 
+    #[test]
+    fn null_result_returns_zero_or_null() {
+        let null: *const FffResult = ptr::null();
+        unsafe {
+            assert!(!fff_result_get_success(null));
+            assert!(fff_result_get_error(null).is_null());
+            assert!(fff_result_get_handle(null).is_null());
+            assert_eq!(fff_result_get_int_value(null), 0);
+        }
+    }
+
     // ── data correctness tests ────────────────────────────────────────────────
 
+    #[test]
+    fn result_getters_return_correct_values() {
+        let error = CString::new("failed").unwrap();
+        let handle = 0x1234usize as *mut std::ffi::c_void;
+        let result = FffResult {
+            success: false,
+            error: error.as_ptr() as *mut std::ffi::c_char,
+            handle,
+            int_value: -7,
+        };
+        let p = &result as *const FffResult;
+        unsafe {
+            assert!(!fff_result_get_success(p));
+            assert_eq!(
+                std::ffi::CStr::from_ptr(fff_result_get_error(p)),
+                error.as_c_str()
+            );
+            assert_eq!(fff_result_get_handle(p), handle);
+            assert_eq!(fff_result_get_int_value(p), -7);
+        }
+    }
+
     #[test]
     fn file_item_getters_return_correct_values() {
         let mut item = make_file_item("src/main.rs", "main.rs");

crates/fff-c/src/ffi_types.rs

@@ -1,8 +1,5 @@
-//! FFI-compatible type definitions
-//!
-//! All result types use `#[repr(C)]` structs for direct memory access from any
-//! language with C FFI support. No JSON serialization is used for search or grep
-//! results — callers read struct fields directly.
+//! FFI-compatible type definitions: all result types are `#[repr(C)]` structs
+//! read directly from any language with C FFI — no JSON serialization.
 
 use std::ffi::{CString, c_char, c_void};
 use std::ptr;
@@ -14,6 +11,79 @@ use fff::{
     MixedSearchResult, Score, SearchResult,
 };
 
+/// Current used version of [`FffCreateOptions`].
+pub const FFF_CREATE_OPTIONS_VERSION: u32 = 2;
+
+/// Options for `fff_create_instance_with`.
+///
+/// Versioned struct: the layout is stable across releases, new fields are
+/// only appended.
+#[repr(C)]
+pub struct FffCreateOptions {
+    /// Set to [`FFF_CREATE_OPTIONS_VERSION`] when allocating; tells the
+    /// library which trailing fields are populated.
+    pub version: u32,
+    /// Directory to index (required, non-NULL).
+    pub base_path: *const c_char,
+    /// Frecency LMDB database path. NULL/empty to skip frecency tracking.
+    pub frecency_db_path: *const c_char,
+    /// Query history LMDB database path. NULL/empty to skip query tracking.
+    pub history_db_path: *const c_char,
+    /// Pre-populate mmap caches for top-frecency files after the initial scan.
+    pub enable_mmap_cache: bool,
+    /// Build content index after the initial scan for faster grep.
+    pub enable_content_indexing: bool,
+    /// Start a background file-system watcher for live updates.
+    pub watch: bool,
+    /// Enable AI-agent optimizations.
+    pub ai_mode: bool,
+    /// Tracing log file path. NULL/empty to skip log init.
+    pub log_file_path: *const c_char,
+    /// Log level: `"trace" | "debug" | "info" | "warn" | "error"`.
+    /// NULL/empty defaults to `"info"`. Ignored when `log_file_path` is unset.
+    pub log_level: *const c_char,
+    /// Content cache file-count cap. 0 = auto.
+    pub cache_budget_max_files: u64,
+    /// Content cache byte cap. 0 = auto.
+    pub cache_budget_max_bytes: u64,
+    /// Per-file byte cap inside the content cache. 0 = auto.
+    pub cache_budget_max_file_size: u64,
+    /// Allow indexing the filesystem root (`/`). Off by default: root is rarely
+    /// intended and floods the watcher with churn.
+    pub enable_fs_root_scanning: bool,
+    /// Allow indexing the user's home directory. Same trade-off as `enable_fs_root_scanning`.
+    pub enable_home_dir_scanning: bool,
+    // ----- v2 fields -----
+    /// Follow symlinks during scan and watcher walks. Off by default: without
+    /// external loop protection cyclic symlinks can wedge the watcher.
+    pub follow_symlinks: bool,
+    // ----- new version 3+ fields go here, ALWAYS appended -----
+}
+
+impl FffCreateOptions {
+    /// Default values for a v1 options struct.
+    pub fn defaults() -> Self {
+        Self {
+            version: FFF_CREATE_OPTIONS_VERSION,
+            base_path: ptr::null(),
+            frecency_db_path: ptr::null(),
+            history_db_path: ptr::null(),
+            enable_mmap_cache: true,
+            enable_content_indexing: true,
+            watch: true,
+            ai_mode: false,
+            log_file_path: ptr::null(),
+            log_level: ptr::null(),
+            cache_budget_max_files: 0,
+            cache_budget_max_bytes: 0,
+            cache_budget_max_file_size: 0,
+            enable_fs_root_scanning: false,
+            enable_home_dir_scanning: false,
+            follow_symlinks: false,
+        }
+    }
+}
+
 /// Allocate a heap CString from a `&str`, returning a raw pointer.
 fn cstring_new(s: &str) -> *mut c_char {
     CString::new(s).unwrap_or_default().into_raw()
@@ -58,10 +128,8 @@ unsafe fn free_cstring_array(arr: *mut *mut c_char, count: u32) {
     }
 }
 
-/// A file item returned by `fff_search`.
-///
-/// All string fields are heap-allocated and owned by the parent `FffSearchResult`.
-/// Free the entire result with `fff_free_search_result`.
+/// A file item returned by `fff_search`. Strings are owned by the parent
+/// `FffSearchResult`; free everything with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffFileItem {
     pub relative_path: *mut c_char,
@@ -155,13 +223,9 @@ impl FffScore {
     }
 }
 
-/// Location parsed from a query string (e.g. `"file.ts:42:10"`).
-///
-/// `tag` encodes the variant:
-///   0 = no location,
-///   1 = line only (`line` is set),
-///   2 = position (`line` + `col`),
-///   3 = range (`line`/`col` = start, `end_line`/`end_col` = end).
+/// Location parsed from a query string (e.g. `"file.ts:42:10"`). `tag`:
+/// 0 = none, 1 = line, 2 = position (`line` + `col`),
+/// 3 = range (`line`/`col` = start, `end_line`/`end_col` = end).
 #[repr(C)]
 pub struct FffLocation {
     pub tag: u8,
@@ -206,14 +270,12 @@ impl From<Option<&Location>> for FffLocation {
     }
 }
 
-/// Search result returned by `fff_search`.
-///
-/// The caller must free this with `fff_free_search_result`.
+/// Search result returned by `fff_search`; free with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffSearchResult {
-    /// Pointer to a heap-allocated array of `FffFileItem` (length = `count`).
+    /// Heap array of `FffFileItem` (length = `count`).
     pub items: *mut FffFileItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -261,10 +323,8 @@ pub struct FffMatchRange {
     pub end: u32,
 }
 
-/// A single grep match with file and line information.
-///
-/// All string fields and arrays are heap-allocated. Free the parent
-/// `FffGrepResult` with `fff_free_grep_result` to release everything.
+/// A single grep match with file and line information. Strings and arrays are
+/// owned by the parent `FffGrepResult`; free everything with `fff_free_grep_result`.
 #[repr(C)]
 pub struct FffGrepMatch {
     // -- pointers (8 bytes each) --
@@ -366,12 +426,11 @@ impl FffGrepMatch {
     }
 }
 
-/// Grep result returned by `fff_live_grep` and `fff_multi_grep`.
-///
-/// The caller must free this with `fff_free_grep_result`.
+/// Grep result returned by `fff_live_grep` and `fff_multi_grep`;
+/// free with `fff_free_grep_result`.
 #[repr(C)]
 pub struct FffGrepResult {
-    /// Pointer to a heap-allocated array of `FffGrepMatch` (length = `count`).
+    /// Heap array of `FffGrepMatch` (length = `count`).
     pub items: *mut FffGrepMatch,
     /// Number of matches in the `items` array.
     pub count: u32,
@@ -420,7 +479,9 @@ impl FffGrepResult {
 
 /// Result envelope returned by all `fff_*` functions.
 ///
-/// Heap-allocated — the caller must free it with `fff_free_result`.
+/// Heap-allocated. The caller must free it with `fff_free_result`. Calling `fff_free_result`
+/// **does not** deallocate the underlying `handle` pointer. It needs to be cleaned separately.
+/// see (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`, `fff_free_string`, etc.).
 ///
 /// Depending on the function, the payload is delivered through different fields:
 ///
@@ -440,18 +501,13 @@ impl FffGrepResult {
 /// | `fff_restart_index`        | (none)        | success flag only             |
 ///
 /// On failure, `success` is false and `error` contains the message.
-///
-/// **Important:** `fff_free_result` frees `error` but does **not** free `handle`.
-/// The caller must free the handle with the appropriate function
-/// (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`,
-///  `fff_free_string`, etc.).
 #[repr(C)]
 pub struct FffResult {
     /// Whether the operation succeeded.
     pub success: bool,
     /// Error message on failure. Null on success.
     pub error: *mut c_char,
-    /// Opaque pointer payload (instance handle, typed result struct, or string). May be null.
+    /// Opaque pointer payload. May be null.
     pub handle: *mut c_void,
     /// Integer payload for simple return values (bool as 0/1, counts, etc.).
     pub int_value: i64,
@@ -511,10 +567,8 @@ impl FffResult {
     }
 }
 
-/// A directory item returned by `fff_search_directories`.
-///
-/// All string fields are heap-allocated and owned by the parent `FffDirSearchResult`.
-/// Free the entire result with `fff_free_dir_search_result`.
+/// A directory item returned by `fff_search_directories`. Strings are owned by
+/// the parent `FffDirSearchResult`; free everything with `fff_free_dir_search_result`.
 #[repr(C)]
 pub struct FffDirItem {
     pub relative_path: *mut c_char,
@@ -545,14 +599,13 @@ impl FffDirItem {
     }
 }
 
-/// Directory search result returned by `fff_search_directories`.
-///
-/// The caller must free this with `fff_free_dir_search_result`.
+/// Directory search result returned by `fff_search_directories`;
+/// free with `fff_free_dir_search_result`.
 #[repr(C)]
 pub struct FffDirSearchResult {
-    /// Pointer to a heap-allocated array of `FffDirItem` (length = `count`).
+    /// Heap array of `FffDirItem` (length = `count`).
     pub items: *mut FffDirItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -587,9 +640,8 @@ impl FffDirSearchResult {
 }
 
 /// A single item in a mixed (files + directories) search result.
-///
-/// `item_type`: 0 = file, 1 = directory.
-/// All string fields are heap-allocated and owned by the parent `FffMixedSearchResult`.
+/// `item_type`: 0 = file, 1 = directory. Strings are owned by the parent
+/// `FffMixedSearchResult`.
 #[repr(C)]
 pub struct FffMixedItem {
     /// 0 = file, 1 = directory.
@@ -600,8 +652,7 @@ pub struct FffMixedItem {
     pub git_status: *mut c_char,
     pub size: u64,
     pub modified: u64,
-    /// The access frecency score for files, or max access frecency among all the immediate
-    /// children for directories.
+    /// Access frecency for files; max among immediate children for directories.
     pub access_frecency_score: i64,
     /// Always 0 for directories
     pub modification_frecency_score: i64,
@@ -658,14 +709,13 @@ impl FffMixedItem {
     }
 }
 
-/// Mixed search result returned by `fff_search_mixed`.
-///
-/// The caller must free this with `fff_free_mixed_search_result`.
+/// Mixed search result returned by `fff_search_mixed`
+/// free with `fff_free_mixed_search_result`.
 #[repr(C)]
 pub struct FffMixedSearchResult {
-    /// Pointer to a heap-allocated array of `FffMixedItem` (length = `count`).
+    /// Heap array of `FffMixedItem` (length = `count`).
     pub items: *mut FffMixedItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -725,3 +775,35 @@ impl From<fff::file_picker::ScanProgress> for FffScanProgress {
         }
     }
 }
+
+#[cfg(test)]
+mod options_layout_tests {
+    use super::FffCreateOptions;
+    use std::mem::{align_of, offset_of, size_of};
+
+    // THIS TEST HAVE TO BE NEVER UPDATED ONLY ADDED NEW FIELDS
+    // this is needed to ensure ABI backward compatibility
+    #[test]
+    #[cfg(target_pointer_width = "64")]
+    fn fff_create_options_layout_is_stable_64bit() {
+        assert_eq!(size_of::<FffCreateOptions>(), 88);
+        assert_eq!(align_of::<FffCreateOptions>(), 8);
+
+        assert_eq!(offset_of!(FffCreateOptions, version), 0);
+        assert_eq!(offset_of!(FffCreateOptions, base_path), 8);
+        assert_eq!(offset_of!(FffCreateOptions, frecency_db_path), 16);
+        assert_eq!(offset_of!(FffCreateOptions, history_db_path), 24);
+        assert_eq!(offset_of!(FffCreateOptions, enable_mmap_cache), 32);
+        assert_eq!(offset_of!(FffCreateOptions, enable_content_indexing), 33);
+        assert_eq!(offset_of!(FffCreateOptions, watch), 34);
+        assert_eq!(offset_of!(FffCreateOptions, ai_mode), 35);
+        assert_eq!(offset_of!(FffCreateOptions, log_file_path), 40);
+        assert_eq!(offset_of!(FffCreateOptions, log_level), 48);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_files), 56);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_bytes), 64);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_file_size), 72);
+        assert_eq!(offset_of!(FffCreateOptions, enable_fs_root_scanning), 80);
+        assert_eq!(offset_of!(FffCreateOptions, enable_home_dir_scanning), 81);
+        assert_eq!(offset_of!(FffCreateOptions, follow_symlinks), 82);
+    }
+}

crates/fff-c/src/lib.rs

@@ -1,26 +1,14 @@
-//! C FFI bindings for fff-core
+//! C FFI bindings for fff-core, usable from any language with C FFI
+//! (Bun, Node.js, Python, Ruby, etc.).
 //!
-//! This crate provides C-compatible FFI exports that can be used from any language
-//! with C FFI support (Bun, Node.js, Python, Ruby, etc.).
+//! All state is owned by an opaque instance handle: create with
+//! `fff_create_instance*`, pass to every call, free with `fff_destroy`.
+//! Multiple instances can coexist in one process.
 //!
-//! # Instance-based API
-//!
-//! All state is owned by an opaque `FffInstance` fff_handle. Callers create an instance
-//! with `fff_create_instance`, pass the fff_handle to every subsequent call, and free it with
-//! `fff_destroy`. Multiple independent instances can coexist in the same process.
-//!
-//! # Memory management
-//!
-//! * Every `fff_*` function that returns `*mut FffResult` requires the caller to
-//!   free the result with `fff_free_result`.
-//! * The instance itself must be freed with `fff_destroy`.
-//!
-//! # Parameter conventions
-//!
-//! * Optional `*const c_char` parameters: pass NULL or an empty string to omit.
-//! * Numeric parameters: 0 means "use default" unless documented otherwise.
-//! * Grep mode (`u8`): 0 = plain text, 1 = regex, 2 = fuzzy.
-//! * Multi-grep patterns are passed as a single newline-separated (`\n`) string.
+//! Conventions: every returned `*mut FffResult` is freed with
+//! `fff_free_result`; optional string params take NULL/empty; numeric 0 means
+//! "use default" unless documented otherwise; grep mode `u8` is 0 = plain
+//! text, 1 = regex, 2 = fuzzy; multi-grep patterns are `\n`-separated.
 
 use std::ffi::{CStr, CString, c_char, c_void};
 use std::path::PathBuf;
@@ -30,31 +18,30 @@ use fff::shared::SharedQueryTracker;
 
 mod accessors;
 mod ffi_types;
+mod watch;
 
 use fff::file_picker::FilePicker;
 use fff::frecency::FrecencyTracker;
 use fff::query_tracker::QueryTracker;
 use fff::{DbHealthChecker, FFFMode, FuzzySearchOptions, PaginationArgs, QueryParser};
-use fff::{SharedFrecency, SharedPicker};
+use fff::{SharedFilePicker, SharedFrecency};
 use ffi_types::{
-    FffDirItem, FffDirSearchResult, FffFileItem, FffGrepMatch, FffGrepResult, FffMixedItem,
-    FffMixedSearchResult, FffResult, FffScanProgress, FffScore, FffSearchResult,
+    FFF_CREATE_OPTIONS_VERSION, FffCreateOptions, FffDirItem, FffDirSearchResult, FffFileItem,
+    FffGrepMatch, FffGrepResult, FffMixedItem, FffMixedSearchResult, FffResult, FffScanProgress,
+    FffScore, FffSearchResult,
 };
 
-/// Opaque fff_handle holding all per-instance state.
-///
-/// The caller receives this as `*mut c_void` and must pass it to every FFI call.
-/// The fff_handle is freed by `fff_destroy`.
+/// Opaque handle holding all per-instance state; freed by `fff_destroy`.
 struct FffInstance {
-    picker: SharedPicker,
+    picker: SharedFilePicker,
     frecency: SharedFrecency,
     query_tracker: SharedQueryTracker,
+    // we keep a single callback type
+    watch_callback: std::sync::Arc<watch::WatchCallbackSlot>,
 }
 
-/// Helper to convert C string to Rust &str.
-///
-/// Returns `None` if the pointer is null or the string is not valid UTF-8.
-unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> {
+/// Convert a C string to `&str`; `None` if null or invalid UTF-8.
+pub(crate) unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> {
     if s.is_null() {
         None
     } else {
@@ -62,17 +49,15 @@ unsafe fn cstr_to_str<'a>(s: *const c_char) -> Option<&'a str> {
     }
 }
 
-/// Helper to convert an optional C string parameter.
-///
-/// Returns `None` if the pointer is null, empty, or not valid UTF-8.
+/// Optional C string param: `None` if null, empty, or invalid UTF-8.
 unsafe fn optional_cstr<'a>(s: *const c_char) -> Option<&'a str> {
     unsafe { cstr_to_str(s) }.filter(|s| !s.is_empty())
 }
 
-/// Recover a `&FffInstance` from the opaque pointer.
-///
-/// Returns an error `FffResult` if the pointer is null.
-unsafe fn instance_ref<'a>(fff_handle: *mut c_void) -> Result<&'a FffInstance, *mut FffResult> {
+/// Recover a `&FffInstance` from the opaque pointer; error `FffResult` if null.
+pub(crate) unsafe fn instance_ref<'a>(
+    fff_handle: *mut c_void,
+) -> Result<&'a FffInstance, *mut FffResult> {
     if fff_handle.is_null() {
         Err(FffResult::err(
             "Instance handle is null. Create one with fff_create_instance first.",
@@ -104,80 +89,56 @@ fn default_i32(val: i32, default: i32) -> i32 {
     if val == 0 { default } else { val }
 }
 
-/// Create a new file finder instance (legacy signature).
+/// Create a new file finder instance (legacy 8-arg positional signature).
 ///
-/// @deprecated prefer `fff_create_instance2`, which also exposes log file and
-/// cache-budget configuration. This function delegates to `fff_create_instance2`
-/// with NULL log paths and auto cache budget, so behaviour is unchanged.
+/// @deprecated Use [`fff_create_instance_with`] (or [`fff_create_instance_with_value`]
+/// for FFI bindings). The `use_unsafe_no_lock` parameter is ignored.
 ///
 /// ## Safety
-/// See `fff_create_instance2`.
+/// See `fff_create_instance_with`.
+#[deprecated(
+    since = "0.8.5",
+    note = "Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks."
+)]
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn fff_create_instance(
     base_path: *const c_char,
     frecency_db_path: *const c_char,
     history_db_path: *const c_char,
-    use_unsafe_no_lock: bool,
+    _use_unsafe_no_lock: bool,
     enable_mmap_cache: bool,
     enable_content_indexing: bool,
     watch: bool,
     ai_mode: bool,
 ) -> *mut FffResult {
-    unsafe {
-        fff_create_instance2(
-            base_path,
-            frecency_db_path,
-            history_db_path,
-            use_unsafe_no_lock,
-            enable_mmap_cache,
-            enable_content_indexing,
-            watch,
-            ai_mode,
-            std::ptr::null(),
-            std::ptr::null(),
-            0,
-            0,
-            0,
-        )
-    }
+    let mut opts = FffCreateOptions::defaults();
+    opts.base_path = base_path;
+    opts.frecency_db_path = frecency_db_path;
+    opts.history_db_path = history_db_path;
+    opts.enable_mmap_cache = enable_mmap_cache;
+    opts.enable_content_indexing = enable_content_indexing;
+    opts.watch = watch;
+    opts.ai_mode = ai_mode;
+    unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) }
 }
 
-/// Create a new file finder instance (v2, with full options).
+/// Create a new file finder instance (legacy 13-arg positional signature).
 ///
-/// Returns an opaque pointer that must be passed to all other `fff_*` calls
-/// and eventually freed with `fff_destroy`.
-///
-/// # Parameters
-///
-/// * `base_path`                   – directory to index (required)
-/// * `frecency_db_path`            – frecency LMDB database path (NULL/empty to skip)
-/// * `history_db_path`             – query history LMDB database path (NULL/empty to skip)
-/// * `use_unsafe_no_lock`          – use MDB_NOLOCK for LMDB (useful in single-process setups)
-/// * `enable_mmap_cache`           – pre-populate mmap caches after the initial scan
-/// * `enable_content_indexing`     – build content index after the initial scan
-/// * `watch`                       – start a background file-system watcher for live updates
-/// * `ai_mode`                     – enable AI-agent optimizations
-/// * `log_file_path`               – tracing log file path (NULL/empty to skip).
-///   Only the first successful call in a process installs the subscriber;
-///   subsequent calls are no-ops at the log layer.
-/// * `log_level`                   – `"trace"`, `"debug"`, `"info"`, `"warn"`, `"error"`
-///   (NULL/empty defaults to `"info"`). Ignored when `log_file_path` is not set.
-/// * `cache_budget_max_files`      – content cache file-count cap (0 = auto)
-/// * `cache_budget_max_bytes`      – content cache byte cap (0 = auto)
-/// * `cache_budget_max_file_size`  – per-file byte cap (0 = auto)
-///
-/// When all three `cache_budget_*` values are 0 the budget is auto-computed
-/// from repo size after the initial scan. Otherwise an explicit budget is
-/// used: any field left at 0 falls back to its `unlimited()` default.
+/// @deprecated Use [`fff_create_instance_with`] (or [`fff_create_instance_with_value`]
+/// for FFI bindings). The `use_unsafe_no_lock` parameter is ignored.
 ///
 /// ## Safety
-/// String parameters must be valid null-terminated UTF-8 or NULL.
+/// See `fff_create_instance_with`.
+#[deprecated(
+    since = "0.8.5",
+    note = "Use fff_create_instance_with (by pointer) or fff_create_instance_with_value (by value) with FffCreateOptions instead. The struct evolves without ABI breaks."
+)]
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn fff_create_instance2(
     base_path: *const c_char,
     frecency_db_path: *const c_char,
     history_db_path: *const c_char,
-    use_unsafe_no_lock: bool,
+    _use_unsafe_no_lock: bool,
     enable_mmap_cache: bool,
     enable_content_indexing: bool,
     watch: bool,
@@ -188,50 +149,89 @@ pub unsafe extern "C" fn fff_create_instance2(
     cache_budget_max_bytes: u64,
     cache_budget_max_file_size: u64,
 ) -> *mut FffResult {
-    let base_path_str = match unsafe { cstr_to_str(base_path) } {
+    let mut opts = FffCreateOptions::defaults();
+    opts.base_path = base_path;
+    opts.frecency_db_path = frecency_db_path;
+    opts.history_db_path = history_db_path;
+    opts.enable_mmap_cache = enable_mmap_cache;
+    opts.enable_content_indexing = enable_content_indexing;
+    opts.watch = watch;
+    opts.ai_mode = ai_mode;
+    opts.log_file_path = log_file_path;
+    opts.log_level = log_level;
+    opts.cache_budget_max_files = cache_budget_max_files;
+    opts.cache_budget_max_bytes = cache_budget_max_bytes;
+    opts.cache_budget_max_file_size = cache_budget_max_file_size;
+    unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) }
+}
+
+/// Create a new file finder instance from a versioned [`FffCreateOptions`] struct.
+///
+/// Populate the struct, set `version` to [`FFF_CREATE_OPTIONS_VERSION`], pass by
+/// pointer. New fields are only appended; older `version` values keep working.
+/// FFI bindings needing struct-by-value should use [`fff_create_instance_with_value`].
+///
+/// `opts.base_path` is required (non-NULL, non-empty). Zero `cache_budget_*`
+/// values are auto-computed from repo size after the initial scan.
+///
+/// ## Safety
+/// * `opts` must be a valid pointer to an `FffCreateOptions` whose `version`
+///   is in the range `1..=FFF_CREATE_OPTIONS_VERSION`.
+/// * All string pointers inside `opts` must be valid null-terminated UTF-8
+///   or NULL.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_create_instance_with(opts: *const FffCreateOptions) -> *mut FffResult {
+    if opts.is_null() {
+        return FffResult::err("opts is null");
+    }
+    let opts = unsafe { &*opts };
+    if opts.version == 0 || opts.version > FFF_CREATE_OPTIONS_VERSION {
+        return FffResult::err(&format!(
+            "Unsupported FffCreateOptions version {} (library understands up to {})",
+            opts.version, FFF_CREATE_OPTIONS_VERSION
+        ));
+    }
+
+    let base_path_str = match unsafe { cstr_to_str(opts.base_path) } {
         Some(s) if !s.is_empty() => s.to_string(),
-        _ => return FffResult::err("base_path is null or empty"),
+        _ => return FffResult::err("opts.base_path is null or empty"),
     };
 
-    if let Some(log_path) = unsafe { optional_cstr(log_file_path) } {
-        let level = unsafe { optional_cstr(log_level) };
-        if let Err(e) = fff::log::init_tracing(log_path, level) {
+    if let Some(log_path) = unsafe { optional_cstr(opts.log_file_path) } {
+        let level = unsafe { optional_cstr(opts.log_level) };
+        if let Err(e) = fff::log::init_tracing(log_path, level, None) {
             return FffResult::err(&format!("Failed to init tracing: {}", e));
         }
     }
 
-    let frecency_path = unsafe { optional_cstr(frecency_db_path) }.map(|s| s.to_string());
-    let history_path = unsafe { optional_cstr(history_db_path) }.map(|s| s.to_string());
+    let frecency_path = unsafe { optional_cstr(opts.frecency_db_path) }.map(|s| s.to_string());
+    let history_path = unsafe { optional_cstr(opts.history_db_path) }.map(|s| s.to_string());
 
-    // Create shared state that background threads will write into.
-    let shared_picker = SharedPicker::default();
+    let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
     let query_tracker = SharedQueryTracker::default();
 
-    // Initialize frecency tracker if path is provided
     if let Some(ref frecency_path) = frecency_path {
         if let Some(parent) = PathBuf::from(frecency_path).parent() {
             let _ = std::fs::create_dir_all(parent);
         }
 
-        match FrecencyTracker::new(frecency_path, use_unsafe_no_lock) {
+        match FrecencyTracker::open(frecency_path) {
             Ok(tracker) => {
                 if let Err(e) = shared_frecency.init(tracker) {
                     return FffResult::err(&format!("Failed to acquire frecency lock: {}", e));
                 }
-                let _ = shared_frecency.spawn_gc(frecency_path.clone(), use_unsafe_no_lock);
             }
             Err(e) => return FffResult::err(&format!("Failed to init frecency db: {}", e)),
         }
     }
 
-    // Initialize query tracker if path is provided
     if let Some(ref history_path) = history_path {
         if let Some(parent) = PathBuf::from(history_path).parent() {
             let _ = std::fs::create_dir_all(parent);
         }
 
-        match QueryTracker::new(history_path, use_unsafe_no_lock) {
+        match QueryTracker::open(history_path) {
             Ok(tracker) => {
                 if let Err(e) = query_tracker.init(tracker) {
                     return FffResult::err(&format!("Failed to acquire query tracker lock: {}", e));
@@ -241,29 +241,31 @@ pub unsafe extern "C" fn fff_create_instance2(
         }
     }
 
-    let mode = if ai_mode {
+    let mode = if opts.ai_mode {
         FFFMode::Ai
     } else {
         FFFMode::Neovim
     };
 
     let cache_budget = fff::ContentCacheBudget::from_overrides(
-        cache_budget_max_files as usize,
-        cache_budget_max_bytes,
-        cache_budget_max_file_size,
+        opts.cache_budget_max_files as usize,
+        opts.cache_budget_max_bytes,
+        opts.cache_budget_max_file_size,
     );
 
-    // Initialize file picker (writes directly into shared_picker)
     if let Err(e) = FilePicker::new_with_shared_state(
         shared_picker.clone(),
         shared_frecency.clone(),
         fff::FilePickerOptions {
             base_path: base_path_str,
-            enable_mmap_cache,
-            enable_content_indexing,
-            watch,
+            enable_mmap_cache: opts.enable_mmap_cache,
+            enable_content_indexing: opts.enable_content_indexing,
+            watch: opts.watch,
             mode,
             cache_budget,
+            follow_symlinks: opts.version >= 2 && opts.follow_symlinks,
+            enable_fs_root_scanning: opts.enable_fs_root_scanning,
+            enable_home_dir_scanning: opts.enable_home_dir_scanning,
         },
     ) {
         return FffResult::err(&format!("Failed to init file picker: {}", e));
@@ -273,12 +275,24 @@ pub unsafe extern "C" fn fff_create_instance2(
         picker: shared_picker,
         frecency: shared_frecency,
         query_tracker,
+        watch_callback: std::sync::Arc::new(watch::WatchCallbackSlot::default()),
     });
 
     let fff_handle = Box::into_raw(instance) as *mut c_void;
     FffResult::ok_handle(fff_handle)
 }
 
+/// [`fff_create_instance_with`] adapter taking [`FffCreateOptions`] **by value**,
+/// for FFI libraries that pass native structs by value (e.g. Node's `ffi-rs`).
+///
+/// ## Safety
+/// All `*const c_char` fields inside `opts` must be valid null-terminated
+/// UTF-8 or NULL. The struct itself is consumed by value.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_create_instance_with_value(opts: FffCreateOptions) -> *mut FffResult {
+    unsafe { fff_create_instance_with(&opts as *const FffCreateOptions) }
+}
+
 /// Destroy a file finder instance and free all its resources.
 ///
 /// ## Safety
@@ -291,10 +305,14 @@ pub unsafe extern "C" fn fff_destroy(fff_handle: *mut c_void) {
 
     let instance = unsafe { Box::from_raw(fff_handle as *mut FffInstance) };
 
+    // The C callback and user_data may be freed as soon as this returns.
+    instance.picker.shutdown_watches_and_wait();
+    instance.watch_callback.clear();
+
     if let Ok(mut guard) = instance.picker.write()
-        && let Some(mut picker) = guard.take()
+        && let Some(picker) = guard.take()
     {
-        picker.stop_background_monitor();
+        drop(picker);
     }
 
     if let Ok(mut guard) = instance.frecency.write() {
@@ -307,16 +325,9 @@ pub unsafe extern "C" fn fff_destroy(fff_handle: *mut c_void) {
 
 /// Perform fuzzy search on indexed files.
 ///
-/// # Parameters
-///
-/// * `fff_handle`              – instance from `fff_create_instance`
-/// * `query`                   – search query string
-/// * `current_file`            – path of the currently open file for deprioritization (NULL/empty to skip)
-/// * `max_threads`             – maximum worker threads (0 = auto-detect)
-/// * `page_index`              – pagination offset (0 = first page)
-/// * `page_size`               – results per page (0 = default 100)
-/// * `combo_boost_multiplier`  – score multiplier for combo matches (0 = default 100)
-/// * `min_combo_count`         – minimum combo count before boost applies (0 = default 3)
+/// `current_file` deprioritizes the currently open file (NULL/empty to skip).
+/// Zero picks the default: `max_threads` auto, `page_size` 100,
+/// `combo_boost_multiplier` 100, `min_combo_count` 3.
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -389,16 +400,72 @@ pub unsafe extern "C" fn fff_search(
     FffResult::ok_handle(search_result as *mut c_void)
 }
 
+/// Glob-only search: filter indexed files by a single glob pattern (passed
+/// through verbatim, no query parsing), rank by frecency, and paginate.
+///
+/// `current_file` deprioritizes the currently open file (NULL/empty to skip).
+/// Zero picks the default: `max_threads` auto, `page_size` 100.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` and `current_file` must be valid null-terminated UTF-8 strings or NULL.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_glob(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    current_file: *const c_char,
+    max_threads: u32,
+    page_index: u32,
+    page_size: u32,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+
+    let pattern_str = match unsafe { cstr_to_str(pattern) } {
+        Some(s) if !s.is_empty() => s,
+        _ => return FffResult::err("Pattern is null, empty, or invalid UTF-8"),
+    };
+
+    let current_file_str = unsafe { optional_cstr(current_file) };
+    let page_size = default_u32(page_size, 100) as usize;
+
+    let picker_guard = match inst.picker.read() {
+        Ok(g) => g,
+        Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
+    };
+
+    let picker = match picker_guard.as_ref() {
+        Some(p) => p,
+        None => {
+            return FffResult::err("File picker not initialized. Call fff_create_instance first.");
+        }
+    };
+
+    let results = picker.glob(
+        pattern_str,
+        FuzzySearchOptions {
+            max_threads: max_threads as usize,
+            current_file: current_file_str,
+            project_path: Some(picker.base_path()),
+            combo_boost_score_multiplier: 0,
+            min_combo_count: 0,
+            pagination: PaginationArgs {
+                offset: page_index as usize,
+                limit: page_size,
+            },
+        },
+    );
+
+    let search_result = FffSearchResult::from_core(&results, picker);
+    FffResult::ok_handle(search_result as *mut c_void)
+}
+
 /// Perform fuzzy search on indexed directories.
 ///
-/// # Parameters
-///
-/// * `fff_handle`   – instance from `fff_create_instance`
-/// * `query`        – search query string
-/// * `current_file` – path of the currently open file for distance scoring (NULL/empty to skip)
-/// * `max_threads`  – maximum worker threads (0 = auto-detect)
-/// * `page_index`   – pagination offset (0 = first page)
-/// * `page_size`    – results per page (0 = default 100)
+/// `current_file` is used for distance scoring (NULL/empty to skip).
+/// Zero picks the default: `max_threads` auto, `page_size` 100.
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -461,20 +528,8 @@ pub unsafe extern "C" fn fff_search_directories(
 
 /// Perform a mixed fuzzy search across both files and directories.
 ///
-/// Returns a single flat list where files and directories are interleaved
-/// by total score in descending order. Each item has an `item_type` field
-/// (0 = file, 1 = directory).
-///
-/// # Parameters
-///
-/// * `fff_handle`              – instance from `fff_create_instance`
-/// * `query`                   – search query string
-/// * `current_file`            – path of the currently open file (NULL/empty to skip)
-/// * `max_threads`             – maximum worker threads (0 = auto-detect)
-/// * `page_index`              – pagination offset (0 = first page)
-/// * `page_size`               – results per page (0 = default 100)
-/// * `combo_boost_multiplier`  – score multiplier for combo matches (0 = default 100)
-/// * `min_combo_count`         – minimum combo count before boost applies (0 = default 3)
+/// Returns one flat list interleaved by descending total score; each item's
+/// `item_type` is 0 = file, 1 = directory. Parameters as in [`fff_search`].
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -548,20 +603,11 @@ pub unsafe extern "C" fn fff_search_mixed(
 
 /// Perform content search (grep) across indexed files.
 ///
-/// # Parameters
-///
-/// * `fff_handle`            – instance from `fff_create_instance`
-/// * `query`                 – search query (supports constraint syntax like `*.rs pattern`)
-/// * `mode`                  – 0 = plain text (SIMD), 1 = regex, 2 = fuzzy
-/// * `max_file_size`         – skip files larger than this in bytes (0 = default 10 MB)
-/// * `max_matches_per_file`  – max matches per file (0 = unlimited)
-/// * `smart_case`            – case-insensitive when query is all lowercase
-/// * `file_offset`           – file-based pagination offset (0 = start)
-/// * `page_limit`            – max matches to return (0 = default 50)
-/// * `time_budget_ms`        – wall-clock budget in ms (0 = unlimited)
-/// * `before_context`        – context lines before each match
-/// * `after_context`         – context lines after each match
-/// * `classify_definitions`  – tag matches that are code definitions
+/// `query` supports constraint syntax like `*.rs pattern`; `mode` is
+/// 0 = plain text (SIMD), 1 = regex, 2 = fuzzy. Zero picks the default:
+/// `max_file_size` 10 MB, `page_limit` 50, `max_matches_per_file` and
+/// `time_budget_ms` unlimited. `smart_case` is case-insensitive for
+/// all-lowercase queries; `classify_definitions` tags code definitions.
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -630,25 +676,11 @@ pub unsafe extern "C" fn fff_live_grep(
     FffResult::ok_handle(grep_result as *mut c_void)
 }
 
-/// Perform multi-pattern OR search (Aho-Corasick) across indexed files.
+/// Multi-pattern OR search (SIMD Aho-Corasick): lines matching ANY pattern.
 ///
-/// Searches for lines matching ANY of the provided patterns using
-/// SIMD-accelerated multi-needle matching.
-///
-/// # Parameters
-///
-/// * `fff_handle`              – instance from `fff_create_instance`
-/// * `patterns_joined`         – patterns separated by `\n` (e.g. `"foo\nbar\nbaz"`)
-/// * `constraints`             – file filter like `"*.rs"` or `"/src/"` (NULL/empty to skip)
-/// * `max_file_size`           – skip files larger than this in bytes (0 = default 10 MB)
-/// * `max_matches_per_file`    – max matches per file (0 = unlimited)
-/// * `smart_case`              – case-insensitive when all patterns are lowercase
-/// * `file_offset`             – file-based pagination offset (0 = start)
-/// * `page_limit`              – max matches to return (0 = default 50)
-/// * `time_budget_ms`          – wall-clock budget in ms (0 = unlimited)
-/// * `before_context`          – context lines before each match
-/// * `after_context`           – context lines after each match
-/// * `classify_definitions`    – tag matches that are code definitions
+/// `patterns_joined` is `\n`-separated (e.g. `"foo\nbar"`); `constraints` is an
+/// optional file filter like `"*.rs"` or `"/src/"` (NULL/empty to skip).
+/// Remaining parameters as in [`fff_live_grep`].
 ///
 /// ## Safety
 /// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -697,19 +729,12 @@ pub unsafe extern "C" fn fff_multi_grep(
         }
     };
 
-    let is_ai = picker.mode().is_ai();
-
     // Parse constraints from the optional string (e.g. "*.rs /src/")
-    let parsed_constraints = constraints_str.map(|c| {
-        if is_ai {
-            fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse(c)
-        } else {
-            fff::grep::parse_grep_query(c)
-        }
-    });
+    let parsed_constraints = constraints_str
+        .map(|c| fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse_constraints(c));
 
     let constraint_refs: &[fff::Constraint<'_>] = match &parsed_constraints {
-        Some(q) => &q.constraints,
+        Some(constraints) => constraints,
         None => &[],
     };
 
@@ -744,18 +769,10 @@ pub unsafe extern "C" fn fff_scan_files(fff_handle: *mut c_void) -> *mut FffResu
         Err(e) => return e,
     };
 
-    let mut guard = match inst.picker.write() {
-        Ok(g) => g,
-        Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
-    };
-
-    let picker = match guard.as_mut() {
-        Some(p) => p,
-        None => return FffResult::err("File picker not initialized"),
-    };
-
-    match picker.trigger_rescan(&inst.frecency) {
-        Ok(_) => FffResult::ok_empty(),
+    // Async: rescan runs on a BG thread, caller returns immediately.
+    // Use `fff_is_scanning` / `fff_wait_for_scan` to observe progress.
+    match inst.picker.trigger_full_rescan_async(&inst.frecency) {
+        Ok(()) => FffResult::ok_empty(),
         Err(e) => FffResult::err(&format!("Failed to trigger rescan: {}", e)),
     }
 }
@@ -778,10 +795,8 @@ pub unsafe extern "C" fn fff_is_scanning(fff_handle: *mut c_void) -> bool {
         .unwrap_or(false)
 }
 
-/// Get the base path of the file picker.
-///
-/// Returns an `FffResult` with a heap-allocated C string in the `handle`
-/// field. Free the string with `fff_free_string` after reading it.
+/// Get the picker's base path as a heap C string in `handle`;
+/// free it with `fff_free_string`.
 ///
 /// ## Safety
 /// `fff_handle` must be a valid instance pointer from `fff_create_instance`.
@@ -898,21 +913,25 @@ pub unsafe extern "C" fn fff_restart_index(
         Err(e) => return FffResult::err(&format!("Failed to canonicalize path: {}", e)),
     };
 
-    let mut guard = match inst.picker.write() {
+    let guard = match inst.picker.write() {
         Ok(g) => g,
         Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
     };
 
-    let (warmup_caches, content_indexing, watch, mode) = if let Some(mut picker) = guard.take() {
-        let warmup = picker.need_enable_mmap_cache();
-        let ci = picker.need_enable_content_indexing();
-        let w = picker.need_watch();
-        let mode = picker.mode();
-        picker.stop_background_monitor();
-        (warmup, ci, w, mode)
-    } else {
-        (false, false, true, FFFMode::default())
-    };
+    let (warmup_caches, content_indexing, watch, mode, fs_root, home_dir, follow_symlinks) =
+        if let Some(ref picker) = *guard {
+            (
+                picker.has_mmap_cache(),
+                picker.has_content_indexing(),
+                picker.has_watcher(),
+                picker.mode(),
+                picker.fs_root_scanning_enabled(),
+                picker.home_dir_scanning_enabled(),
+                picker.follows_symlinks(),
+            )
+        } else {
+            (false, true, true, FFFMode::default(), false, false, false)
+        };
 
     drop(guard);
 
@@ -926,6 +945,9 @@ pub unsafe extern "C" fn fff_restart_index(
             watch,
             mode,
             cache_budget: None,
+            follow_symlinks,
+            enable_fs_root_scanning: fs_root,
+            enable_home_dir_scanning: home_dir,
         },
     ) {
         Ok(()) => FffResult::ok_empty(),
@@ -1236,10 +1258,8 @@ pub unsafe extern "C" fn fff_health_check(
     }
 }
 
-/// Free a search result returned by `fff_search`.
-///
-/// This frees the `FffSearchResult` struct, its `items` and `scores` arrays,
-/// and all heap-allocated strings within each item and score.
+/// Free a search result returned by `fff_search`: the struct, its `items`
+/// and `scores` arrays, and every string within.
 ///
 /// ## Safety
 /// `result` must be a valid pointer previously returned via `FffResult.handle`
@@ -1269,10 +1289,8 @@ pub unsafe extern "C" fn fff_free_search_result(result: *mut FffSearchResult) {
     }
 }
 
-/// Get a pointer to the `index`-th `FffFileItem` in a search result.
-///
-/// Returns null if `result` is null or `index >= result->count`.
-/// The returned pointer is valid until the search result is freed.
+/// Pointer to the `index`-th `FffFileItem`; null if `result` is null or
+/// `index >= count`. Valid until the search result is freed.
 ///
 /// ## Safety
 /// `result` must be a valid `FffSearchResult` pointer from `fff_search`.
@@ -1291,10 +1309,8 @@ pub unsafe extern "C" fn fff_search_result_get_item(
     unsafe { result.items.add(index as usize) }
 }
 
-/// Get a pointer to the `index`-th `FffScore` in a search result.
-///
-/// Returns null if `result` is null or `index >= result->count`.
-/// The returned pointer is valid until the search result is freed.
+/// Pointer to the `index`-th `FffScore`; null if `result` is null or
+/// `index >= count`. Valid until the search result is freed.
 ///
 /// ## Safety
 /// `result` must be a valid `FffSearchResult` pointer from `fff_search`.
@@ -1313,10 +1329,8 @@ pub unsafe extern "C" fn fff_search_result_get_score(
     unsafe { result.scores.add(index as usize) }
 }
 
-/// Free a grep result returned by `fff_live_grep` or `fff_multi_grep`.
-///
-/// This frees the `FffGrepResult` struct, its `items` array, and all
-/// heap-allocated strings, match ranges, and context arrays within each match.
+/// Free a grep result returned by `fff_live_grep` or `fff_multi_grep`:
+/// the struct, its `items` array, and all strings/ranges/context within.
 ///
 /// ## Safety
 /// `result` must be a valid pointer previously returned via `FffResult.handle`
@@ -1343,10 +1357,8 @@ pub unsafe extern "C" fn fff_free_grep_result(result: *mut FffGrepResult) {
     }
 }
 
-/// Get a pointer to the `index`-th `FffGrepMatch` in a grep result.
-///
-/// Returns null if `result` is null or `index >= result->count`.
-/// The returned pointer is valid until the grep result is freed.
+/// Pointer to the `index`-th `FffGrepMatch`; null if `result` is null or
+/// `index >= count`. Valid until the grep result is freed.
 ///
 /// ## Safety
 /// `result` must be a valid `FffGrepResult` pointer from `fff_live_grep` or `fff_multi_grep`.
@@ -1377,10 +1389,8 @@ pub unsafe extern "C" fn fff_free_scan_progress(result: *mut FffScanProgress) {
     }
 }
 
-/// Offset a pointer by `byte_offset` bytes.
-///
-/// General-purpose utility for FFI consumers that need pointer arithmetic
-/// (e.g. iterating over arrays). Returns null if `base` is null.
+/// Offset a pointer by `byte_offset` bytes (FFI array iteration helper).
+/// Returns null if `base` is null.
 ///
 /// ## Safety
 /// The resulting pointer must be within the bounds of the original allocation.
@@ -1392,7 +1402,9 @@ pub unsafe extern "C" fn fff_ptr_offset(base: *const c_void, byte_offset: usize)
     unsafe { (base as *const u8).add(byte_offset) as *const c_void }
 }
 
-/// Free a result returned by any `fff_*` function.
+/// Free a result envelope returned by any `fff_*` function.
+/// **IMPORTANT:** the `handle` payload is NOT freed release it separately
+/// using handle specific cleaning methods (`fff_destroy`, `fff_free_search_result`, etc.).
 ///
 /// ## Safety
 /// `result_ptr` must be a valid pointer returned by a `fff_*` function.
@@ -1407,9 +1419,8 @@ pub unsafe extern "C" fn fff_free_result(result_ptr: *mut FffResult) {
         if !result.error.is_null() {
             drop(CString::from_raw(result.error));
         }
-        // Note: `handle` is NOT freed here — the caller must free it
-        // with the appropriate function (fff_destroy, fff_free_search_result,
-        // fff_free_grep_result, fff_free_string, fff_free_scan_progress, etc.).
+
+        // note: handle is not freed by design
     }
 }
 

crates/fff-c/src/watch.rs

@@ -0,0 +1,351 @@
+use std::ffi::{CString, c_char, c_void};
+use std::ptr;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use fff::{WatchEvent, WatchId, WatchOptions};
+
+use crate::ffi_types::FffResult;
+use crate::instance_ref;
+
+/// Current version of [`FffWatchOptions`].
+pub const FFF_WATCH_OPTIONS_VERSION: u32 = 1;
+
+/// Options for `fff_watch`. Versioned: new fields are only appended.
+#[repr(C)]
+pub struct FffWatchOptions {
+    /// Set to [`FFF_WATCH_OPTIONS_VERSION`] when allocating.
+    pub version: u32,
+    /// Per-subscription excludes (parcel-watcher style): entries with wildcards
+    /// are base-relative globs, entries without are path prefixes. NULL when
+    /// `ignore_count` is 0.
+    pub ignore: *const *const c_char,
+    pub ignore_count: u32,
+    // ----- new version 2+ fields go here, ALWAYS appended -----
+}
+
+/// A single watch event. `kind`: 0 = created, 1 = modified, 2 = removed,
+/// 3 = rescan (events were lost; re-stat what you care about).
+#[repr(C)]
+pub struct FffWatchEvent {
+    /// Absolute path (heap C string owned by the parent batch).
+    pub path: *mut c_char,
+    pub kind: u8,
+}
+
+/// A batch of watch events. Free with `fff_free_watch_events`.
+#[repr(C)]
+pub struct FffWatchEventBatch {
+    pub events: *mut FffWatchEvent,
+    pub count: u32,
+}
+
+/// Instance-wide callback invoked with `(watch_id, batch)` for every `fff_watch`
+/// subscription. The callee owns and frees `batch` via `fff_free_watch_events`.
+pub type FffWatchCallback =
+    unsafe extern "C" fn(watch_id: u64, batch: *mut FffWatchEventBatch, user_data: *mut c_void);
+
+fn batch_into_raw(events: &[WatchEvent]) -> *mut FffWatchEventBatch {
+    let items: Vec<FffWatchEvent> = events
+        .iter()
+        .map(|ev| FffWatchEvent {
+            path: CString::new(ev.path.to_string_lossy().as_bytes())
+                .unwrap_or_default()
+                .into_raw(),
+            kind: ev.kind as u8,
+        })
+        .collect();
+
+    let count = items.len() as u32;
+    let events_ptr = if items.is_empty() {
+        ptr::null_mut()
+    } else {
+        let mut boxed = items.into_boxed_slice();
+        let p = boxed.as_mut_ptr();
+        std::mem::forget(boxed);
+        p
+    };
+
+    Box::into_raw(Box::new(FffWatchEventBatch {
+        events: events_ptr,
+        count,
+    }))
+}
+
+unsafe fn watch_options_from_ffi(
+    opts: *const FffWatchOptions,
+) -> Result<WatchOptions, *mut FffResult> {
+    if opts.is_null() {
+        return Ok(WatchOptions::default());
+    }
+    let opts = unsafe { &*opts };
+    if opts.version == 0 || opts.version > FFF_WATCH_OPTIONS_VERSION {
+        return Err(FffResult::err(&format!(
+            "Unsupported FffWatchOptions version {} (library understands up to {})",
+            opts.version, FFF_WATCH_OPTIONS_VERSION
+        )));
+    }
+
+    let mut ignore = Vec::with_capacity(opts.ignore_count as usize);
+    if opts.ignore_count > 0 {
+        if opts.ignore.is_null() {
+            return Err(FffResult::err("ignore_count > 0 but ignore is NULL"));
+        }
+        for i in 0..opts.ignore_count as usize {
+            let entry = unsafe { *opts.ignore.add(i) };
+            match unsafe { crate::cstr_to_str(entry) } {
+                Some(s) if !s.is_empty() => ignore.push(s.to_string()),
+                Some(_) => {}
+                None => return Err(FffResult::err("ignore entry is NULL or invalid UTF-8")),
+            }
+        }
+    }
+
+    Ok(WatchOptions { ignore })
+}
+
+// The caller guarantees user_data is safe on the callback thread.
+struct UserData(*mut c_void);
+unsafe impl Send for UserData {}
+unsafe impl Sync for UserData {}
+
+// Shared so a closure surviving an unwatch race never dangles.
+#[derive(Default)]
+pub(crate) struct WatchCallbackSlot(Mutex<Option<(FffWatchCallback, UserData)>>);
+
+impl WatchCallbackSlot {
+    fn get(&self) -> Option<(FffWatchCallback, *mut c_void)> {
+        self.0
+            .lock()
+            .ok()
+            .and_then(|guard| guard.as_ref().map(|(cb, ud)| (*cb, ud.0)))
+    }
+
+    fn set(&self, callback: FffWatchCallback, user_data: *mut c_void) {
+        if let Ok(mut guard) = self.0.lock() {
+            *guard = Some((callback, UserData(user_data)));
+        }
+    }
+
+    pub(crate) fn clear(&self) {
+        if let Ok(mut guard) = self.0.lock() {
+            *guard = None;
+        }
+    }
+}
+
+/// Register the instance-wide watch callback used by all `fff_watch`
+/// subscriptions; call before the first `fff_watch`, calling again replaces it.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `callback` must remain callable until fff_unwatch called
+///   `fff_destroy(fff_handle)` returns.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_set_watch_callback(
+    fff_handle: *mut c_void,
+    callback: FffWatchCallback,
+    user_data: *mut c_void,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    inst.watch_callback.set(callback, user_data);
+    FffResult::ok_empty()
+}
+
+/// Subscribe to filesystem changes, delivered through the instance callback
+/// registered by `fff_set_watch_callback`.
+///
+/// Returns the watch id, pass it to `fff_unwatch` to stop.
+///
+/// `pattern` if non `NULL` can be wildcard pattern, absolute, or relative path
+/// that will be used to filter the events triggering exact subscription.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` must be NULL or valid null-terminated UTF-8.
+/// * `opts` must be NULL or a valid `FffWatchOptions` pointer.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    opts: *const FffWatchOptions,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    // NULL pattern = watch the entire indexed tree ("" in core).
+    let pattern_str = if pattern.is_null() {
+        ""
+    } else {
+        match unsafe { crate::cstr_to_str(pattern) } {
+            Some(s) => s,
+            None => return FffResult::err("Pattern is not valid UTF-8"),
+        }
+    };
+    let options = match unsafe { watch_options_from_ffi(opts) } {
+        Ok(o) => o,
+        Err(e) => return e,
+    };
+    if inst.watch_callback.get().is_none() {
+        return FffResult::err("No watch callback registered. Call fff_set_watch_callback first.");
+    }
+
+    let slot = Arc::clone(&inst.watch_callback);
+    let result = inst.picker.watch(pattern_str, options, move |id, events| {
+        if let Some((cb, user_data)) = slot.get() {
+            let batch = batch_into_raw(events);
+            unsafe { cb(id.0, batch, user_data) };
+        }
+    });
+
+    match result {
+        Ok(id) => FffResult::ok_int(id.0 as i64),
+        Err(e) => FffResult::err(&format!("Failed to subscribe: {}", e)),
+    }
+}
+
+/// [`fff_watch`] adapter with flattened options, for FFI libraries that cannot
+/// marshal pointer arrays inside structs (e.g. Node's `ffi-rs`).
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` must be NULL (watch everything) or valid null-terminated UTF-8.
+/// * `ignore` must be NULL or point to `ignore_count` valid C strings.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_args(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    ignore: *const *const c_char,
+    ignore_count: u32,
+) -> *mut FffResult {
+    let opts = FffWatchOptions {
+        version: FFF_WATCH_OPTIONS_VERSION,
+        ignore,
+        ignore_count,
+    };
+    unsafe { fff_watch(fff_handle, pattern, &opts) }
+}
+
+/// Remove a watch subscription. `int_value` = 1 if the id existed, 0 otherwise.
+///
+/// ## Safety
+/// `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_unwatch(fff_handle: *mut c_void, watch_id: u64) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    FffResult::ok_int(inst.picker.unwatch(WatchId(watch_id)) as i64)
+}
+
+/// Number of events in a batch, 0 if `batch` is null.
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_count(batch: *const FffWatchEventBatch) -> u32 {
+    if batch.is_null() {
+        return 0;
+    }
+    unsafe { (*batch).count }
+}
+
+/// Absolute path of event `index`, will be null when out of bounds
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_get_path(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> *const c_char {
+    match unsafe { watch_event_at(batch, index) } {
+        Some(ev) => ev.path,
+        None => ptr::null(),
+    }
+}
+
+/// Kind of event `index` (0 = created, 1 = modified, 2 = removed, 3 = rescan)
+/// 3 (rescan aka "re-stat something" kind) returned when OS based buffer
+/// has been overflown and some events might be loss. Paths will contain a list of
+/// directories that needs to be rescanned to ensure consistency.
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_get_kind(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> u8 {
+    match unsafe { watch_event_at(batch, index) } {
+        Some(ev) => ev.kind,
+        None => 3,
+    }
+}
+
+unsafe fn watch_event_at<'a>(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> Option<&'a FffWatchEvent> {
+    if batch.is_null() {
+        return None;
+    }
+    let batch = unsafe { &*batch };
+    if batch.events.is_null() || index >= batch.count {
+        return None;
+    }
+    Some(unsafe { &*batch.events.add(index as usize) })
+}
+
+/// Free a watch event batch delivered to the instance callback.
+///
+/// ## Safety
+/// `batch` must be a pointer produced by this library, or null (no-op).
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_free_watch_events(batch: *mut FffWatchEventBatch) {
+    if batch.is_null() {
+        return;
+    }
+    unsafe {
+        let batch = Box::from_raw(batch);
+        if !batch.events.is_null() {
+            let events =
+                Vec::from_raw_parts(batch.events, batch.count as usize, batch.count as usize);
+            for ev in events {
+                if !ev.path.is_null() {
+                    drop(CString::from_raw(ev.path));
+                }
+            }
+        }
+    }
+}
+
+// THESE TESTS MUST NEVER BE UPDATED, ONLY EXTENDED WITH NEW FIELDS —
+// bindings hardcode these offsets (ABI stability).
+#[cfg(test)]
+mod layout_tests {
+    use super::*;
+    use std::mem::{offset_of, size_of};
+
+    #[test]
+    #[cfg(target_pointer_width = "64")]
+    fn watch_ffi_layouts_are_stable_64bit() {
+        assert_eq!(size_of::<FffWatchOptions>(), 24);
+        assert_eq!(offset_of!(FffWatchOptions, version), 0);
+        assert_eq!(offset_of!(FffWatchOptions, ignore), 8);
+        assert_eq!(offset_of!(FffWatchOptions, ignore_count), 16);
+
+        assert_eq!(size_of::<FffWatchEvent>(), 16);
+        assert_eq!(offset_of!(FffWatchEvent, path), 0);
+        assert_eq!(offset_of!(FffWatchEvent, kind), 8);
+
+        assert_eq!(size_of::<FffWatchEventBatch>(), 16);
+        assert_eq!(offset_of!(FffWatchEventBatch, events), 0);
+        assert_eq!(offset_of!(FffWatchEventBatch, count), 8);
+    }
+}

crates/fff-c/tests/smoke.c

@@ -0,0 +1,275 @@
+/*
+ * Smoke test for libfff_c — the smallest possible end-to-end exercise of
+ * the public C API. We:
+ *
+ *   1. Create a picker with an `FffCreateOptions` populated via C99
+ *      designated initializers (the recommended idiom for direct C use).
+ *   2. Wait for the initial scan to complete.
+ *   3. Search for "smoke.c".
+ *   4. Fail unless this very file appears in the results.
+ *
+ * Build + run via `make test-c-smoke`. Override $(CC) to test other
+ * compilers.
+ */
+
+/* expose mkdtemp/usleep under -std=c99 on glibc; harmless on musl/darwin */
+#define _DEFAULT_SOURCE
+#define _BSD_SOURCE
+
+#include <fff.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+// simple mock function to make sure that both globbing patterns and dir based pattern work
+static int watch_glob_hits = 0;
+static int watch_dir_hits = 0;
+static int watch_all_hits = 0;
+static int watch_ignored_leaks = 0;
+static uint64_t watch_glob_id = 0;
+static uint64_t watch_dir_id = 0;
+static uint64_t watch_all_id = 0;
+
+static void on_watch_batch(uint64_t watch_id, struct FffWatchEventBatch *batch, void *user_data) {
+    (void)user_data;
+    /* route by id like real SDKs do; unknown ids are benign no-ops */
+    for (uint32_t i = 0; i < batch->count; i++) {
+        const char *path = batch->events[i].path;
+        if (!path) continue;
+        if (watch_id == watch_glob_id && strstr(path, "hello.txt")) {
+            watch_glob_hits++;
+        }
+        if (watch_id == watch_dir_id) {
+            if (strstr(path, "hello.txt")) watch_dir_hits++;
+            if (strstr(path, "noise.log")) watch_ignored_leaks++;
+        }
+        if (watch_id == watch_all_id && strstr(path, "hello.txt")) {
+            watch_all_hits++;
+        }
+    }
+
+    fff_free_watch_events(batch); // need to clean dynamic array of events
+}
+
+static int watch_smoke(void) {
+    char tmpl[] = "/tmp/fff-c-watch-XXXXXX";
+    char *dir = mkdtemp(tmpl);
+    if (!dir) {
+        fprintf(stderr, "watch_smoke: mkdtemp failed\n");
+        return 1;
+    }
+
+    struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){
+        .version = FFF_CREATE_OPTIONS_VERSION,
+        .base_path = dir,
+        .enable_mmap_cache = false,
+        .enable_content_indexing = false,
+        .watch = true,
+    });
+    if (!create_result->success) {
+        fprintf(stderr, "watch_smoke: create failed: %s\n",
+                create_result->error ? create_result->error : "?");
+        fff_free_result(create_result);
+        return 1;
+    }
+    void *picker = create_result->handle;
+    fff_free_result(create_result);
+
+    struct FffResult *r = fff_wait_for_scan(picker, 10000);
+    fff_free_result(r);
+    r = fff_wait_for_watcher(picker, 10000);
+    fff_free_result(r);
+    usleep(300 * 1000); /* let the FSEvents stream settle */
+
+    /* instance-wide callback, then two subscriptions routed by id */
+    r = fff_set_watch_callback(picker, on_watch_batch, NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: fff_set_watch_callback failed: %s\n",
+                r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    fff_free_result(r);
+
+    r = fff_watch(picker, "**/*.txt", NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: fff_watch failed: %s\n", r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_glob_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    /* whole-tree dir subscription with an ignore glob */
+    const char *ignores[] = {"*.log"};
+    r = fff_watch(picker, dir,
+                  &(struct FffWatchOptions){.version = FFF_WATCH_OPTIONS_VERSION,
+                                            .ignore = ignores,
+                                            .ignore_count = 1});
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: dir fff_watch failed: %s\n", r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_dir_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    /* NULL pattern subscribes to the entire indexed tree */
+    r = fff_watch(picker, NULL, NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: NULL-pattern fff_watch failed: %s\n",
+                r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_all_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    char file_path[512];
+    snprintf(file_path, sizeof(file_path), "%s/hello.txt", dir);
+    FILE *f = fopen(file_path, "w");
+    if (!f) {
+        fprintf(stderr, "watch_smoke: fopen failed\n");
+        fff_destroy(picker);
+        return 1;
+    }
+    fputs("hello watch\n", f);
+    fclose(f);
+
+    /* must be filtered out by the dir subscription's ignore glob */
+    char log_path[512];
+    snprintf(log_path, sizeof(log_path), "%s/noise.log", dir);
+    FILE *lf = fopen(log_path, "w");
+    if (lf) {
+        fputs("noise\n", lf);
+        fclose(lf);
+    }
+
+    for (int attempt = 0;
+         attempt < 100 && (watch_glob_hits == 0 || watch_dir_hits == 0 || watch_all_hits == 0);
+         attempt++) {
+        usleep(100 * 1000);
+    }
+
+    r = fff_unwatch(picker, watch_glob_id);
+    fff_free_result(r);
+    r = fff_unwatch(picker, watch_dir_id);
+    fff_free_result(r);
+    r = fff_unwatch(picker, watch_all_id);
+    fff_free_result(r);
+    /* unwatch of an unknown id reports 0, not an error */
+    r = fff_unwatch(picker, watch_dir_id);
+    int unwatch_idempotent = r->success && r->int_value == 0;
+    fff_free_result(r);
+
+    /* fff_destroy is the quiescence barrier: after it returns the callback
+     * will never run again and could be freed (ours is static). */
+    fff_destroy(picker);
+
+    if (watch_glob_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: glob subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_dir_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: dir subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_all_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: NULL-pattern subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_ignored_leaks > 0) {
+        fprintf(stderr, "watch_smoke FAIL: ignore glob leaked %d events\n", watch_ignored_leaks);
+        return 1;
+    }
+    if (!unwatch_idempotent) {
+        fprintf(stderr, "watch_smoke FAIL: repeated unwatch was not a no-op\n");
+        return 1;
+    }
+
+    fprintf(stderr, "watch_smoke PASS (glob=%d dir=%d all=%d)\n", watch_glob_hits, watch_dir_hits,
+            watch_all_hits);
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    const char *base_path = argc > 1 ? argv[1] : ".";
+
+    // make sure that FFF C api is designed more for FFI rather than for direct C usage (I'm sorry)
+    struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){
+        .version = FFF_CREATE_OPTIONS_VERSION,
+        .base_path = base_path,
+        .enable_mmap_cache = false,
+        .enable_content_indexing = false,
+        .watch = false,
+    });
+
+    if (!create_result->success) {
+        fprintf(stderr, "fff couldn't create instance: %s\n",
+                create_result->error ? create_result->error : "?");
+        fff_free_result(create_result);
+        return 1;
+    }
+
+    void *file_picker = create_result->handle;
+    fff_free_result(create_result); // safe to drop now: handle outlives the envelope
+
+    struct FffResult *scan_result = fff_wait_for_scan(file_picker, 5000);
+    if (!scan_result->success) {
+        fprintf(stderr, "wait_for_scan failed: %s\n",
+                scan_result->error ? scan_result->error : "?");
+        fff_free_result(scan_result);
+        fff_destroy(file_picker);
+        return 1;
+    }
+    // int_value: 1 = scan completed in time, 0 = timed out.
+    if (scan_result->int_value == 0) {
+        fprintf(stderr, "wait_for_scan: timed out before initial scan finished\n");
+        fff_free_result(scan_result);
+        fff_destroy(file_picker);
+        return 1;
+    }
+    fff_free_result(scan_result);
+
+    struct FffResult *res = fff_search(file_picker, "smkoe.c", "", 0, 0, 50, 0, 0);
+    if (!res->success) {
+        fprintf(stderr, "search failed: %s\n", res->error ? res->error : "?");
+        fff_free_result(res);
+        fff_destroy(file_picker);
+        return 1;
+    }
+
+    struct FffSearchResult *sr = (struct FffSearchResult *)res->handle;
+    uint32_t total = sr->count;
+    int found = 0;
+    for (uint32_t i = 0; i < sr->count; i++) {
+        const char *path = sr->items[i].relative_path;
+        if (path && strstr(path, "smoke.c")) {
+            found = 1;
+            fprintf(stderr, "found self: %s\n", path);
+            break;
+        }
+    }
+
+    fff_free_search_result(sr);
+    fff_free_result(res);
+    fff_destroy(file_picker);
+
+    if (!found) {
+        fprintf(stderr, "FAIL: smoke.c not in search results (count=%u)\n", total);
+        return 1;
+    }
+
+    if (watch_smoke() != 0) {
+        fprintf(stderr, "FAIL: watch test failed\n");
+        return 1;
+    }
+
+    fprintf(stderr, "PASS\n");
+    return 0;
+}

crates/fff-core/Cargo.toml

@@ -1,11 +1,14 @@
 [package]
 name = "fff-search"
-version = "0.6.4"
+version = "0.10.4"
 edition = "2024"
 license = "MIT"
 authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>"]
 description = "Faboulous & Fast File Finder - a fast and extremely correct file finder SDK with typo resistance, SIMD, prefiltering, and more"
 
+[lints]
+workspace = true
+
 [lib]
 path = "src/lib.rs"
 crate-type = ["rlib", "staticlib", "cdylib"]
@@ -22,10 +25,29 @@ harness = false
 name = "memmem_bench"
 harness = false
 
+[[bench]]
+name = "glob_bench"
+harness = false
+required-features = ["zlob"]
+
+[[bench]]
+name = "grep_bench"
+harness = false
+
 [features]
-default = []
+# `ripgrep` is the pure-Rust walker/glob backend and is on by default so
+# consumers build without a Zig toolchain. CI/release opt into zlob via
+# `--no-default-features --features zlob`.
+default = ["ripgrep"]
 # Enable C FFI exports
 ffi = []
+# Count full rescans and their causes. Always on in debug builds; enable this
+# to keep the accounting in a release build (used by the rescan_probe binary).
+rescan-stats = []
+# Enables POC definition classification for grep result matched lines
+definitions = []
+# Pure-Rust filesystem walker + glob matcher (ignore + globset crates).
+ripgrep = ["dep:ignore", "dep:globset", "fff-query-parser/ripgrep"]
 # Call mi_collect(true) after large allocator churn (bigram build).
 # Requires mimalloc to be the global allocator (linked by fff-nvim).
 mimalloc-collect = ["dep:libmimalloc-sys"]
@@ -39,35 +61,27 @@ rayon = { workspace = true }
 smallvec = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
-fff-query-parser = { workspace = true }
-
-# External dependencies
-bindet = { workspace = true }
+fff-query-parser = { workspace = true , version = "0.9.0" }
 blake3 = { workspace = true }
-chrono = { workspace = true }
 dirs = { workspace = true }
 libc = "0.2"
 git2 = { workspace = true }
 glidesort = { workspace = true }
-globset = { workspace = true }
-fff-grep = { workspace = true }
+globset = { workspace = true, optional = true }
+fff-grep = { workspace = true , version = "0.9.0" }
 aho-corasick = "1"
 memchr = "2"
 heed = { workspace = true }
-ignore = { workspace = true }
+ignore = { workspace = true, optional = true }
 memmap2 = { workspace = true }
 neo_frizbee = { workspace = true }
 notify = { workspace = true }
 notify-debouncer-full = { workspace = true }
-once_cell = { workspace = true }
 parking_lot = { workspace = true }
 pathdiff = { workspace = true }
 regex = { workspace = true }
 regex-syntax = "0.8"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-smartstring = { version = "1.0.1", features = ["serde"] }
-toml = "0.8"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zlob = { workspace = true, optional = true }
@@ -77,7 +91,15 @@ mimalloc = { version = "0.1", optional = true, features = ["local_dynamic_tls"]
 [target.'cfg(windows)'.dependencies]
 dunce = { workspace = true }
 
+# signal-hook only compiles on unix; we wrap the SIGSEGV handler behind cfg(unix)
+[target.'cfg(unix)'.dependencies]
+signal-hook-registry = { workspace = true }
+
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
+ctor = "0.2"
+proptest = { version = "1", default-features = false, features = ["std", "fork"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tempfile = "3.8"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+

crates/fff-core/README.md

@@ -2,6 +2,9 @@
 
 fff is a file search toolkit. It is faster than ripgrep and fzf and designed for a long running applications like file editors, ai agents, or file exploerers.
 
+> [!Important performance information]  
+> For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine.
+
 ## Features
 
 - Fuzzy file name search 

crates/fff-core/benches/bigram_bench.rs

@@ -90,7 +90,8 @@ fn bench_bigram_build(c: &mut Criterion) {
     let file_counts = [10_000, 100_000];
 
     for &file_count in &file_counts {
-        // Pre-generate content so we only measure index building
+        // Pre-generate content so we only measure index building.
+        // Short content (~85 bytes/file) exercises the scalar fast path.
         let contents: Vec<String> = (0..file_count)
             .map(|i| {
                 format!(
@@ -100,7 +101,7 @@ fn bench_bigram_build(c: &mut Criterion) {
             .collect();
 
         group.bench_with_input(
-            BenchmarkId::new("build_and_compress", file_count),
+            BenchmarkId::new("short_content", file_count),
             &file_count,
             |b, &fc| {
                 b.iter(|| {
@@ -114,6 +115,40 @@ fn bench_bigram_build(c: &mut Criterion) {
                 });
             },
         );
+
+        // Long content (~4 KB/file) exercises the SIMD pre-pass path.
+        // Build a realistic-looking source-like blob by repeating snippets.
+        let long_contents: Vec<String> = (0..file_count)
+            .map(|i| {
+                let mut s = String::with_capacity(4096);
+                for j in 0..50 {
+                    s.push_str(&format!(
+                        "pub fn handler_{i}_{j}(ctx: &Context) -> Result<Response, Error> {{\n"
+                    ));
+                    s.push_str("    let parsed = ctx.parse()?;\n");
+                    s.push_str("    let validated = parsed.validate()?;\n");
+                    s.push_str(&format!("    ctx.respond(validated, {}).await\n", j));
+                    s.push_str("}\n\n");
+                }
+                s
+            })
+            .collect();
+
+        group.bench_with_input(
+            BenchmarkId::new("long_content", file_count),
+            &file_count,
+            |b, &fc| {
+                b.iter(|| {
+                    let builder = BigramIndexBuilder::new(fc);
+                    let skip_builder = BigramIndexBuilder::new(fc);
+                    for (i, content) in long_contents.iter().enumerate() {
+                        builder.add_file_content(&skip_builder, i, content.as_bytes());
+                    }
+                    let index = builder.compress(None);
+                    black_box(index.columns_used())
+                });
+            },
+        );
     }
 
     group.finish();

crates/fff-core/benches/glob_bench.rs

@@ -0,0 +1,380 @@
+//! Compare three glob-matching strategies for `match_glob_pattern` in constraints.rs
+//! need to make sure that my zlob changes are not affecting fff's internal globbing
+use ahash::AHashSet;
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+use zlob::{ZlobFlags, ZlobPattern, zlob_match_paths, zlob_match_paths_indices};
+
+fn make_paths(n: usize) -> Vec<String> {
+    let exts = ["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"];
+    let dirs = [
+        "src/core",
+        "src/ui",
+        "crates/fff-core/src",
+        "lua/fff",
+        "tests/integration",
+        "vendor/lib",
+        "node_modules/foo/bar",
+        "docs/internal",
+    ];
+    let mut out = Vec::with_capacity(n);
+    for i in 0..n {
+        let dir = dirs[i % dirs.len()];
+        let ext = exts[i % exts.len()];
+        out.push(format!("{dir}/file_{i}.{ext}"));
+    }
+    out
+}
+
+fn current_impl(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(Some(matches)) = zlob_match_paths(pattern, paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    let matched_set: AHashSet<usize> = matches.iter().map(|s| s.as_ptr() as usize).collect();
+    paths
+        .iter()
+        .enumerate()
+        .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
+        .map(|(i, _)| i)
+        .collect()
+}
+
+fn indices_free_fn(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(hits) = zlob_match_paths_indices(pattern, paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    hits.to_iter().collect()
+}
+
+fn compiled_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(p) = ZlobPattern::compile(pattern, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    let Ok(hits) = p.match_indices(paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    hits.to_iter().collect()
+}
+
+fn bench_glob_strategies(c: &mut Criterion) {
+    let path_counts = [1_000usize, 10_000, 100_000];
+    let patterns: &[(&str, &str)] = &[
+        ("ext_rs", "**/*.rs"),
+        ("dir_glob", "src/**/*.{ts,lua}"),
+        ("literal_seg", "**/node_modules/**"),
+        ("brace_multi", "**/*.{rs,ts,lua,md}"),
+    ];
+
+    for &count in &path_counts {
+        let owned = make_paths(count);
+        let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+
+        let mut group = c.benchmark_group(format!("glob_{count}"));
+        group.sample_size(50);
+
+        for &(name, pat) in patterns {
+            let id_curr = BenchmarkId::new("current_ptr_trick", name);
+            group.bench_with_input(id_curr, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = current_impl(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+
+            let id_idx = BenchmarkId::new("match_indices_fn", name);
+            group.bench_with_input(id_idx, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = indices_free_fn(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+
+            let id_comp = BenchmarkId::new("compiled_pattern", name);
+            group.bench_with_input(id_comp, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = compiled_pattern(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+        }
+
+        group.finish();
+    }
+}
+
+/// Hot-loop: pattern compiled ONCE, matched many times against fresh path slices.
+/// Models a hypothetical change where we cache compiled patterns across calls.
+fn bench_compiled_reuse(c: &mut Criterion) {
+    let owned = make_paths(10_000);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let pat = "**/*.{rs,ts,lua,md}";
+
+    let mut group = c.benchmark_group("glob_reuse_10k");
+    group.sample_size(100);
+
+    group.bench_function("recompile_each_time", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let hits = p
+                .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED)
+                .unwrap();
+            black_box(hits.len());
+        });
+    });
+
+    let compiled = ZlobPattern::compile(pat, ZlobFlags::RECOMMENDED).unwrap();
+    group.bench_function("reuse_compiled", |b| {
+        b.iter(|| {
+            let hits = compiled
+                .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED)
+                .unwrap();
+            black_box(hits.len());
+        });
+    });
+
+    group.finish();
+}
+
+/// End-to-end: build the lookup AND iterate items checking membership, modeling the
+/// real call shape in `apply_constraints` (filter loop reads the result for every item).
+fn bench_full_pipeline(c: &mut Criterion) {
+    bench_full_pipeline_size(c, 100_000);
+    bench_full_pipeline_size(c, 500_000);
+}
+
+fn bench_full_pipeline_size(c: &mut Criterion, count: usize) {
+    let owned = make_paths(count);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let pat = "**/*.{rs,ts,lua,md}";
+
+    let mut group = c.benchmark_group(format!("glob_full_pipeline_{count}"));
+    group.sample_size(50);
+
+    // (A) current: indices -> AHashSet -> per-item set.contains
+    group.bench_function("indices_to_ahashset_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let set: AHashSet<usize> = hits.to_iter().collect();
+            let count = (0..paths.len()).filter(|i| set.contains(i)).count();
+            black_box(count);
+        });
+    });
+
+    // (B) indices -> Vec<bool> bitmap -> per-item array lookup
+    group.bench_function("indices_to_bitmap_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for i in hits.to_iter() {
+                mask[i] = true;
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    // (C) compiled pattern + per-item matches() inside the filter loop. No batch.
+    group.bench_function("compiled_per_item_matches", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let count = paths.iter().filter(|path| p.matches_default(path)).count();
+            black_box(count);
+        });
+    });
+
+    // (D) compiled pattern + chunked batch -> Vec<bool> bitmap. Best of both:
+    //     SIMD batch wins inside chunks, no global allocation pressure, O(1) lookup.
+    group.bench_function("compiled_chunked_to_bitmap", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for (chunk_idx, chunk) in paths.chunks(512).enumerate() {
+                let base = chunk_idx * 512;
+                let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap();
+                for i in hits.to_iter() {
+                    mask[base + i] = true;
+                }
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    // (E') indices -> bit-packed Vec<u64> -> per-item bit test
+    group.bench_function("indices_to_bitset_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let words = paths.len().div_ceil(64);
+            let mut bits = vec![0u64; words];
+            for i in hits.to_iter() {
+                bits[i >> 6] |= 1u64 << (i & 63);
+            }
+            let count = (0..paths.len())
+                .filter(|&i| (bits[i >> 6] >> (i & 63)) & 1 == 1)
+                .count();
+            black_box(count);
+        });
+    });
+
+    // (E) (D) but larger chunk
+    group.bench_function("compiled_chunked_4096_to_bitmap", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for (chunk_idx, chunk) in paths.chunks(4096).enumerate() {
+                let base = chunk_idx * 4096;
+                let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap();
+                for i in hits.to_iter() {
+                    mask[base + i] = true;
+                }
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+/// Mixed-constraint pipeline: glob + ext. Compare pre-pass batch (current) vs
+/// inline `ZlobPattern::matches` after the cheap ext check rejects items.
+///
+/// Variables: ext rejection rate. Extreme cases reveal where each strategy wins.
+fn bench_mixed_pipeline(c: &mut Criterion) {
+    let count = 100_000;
+    let owned = make_paths(count);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let glob_pat = "**/*.{rs,ts,lua,md}";
+
+    // 4 ext sets: from very selective (1/10 paths kept) to permissive (kept all).
+    let scenarios: &[(&str, &[&str])] = &[
+        ("ext_1of10", &["rs"]),
+        ("ext_4of10", &["rs", "ts", "lua", "md"]),
+        (
+            "ext_8of10",
+            &["rs", "ts", "lua", "md", "toml", "go", "py", "c"],
+        ),
+        (
+            "ext_all",
+            &["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"],
+        ),
+    ];
+
+    fn ext_match(name: &str, exts: &[&str]) -> bool {
+        exts.iter().any(|e| {
+            let bytes = name.as_bytes();
+            let elen = e.len();
+            bytes.len() > elen + 1
+                && bytes[bytes.len() - elen - 1] == b'.'
+                && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes())
+        })
+    }
+
+    let mut group = c.benchmark_group("glob_mixed_100k");
+    group.sample_size(50);
+
+    for &(name, exts) in scenarios {
+        // (A) PRE-PASS: build bitmap for ALL paths, then per-item ext-then-bitmap.
+        let id_pre = BenchmarkId::new("prepass_bitmap", name);
+        group.bench_with_input(id_pre, &exts, |b, &exts| {
+            b.iter(|| {
+                let hits =
+                    zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED)
+                        .unwrap();
+                let mut mask = vec![false; paths.len()];
+                for i in hits.to_iter() {
+                    mask[i] = true;
+                }
+                let count = paths
+                    .iter()
+                    .enumerate()
+                    .filter(|&(_, p)| ext_match(p, exts))
+                    .filter(|&(i, _)| mask[i])
+                    .count();
+                black_box(count);
+            });
+        });
+
+        // (B) INLINE: compile once, per-item ext check first, then matches() only on survivors.
+        let id_inline = BenchmarkId::new("inline_compiled", name);
+        group.bench_with_input(id_inline, &exts, |b, &exts| {
+            b.iter(|| {
+                let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap();
+                let count = paths
+                    .iter()
+                    .filter(|path| ext_match(path, exts))
+                    .filter(|path| p.matches_default(path))
+                    .count();
+                black_box(count);
+            });
+        });
+    }
+
+    group.finish();
+}
+
+/// Compare hand-rolled `file_has_extension` byte compare vs compiling extensions
+/// into a single brace glob `**/*.{rs,ts,lua,md}` and dispatching through zlob.
+/// Both share the same per-item "filter then count" shape.
+fn bench_extensions_vs_glob(c: &mut Criterion) {
+    let owned = make_paths(100_000);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let exts = ["rs", "ts", "lua", "md"];
+    let glob_pat = "**/*.{rs,ts,lua,md}";
+
+    fn ext_match(name: &str, exts: &[&str]) -> bool {
+        let bytes = name.as_bytes();
+        exts.iter().any(|e| {
+            let elen = e.len();
+            bytes.len() > elen + 1
+                && bytes[bytes.len() - elen - 1] == b'.'
+                && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes())
+        })
+    }
+
+    let mut group = c.benchmark_group("ext_vs_glob_100k");
+    group.sample_size(50);
+
+    group.bench_function("file_has_extension_loop", |b| {
+        b.iter(|| {
+            let count = paths.iter().filter(|p| ext_match(p, &exts)).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("compiled_brace_glob_inline", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap();
+            let count = paths.iter().filter(|path| p.matches_default(path)).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("brace_glob_prepass_bitmap", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED)
+                    .unwrap();
+            let mut mask = vec![false; paths.len()];
+            for i in hits.to_iter() {
+                mask[i] = true;
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_glob_strategies,
+    bench_compiled_reuse,
+    bench_full_pipeline,
+    bench_mixed_pipeline,
+    bench_extensions_vs_glob
+);
+criterion_main!(benches);

crates/fff-core/benches/grep_bench.rs

@@ -0,0 +1,105 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use fff_search::file_picker::{FilePicker, FilePickerOptions};
+use fff_search::{GrepMode, GrepSearchOptions, parse_grep_query};
+use std::io::Write;
+
+/// Synthetic repo: half the files contain the needle on every line (stresses
+/// the per-match find/highlight path), half are pure noise (stresses the
+/// whole-file prefilter path).
+fn setup_repo(dir: &std::path::Path) {
+    for i in 0..400 {
+        let mut f = std::fs::File::create(dir.join(format!("match_{i}.rs"))).unwrap();
+        for j in 0..100 {
+            writeln!(
+                f,
+                "fn handle_{j}() {{ let controller = Controller::new({j}); controller.run(); }}"
+            )
+            .unwrap();
+        }
+    }
+    for i in 0..400 {
+        let mut f = std::fs::File::create(dir.join(format!("noise_{i}.rs"))).unwrap();
+        for j in 0..100 {
+            writeln!(
+                f,
+                "fn compute_{j}() {{ let value = {j} * 42; process(value); }}"
+            )
+            .unwrap();
+        }
+    }
+}
+
+fn options(mode: GrepMode) -> GrepSearchOptions {
+    GrepSearchOptions {
+        // Force a full scan of every file so we measure matcher/sink work,
+        // not pagination early-exit.
+        page_limit: usize::MAX,
+        max_matches_per_file: 0,
+        mode,
+        ..Default::default()
+    }
+}
+
+fn bench_grep(c: &mut Criterion) {
+    let dir = tempfile::tempdir().unwrap();
+    setup_repo(dir.path());
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: dir.path().to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+    assert_eq!(picker.get_files().len(), 800);
+
+    let mut group = c.benchmark_group("grep_e2e");
+    group.sample_size(30);
+
+    // Case-sensitive, 40k matched lines: hottest find_at/highlight path
+    let query = parse_grep_query("Controller");
+    let opts = options(GrepMode::PlainText);
+    group.bench_function("plain_case_sensitive_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    // Case-insensitive (SIMD folding path), 120k matched spans
+    let query = parse_grep_query("controller");
+    group.bench_function("plain_case_insensitive_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    // No matches anywhere: whole-file prefilter dominates
+    let query = parse_grep_query("Qqzyx");
+    group.bench_function("plain_no_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 0);
+            std::hint::black_box(r.total_files_searched)
+        });
+    });
+
+    // Regex mode: must be unaffected by NeedleFinder changes
+    let query = parse_grep_query("Contr[a-z]+ller");
+    let regex_opts = options(GrepMode::Regex);
+    group.bench_function("regex_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &regex_opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_grep);
+criterion_main!(benches);

crates/fff-core/benches/memmem_bench.rs

@@ -1,5 +1,5 @@
 use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
-use fff_search::case_insensitive_memmem;
+use fff_search::simd_string_utils::memmem;
 use std::path::Path;
 
 /// Load real source files from the repository as benchmark haystacks.
@@ -41,7 +41,7 @@ fn load_real_files() -> Vec<(&'static str, Vec<u8>)> {
 }
 
 fn bench_memmem(c: &mut Criterion) {
-    let mut group = c.benchmark_group("case_insensitive_memmem");
+    let mut group = c.benchmark_group("simd_string_utils_memmem");
 
     let files = load_real_files();
     assert!(!files.is_empty(), "No source files found for benchmarking");
@@ -69,26 +69,10 @@ fn bench_memmem(c: &mut Criterion) {
             let id = format!("{file_label}/{needle_label}");
 
             group.bench_with_input(
-                BenchmarkId::new("packed_pair", &id),
+                BenchmarkId::new("find", &id),
                 &(haystack, &needle_lower),
                 |b, &(h, n)| {
-                    b.iter(|| black_box(case_insensitive_memmem::search_packed_pair(h, n)));
-                },
-            );
-
-            group.bench_with_input(
-                BenchmarkId::new("memchr2_search", &id),
-                &(haystack, &needle_lower),
-                |b, &(h, n)| {
-                    b.iter(|| black_box(case_insensitive_memmem::search(h, n)));
-                },
-            );
-
-            group.bench_with_input(
-                BenchmarkId::new("scalar_baseline", &id),
-                &(haystack, &needle_lower),
-                |b, &(h, n)| {
-                    b.iter(|| black_box(case_insensitive_memmem::search_scalar(h, n)));
+                    b.iter(|| black_box(memmem::find(h, n)));
                 },
             );
         }

crates/fff-core/build.rs

@@ -1,36 +1,45 @@
 fn main() {
+    // Opt-in cfg for the long-running randomized stress tests
+    // used by tests/fuzz_git_watcher_stress.rs
+    println!("cargo::rustc-check-cfg=cfg(stress)");
+
+    // Full-rescan accounting. Debug builds get it for free; a release build has
+    // to opt in with `--features rescan-stats` (what the rescan_probe needs).
+    println!("cargo::rustc-check-cfg=cfg(rescan_stats)");
+    if std::env::var("DEBUG").is_ok_and(|debug| debug != "false")
+        || std::env::var("CARGO_FEATURE_RESCAN_STATS").is_ok()
+    {
+        println!("cargo::rustc-cfg=rescan_stats");
+    }
+
     // When the `zlob` feature is enabled (Zig-compiled C library):
     // On Windows MSVC, explicitly link the C runtime libraries.
     // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the
     // MSVC CRT, so symbols like strcmp, memcpy etc. would be unresolved.
     if std::env::var("CARGO_FEATURE_ZLOB").is_ok() {
+        if !zig_available() {
+            panic!(
+                "The `zlob` feature is enabled but Zig is not installed. \
+                 Install Zig (https://ziglang.org/download/) or build without \
+                 `--features zlob`."
+            );
+        }
+
         let target = std::env::var("TARGET").unwrap_or_default();
         if target.contains("windows") && target.contains("msvc") {
             println!("cargo:rustc-link-lib=msvcrt");
             println!("cargo:rustc-link-lib=ucrt");
             println!("cargo:rustc-link-lib=vcruntime");
         }
-    } else if std::env::var("CI").is_ok() {
-        // CI must always build with zlob for production-quality binaries.
-        if !zig_available() {
-            panic!(
-                "CI detected but Zig is not installed. \
-                 Please install Zig and build with `--features zlob`."
-            );
-        }
-        panic!(
-            "CI detected but `zlob` feature is not enabled. \
-             Build with `--features zlob`."
-        );
-    } else {
+    } else if std::env::var("CARGO_PRIMARY_PACKAGE").is_ok() && zig_available() {
         // Hint: if Zig is available but the zlob feature wasn't enabled,
         // let the developer know they can get faster glob matching.
-        if zig_available() {
-            println!(
-                "cargo:warning=Zig detected but `zlob` feature is not enabled. \
-                 Build with `--features zlob` for faster glob matching."
-            );
-        }
+        // Only emit this hint when this crate is the primary package to
+        // avoid noisy warnings for downstream consumers.
+        println!(
+            "cargo:warning=Zig detected but `zlob` feature is not enabled. \
+             Build with `--features zlob` for faster glob matching."
+        );
     }
 }
 
@@ -41,6 +50,5 @@ fn zig_available() -> bool {
         .stdout(std::process::Stdio::null())
         .stderr(std::process::Stdio::null())
         .status()
-        .map(|s| s.success())
-        .unwrap_or(false)
+        .is_ok()
 }

crates/fff-core/src/background_watcher.rs

@@ -1,716 +0,0 @@
-use crate::error::Error;
-use crate::file_picker::{FFFMode, FilePicker};
-use crate::git::GitStatusCache;
-use crate::shared::{SharedFrecency, SharedPicker};
-use crate::sort_buffer::sort_with_buffer;
-use git2::Repository;
-use notify::event::{AccessKind, AccessMode};
-use notify::{Config, EventKind, EventKindMask, RecursiveMode};
-use notify_debouncer_full::{DebounceEventResult, DebouncedEvent, NoCache, new_debouncer_opt};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc;
-use std::time::Duration;
-use tracing::{Level, debug, error, info, warn};
-
-type Debouncer = notify_debouncer_full::Debouncer<notify::RecommendedWatcher, NoCache>;
-
-/// Owns the file-system watcher and guarantees that all background threads
-/// are fully joined before `stop()` / `Drop` returns.
-///
-/// Architecture:
-///   - The debouncer (and its internal watcher) live inside an **owner thread**
-///     that we spawn and hold the `JoinHandle` for.
-///   - `stop()` sets a flag, unparks the owner thread, and **joins** it.
-///   - Inside the owner thread, `Debouncer::stop()` is called which joins the
-///     debouncer's event-processing thread.
-///   - On Windows an additional short sleep is added after `Debouncer::stop()`
-///     because `notify`'s `ReadDirectoryChangesWatcher` discards its thread
-///     `JoinHandle`, so we cannot join it directly. The watcher's `Drop` does
-///     signal the thread via semaphore so it exits almost immediately, but we
-///     need to give the OS a moment to reclaim it.
-pub struct BackgroundWatcher {
-    stop_signal: Arc<AtomicBool>,
-    owner_thread: Option<std::thread::JoinHandle<()>>,
-}
-
-const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(250);
-const MAX_PATHS_THRESHOLD: usize = 1024;
-/// On macOS, each `watch()` call creates a separate FSEventStream. When the
-/// number of directories exceeds this threshold we fall back to a single
-/// recursive watch to avoid exhausting the per-process stream limit.
-const MAX_MACOS_NONRECURSIVE_WATCHES: usize = 4096;
-/// Minimum seconds between frecency tracks of the same file in AI mode.
-/// Prevents score inflation from rapid burst edits by AI agents.
-const AI_MODE_COOLDOWN_SECS: u64 = 5 * 60;
-
-impl BackgroundWatcher {
-    pub fn new(
-        base_path: PathBuf,
-        git_workdir: Option<PathBuf>,
-        shared_picker: SharedPicker,
-        shared_frecency: SharedFrecency,
-        mode: FFFMode,
-        watch_dirs: Vec<PathBuf>,
-    ) -> Result<Self, Error> {
-        info!(
-            "Initializing background watcher for path: {}, mode: {:?}",
-            base_path.display(),
-            mode,
-        );
-
-        let (watch_tx, watch_rx) = mpsc::channel::<PathBuf>();
-
-        // Clone shared state for the owner thread (needed for injecting
-        // files that existed before a watch was registered on their directory).
-        let owner_picker = shared_picker.clone();
-        let owner_git_workdir = git_workdir.clone();
-
-        let debouncer = Self::create_debouncer(
-            base_path,
-            git_workdir,
-            shared_picker,
-            shared_frecency,
-            mode,
-            watch_dirs,
-            watch_tx,
-        )?;
-        info!("Background file watcher initialized successfully");
-
-        let stop_signal = Arc::new(AtomicBool::new(false));
-        let stop_clone = Arc::clone(&stop_signal);
-
-        // The owner thread keeps the debouncer alive and ensures proper
-        // cleanup: `Debouncer::stop()` joins its internal thread, then the
-        // watcher `Drop` signals its I/O thread to exit.
-        let owner_thread = std::thread::Builder::new()
-            .name("fff-watcher-owner".into())
-            .spawn(move || {
-                let mut debouncer = debouncer;
-                while !stop_clone.load(Ordering::Acquire) {
-                    // Process pending watch requests from the event handler
-                    // (new directories that need to be watched).
-                    while let Ok(dir) = watch_rx.try_recv() {
-                        match debouncer.watch(dir.as_path(), RecursiveMode::NonRecursive) {
-                            Ok(()) => {
-                                debug!("Added watch for new directory: {}", dir.display());
-                            }
-                            Err(e) => {
-                                warn!("Failed to watch new directory {}: {}", dir.display(), e);
-                            }
-                        }
-
-                        // Files created before the watch was registered don't
-                        // generate events. Do a flat (non-recursive) read_dir
-                        // to inject any files that already exist. Subdirectories
-                        // are not descended — they get their own watches via
-                        // future Create events from this directory's watch.
-                        inject_existing_files(&dir, &owner_picker, &owner_git_workdir);
-                    }
-                    std::thread::park_timeout(Duration::from_secs(1));
-                }
-                // Debouncer::stop() joins the debouncer's event thread, then
-                // drops the watcher (whose Drop signals the I/O thread).
-                debouncer.stop();
-                // On Windows the notify crate discards the ReadDirectoryChangesW
-                // thread's JoinHandle — we cannot join it. Its Drop signals the
-                // thread via semaphore so it exits almost immediately; give the
-                // OS a moment to fully reclaim it.
-                #[cfg(windows)]
-                std::thread::sleep(Duration::from_millis(250));
-            })
-            .expect("failed to spawn fff-watcher-owner thread");
-
-        Ok(Self {
-            stop_signal,
-            owner_thread: Some(owner_thread),
-        })
-    }
-
-    fn create_debouncer(
-        base_path: PathBuf,
-        git_workdir: Option<PathBuf>,
-        shared_picker: SharedPicker,
-        shared_frecency: SharedFrecency,
-        mode: FFFMode,
-        watch_dirs: Vec<PathBuf>,
-        watch_tx: mpsc::Sender<PathBuf>,
-    ) -> Result<Debouncer, Error> {
-        let config = Config::default()
-            // do not follow symlinks as then notifiers spawns a bunch of events for symlinked
-            // files that could be git ignored, we have to property differentiate those and if
-            // the file was edited through a
-            .with_follow_symlinks(false)
-            // only the actual modification events, ignore the open syscals that we can generate by
-            // our own grep calls and preview window rendering
-            .with_event_kinds(EventKindMask::CORE);
-
-        // Decide the watching strategy up-front so the event handler closure
-        // knows whether it needs to request dynamic directory watches.
-        let use_recursive =
-            cfg!(target_os = "macos") && watch_dirs.len() > MAX_MACOS_NONRECURSIVE_WATCHES;
-
-        let git_workdir_for_handler = git_workdir.clone();
-        let mut debouncer = new_debouncer_opt(
-            DEBOUNCE_TIMEOUT,
-            Some(DEBOUNCE_TIMEOUT / 2), // tick rate for the event span
-            {
-                move |result: DebounceEventResult| match result {
-                    Ok(events) => {
-                        let new_dirs = handle_debounced_events(
-                            events,
-                            &git_workdir_for_handler,
-                            &shared_picker,
-                            &shared_frecency,
-                            mode,
-                        );
-
-                        // In NonRecursive mode, register watches for newly
-                        // discovered directories so future file events in them
-                        // are captured. In Recursive mode the single stream
-                        // already covers new subdirectories.
-                        if !use_recursive {
-                            for dir in new_dirs {
-                                let _ = watch_tx.send(dir);
-                            }
-                        }
-                    }
-                    Err(errors) => {
-                        error!("File watcher errors: {:?}", errors);
-                    }
-                }
-            },
-            // There is an issue with recommended cache implementation on macos
-            // it keeps track of all the files added to the watcher which is not a problem
-            // for us because any rename to the file will anyway require the removing from the
-            // ordedred index and adding it back with the new name
-            NoCache::new(),
-            config,
-        )?;
-
-        // Watching strategy:
-        //
-        // For small-to-medium repos we watch each indexed directory individually
-        // (NonRecursive). This avoids receiving events for gitignored paths like
-        // node_modules/ and keeps the event volume low.
-        //
-        // On macOS, each `watch()` call creates a separate FSEventStream. Large
-        // repos (e.g. Chromium with 487K+ files) can have tens of thousands of
-        // directories, which exhausts the per-process FSEvents stream limit and
-        // causes "unable to start FSEvent stream" errors. When the directory
-        // count exceeds the threshold we fall back to a single Recursive watch
-        // on the base path. FSEvents handles this efficiently with one kernel
-        // stream for the entire subtree. Gitignored paths are already filtered
-        // in the event handler via `should_include_file()`.
-        //
-        // On Linux (inotify), RecursiveMode::Recursive creates one kernel watch
-        // per subdirectory *including* gitignored ones, wasting file descriptors.
-        // The per-directory NonRecursive approach is always used on Linux.
-        //
-        // New directories created at runtime are detected via Create events on
-        // the parent and dynamically added by the owner thread via watch_tx.
-
-        if use_recursive {
-            debouncer.watch(base_path.as_path(), RecursiveMode::Recursive)?;
-            info!(
-                "File watcher initialized with single recursive watch on {} \
-                 ({} directories exceeded threshold of {})",
-                base_path.display(),
-                watch_dirs.len(),
-                MAX_MACOS_NONRECURSIVE_WATCHES,
-            );
-        } else {
-            debouncer.watch(base_path.as_path(), RecursiveMode::NonRecursive)?;
-
-            for dir in &watch_dirs {
-                match debouncer.watch(dir.as_path(), RecursiveMode::NonRecursive) {
-                    Ok(()) => {}
-                    Err(e) => {
-                        // Non-fatal: directory may have been removed between discovery and watch
-                        warn!("Failed to watch directory {}: {}", dir.display(), e);
-                    }
-                }
-            }
-
-            info!(
-                "File watcher initialized for {} directories (NonRecursive) under {}",
-                watch_dirs.len(),
-                base_path.display()
-            );
-        }
-
-        // The .git directory is excluded from the file list but we still need
-        // to observe changes that affect git status (staging, unstaging,
-        // committing, branch switches, merges, etc.).
-        // When using recursive mode the base watch already covers .git/,
-        // but these targeted watches are cheap (at most 3 extra streams)
-        // and ensure we catch status changes even if the recursive backend
-        // coalesces or delays .git events.
-        watch_git_status_paths(&mut debouncer, git_workdir.as_ref());
-
-        Ok(debouncer)
-    }
-
-    pub fn stop(&mut self) {
-        self.stop_signal.store(true, Ordering::Release);
-        if let Some(handle) = self.owner_thread.take() {
-            handle.thread().unpark();
-
-            if let Err(e) = handle.join() {
-                error!("Watcher owner thread panicked: {:?}", e);
-            }
-        }
-
-        info!("Background file watcher stopped successfully");
-    }
-}
-
-impl Drop for BackgroundWatcher {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}
-
-#[tracing::instrument(name = "fs_events", skip(events, shared_picker, shared_frecency), level = Level::DEBUG)]
-fn handle_debounced_events(
-    events: Vec<DebouncedEvent>,
-    git_workdir: &Option<PathBuf>,
-    shared_picker: &SharedPicker,
-    shared_frecency: &SharedFrecency,
-    mode: FFFMode,
-) -> Vec<PathBuf> {
-    // this will be called very often, we have to minimiy the lock time for file picker
-    let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok());
-    let mut need_full_rescan = false;
-    let mut need_full_git_rescan = false;
-    let mut paths_to_remove = Vec::new();
-    let mut paths_to_add_or_modify = Vec::new();
-    let mut new_dirs_to_watch = Vec::new();
-    let mut affected_paths_count = 0usize;
-
-    for debounced_event in &events {
-        // It is very important to not react to the access errors because we inevitably
-        // gonna trigger the sync by our own preview or other unnecessary noise
-        if matches!(
-            debounced_event.event.kind,
-            EventKind::Access(
-                AccessKind::Read
-                    | AccessKind::Open(_)
-                    | AccessKind::Close(AccessMode::Read | AccessMode::Execute)
-            )
-        ) {
-            continue;
-        }
-
-        // When macOS FSEvents (or other backends) overflow their event buffer, the kernel
-        // drops individual events and emits a Rescan flag telling us to re-scan the subtree.
-        // Without handling this, modified source files can be silently missed.
-        if debounced_event.event.need_rescan() {
-            warn!(
-                "Received rescan event for paths {:?}, triggering full rescan",
-                debounced_event.event.paths
-            );
-            need_full_rescan = true;
-            break;
-        }
-
-        tracing::debug!(event = ?debounced_event.event, "Processing FS event");
-        for path in &debounced_event.event.paths {
-            if is_ignore_definition_path(path) {
-                info!(
-                    "Detected change in ignore definition file: {}",
-                    path.display()
-                );
-                need_full_rescan = true;
-                break;
-            }
-
-            if is_dotgit_change_affecting_status(path, &repo) {
-                need_full_git_rescan = true;
-            }
-
-            if is_git_file(path) {
-                continue;
-            }
-
-            // Use a combination of event kind and filesystem state to decide
-            // whether a path is an addition/modification or a removal.
-            //
-            // We cannot rely on `path.exists()` alone because:
-            //   - A freshly created file might not be visible yet (race).
-            //   - macOS FSEvents uses Modify(Name(Any)) for both rename-in
-            //     and rename-out, so we must stat the path to disambiguate.
-            //
-            // We cannot rely on event kind alone because:
-            //   - Remove events are not always emitted (macOS often sends
-            //     Modify(Name(Any)) instead of Remove).
-            let is_removal = matches!(debounced_event.event.kind, EventKind::Remove(_));
-
-            if is_removal || !path.exists() {
-                paths_to_remove.push(path.as_path());
-            } else if path.is_dir() {
-                // New directory — collect it so the caller can register a
-                // watcher. No filesystem scanning: files that arrive later
-                // will be handled by the newly registered watch.
-                if !is_path_ignored(path, &repo) {
-                    new_dirs_to_watch.push(path.to_path_buf());
-                }
-            } else {
-                // For additions/modifications, still filter gitignored files.
-                if should_include_file(path, &repo) {
-                    paths_to_add_or_modify.push(path.as_path());
-                }
-            }
-        }
-
-        affected_paths_count += debounced_event.event.paths.len();
-        if affected_paths_count > MAX_PATHS_THRESHOLD {
-            warn!(
-                "Too many affected paths ({}) in a single batch, triggering full rescan",
-                affected_paths_count
-            );
-
-            need_full_rescan = true;
-            break;
-        }
-
-        if need_full_rescan {
-            break;
-        }
-    }
-
-    if need_full_rescan {
-        info!(?affected_paths_count, "Triggering full rescan");
-        trigger_full_rescan(shared_picker, shared_frecency);
-        return Vec::new();
-    }
-
-    // It's important to get the allocated sort
-    sort_with_buffer(paths_to_add_or_modify.as_mut_slice(), |a, b| {
-        a.as_os_str().cmp(b.as_os_str())
-    });
-    paths_to_add_or_modify.dedup_by(|a, b| a.as_os_str().eq(b.as_os_str()));
-
-    info!(
-        "Event processing summary: {} to remove, {} to add/modify, {} new dirs",
-        paths_to_remove.len(),
-        paths_to_add_or_modify.len(),
-        new_dirs_to_watch.len()
-    );
-
-    // Apply file index updates (add/remove) unconditionally — these must
-    // happen even when there is no git repository.
-    let files_to_update_git_status =
-        if !paths_to_remove.is_empty() || !paths_to_add_or_modify.is_empty() {
-            debug!(
-                "Applying file index changes: {} to remove, {} to add/modify",
-                paths_to_remove.len(),
-                paths_to_add_or_modify.len(),
-            );
-
-            let apply_changes = |picker: &mut FilePicker| -> Vec<PathBuf> {
-                for path in &paths_to_remove {
-                    let removed = picker.remove_file_by_path(path);
-                    debug!("remove_file_by_path({:?}) -> {}", path, removed);
-                }
-
-                let mut files_to_update = Vec::with_capacity(paths_to_add_or_modify.len());
-                for path in &paths_to_add_or_modify {
-                    let added = picker.on_create_or_modify(path).is_some();
-                    if added {
-                        debug!("on_create_or_modify({:?}) -> Some", path);
-                        files_to_update.push(path.to_path_buf());
-                    } else {
-                        error!("on_create_or_modify({:?}) -> None (file not added!)", path);
-                    }
-                }
-                info!(
-                    "apply_changes complete: {} files to update git status",
-                    files_to_update.len()
-                );
-                files_to_update
-            };
-
-            let Ok(mut guard) = shared_picker.write() else {
-                error!("Failed to acquire file picker write lock");
-                return new_dirs_to_watch;
-            };
-            let Some(ref mut picker) = *guard else {
-                error!("File picker not initialized");
-                return new_dirs_to_watch;
-            };
-            apply_changes(picker)
-        } else {
-            debug!("No file index changes to apply");
-            Vec::new()
-        };
-
-    // AI mode: auto-track frecency for all modified/created files.
-    // Uses a 5-minute cooldown per file to prevent score inflation from rapid
-    // burst edits (AI agents often edit the same file many times in minutes).
-    // This runs after apply_changes so the picker write lock is released.
-    if mode.is_ai() && !paths_to_add_or_modify.is_empty() {
-        let mut tracked_count = 0usize;
-        if let Ok(frecency_guard) = shared_frecency.read()
-            && let Some(ref frecency) = *frecency_guard
-        {
-            for path in &paths_to_add_or_modify {
-                // Skip if this file was tracked less than 5 minutes ago
-                let should_track = match frecency.seconds_since_last_access(path) {
-                    Ok(Some(secs)) => secs >= AI_MODE_COOLDOWN_SECS,
-                    Ok(None) => true, // Never tracked before
-                    Err(_) => true,   // DB error, track anyway
-                };
-                if !should_track {
-                    continue;
-                }
-
-                if let Err(e) = frecency.track_access(path) {
-                    error!("Failed to track frecency for {:?}: {:?}", path, e);
-                } else {
-                    tracked_count += 1;
-                }
-            }
-            if tracked_count > 0 {
-                info!("AI mode: tracked frecency for {} files", tracked_count);
-            }
-        }
-
-        // Update in-memory frecency scores for tracked files
-        if tracked_count > 0
-            && let Ok(mut picker_guard) = shared_picker.write()
-            && let Some(ref mut picker) = *picker_guard
-            && let Ok(frecency_guard) = shared_frecency.read()
-            && let Some(ref frecency) = *frecency_guard
-        {
-            for path in &paths_to_add_or_modify {
-                let _ = picker.update_single_file_frecency(path, frecency);
-            }
-        }
-    }
-
-    // Git status updates require a repository.
-    let Some(repo) = repo.as_ref() else {
-        debug!("No git repo available, skipping git status updates");
-        return new_dirs_to_watch;
-    };
-
-    if need_full_git_rescan {
-        info!("Triggering full git rescan");
-
-        let result = shared_picker.refresh_git_status(shared_frecency);
-        if let Err(e) = result {
-            error!("Failed to refresh git status: {:?}", e);
-        }
-        return new_dirs_to_watch;
-    }
-
-    if !files_to_update_git_status.is_empty() {
-        info!(
-            "Fetching git status for {} files",
-            files_to_update_git_status.len()
-        );
-
-        let status = match GitStatusCache::git_status_for_paths(repo, &files_to_update_git_status) {
-            Ok(status) => status,
-            Err(e) => {
-                tracing::error!(?e, "Failed to query git status");
-                return new_dirs_to_watch;
-            }
-        };
-
-        if let Ok(mut guard) = shared_picker.write()
-            && let Some(ref mut picker) = *guard
-        {
-            if let Err(e) = picker.update_git_statuses(status, shared_frecency) {
-                error!("Failed to update git statuses: {:?}", e);
-            } else {
-                info!("Successfully updated git statuses in picker");
-            }
-        } else {
-            error!("Failed to acquire picker lock for git status update");
-        }
-    }
-
-    new_dirs_to_watch
-}
-
-fn trigger_full_rescan(shared_picker: &SharedPicker, shared_frecency: &SharedFrecency) {
-    info!("Triggering full filesystem rescan");
-
-    // Note: no need to clear mmaps — they are backed by the kernel page cache
-    // and automatically reflect file changes. Old FileItems (and their mmaps)
-    // are dropped when the picker rebuilds its file list.
-
-    let Ok(mut guard) = shared_picker.write() else {
-        error!("Failed to acquire file picker write lock for full rescan");
-        return;
-    };
-    let Some(ref mut picker) = *guard else {
-        error!("File picker not initialized, cannot trigger rescan");
-        return;
-    };
-    if let Err(e) = picker.trigger_rescan(shared_frecency) {
-        error!("Failed to trigger full rescan: {:?}", e);
-        return;
-    }
-    info!("Full filesystem rescan completed successfully");
-
-    // Spawn background warmup + bigram rebuild (mirrors the initial scan's
-    // post-scan phase). The write lock is still held here but the spawned
-    // thread re-acquires it later — safe because the guard drops at function end.
-    // NOTE: must NOT call shared_picker.need_complex_rebuild() here — that would
-    // try to read-lock the same RwLock we already hold as write, causing a deadlock.
-    if picker.need_enable_mmap_cache() || picker.need_enable_content_indexing() {
-        picker.spawn_post_rescan_rebuild(shared_picker.clone());
-    }
-}
-
-/// After registering a watch on a newly created directory, list its
-/// immediate children and add any files to the picker.
-fn inject_existing_files(dir: &Path, shared_picker: &SharedPicker, git_workdir: &Option<PathBuf>) {
-    let Ok(entries) = std::fs::read_dir(dir) else {
-        return;
-    };
-
-    let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok());
-    let mut files_to_add = Vec::new();
-
-    for entry in entries.flatten() {
-        if entry.file_type().is_ok_and(|ft| ft.is_file()) {
-            let path = entry.path();
-            if should_include_file(&path, &repo) {
-                files_to_add.push(path);
-            }
-        }
-    }
-
-    if files_to_add.is_empty() {
-        return;
-    }
-
-    let Ok(mut guard) = shared_picker.write() else {
-        return;
-    };
-    let Some(ref mut picker) = *guard else {
-        return;
-    };
-
-    for path in &files_to_add {
-        picker.on_create_or_modify(path);
-    }
-
-    debug!(
-        "Injected {} existing files from new directory {}",
-        files_to_add.len(),
-        dir.display(),
-    );
-}
-
-fn should_include_file(path: &Path, repo: &Option<Repository>) -> bool {
-    // Directories are not indexed — only regular files (and symlinks to files).
-    if path.is_dir() {
-        return false;
-    }
-
-    match repo.as_ref() {
-        Some(repo) => repo.is_path_ignored(path) != Ok(true),
-        None => {
-            // No git repo — apply basic sanity filters.
-            // Hidden directories are skipped by the watcher setup (hidden(true)),
-            // but events can still arrive for files in known non-code directories.
-            !is_non_code_directory(path)
-        }
-    }
-}
-
-fn is_non_code_directory(path: &Path) -> bool {
-    crate::ignore::is_non_code_directory(path)
-}
-
-#[inline]
-fn is_path_ignored(path: &Path, repo: &Option<Repository>) -> bool {
-    match repo.as_ref() {
-        Some(repo) => repo.is_path_ignored(path) == Ok(true),
-        None => is_non_code_directory(path),
-    }
-}
-
-#[inline]
-fn is_git_file(path: &Path) -> bool {
-    path.components()
-        .any(|component| component.as_os_str() == ".git")
-}
-
-fn is_dotgit_change_affecting_status(changed: &Path, repo: &Option<Repository>) -> bool {
-    let Some(repo) = repo.as_ref() else {
-        return false;
-    };
-
-    let git_dir = repo.path();
-
-    if let Ok(path_in_git_dir) = changed.strip_prefix(git_dir) {
-        // Only react to changes that rewrite the worktree state: commits,
-        // staging, checkouts, merges, conflict resolution. Ref-only updates
-        // under refs/ (fetch, push, tag writes, pack-refs) do not change
-        // which files are modified/untracked, so we deliberately skip them —
-        // watching refs/ recursively would cost one inotify watch per ref
-        // namespace on repos with many branches/remotes.
-        if path_in_git_dir == Path::new("index") || path_in_git_dir == Path::new("index.lock") {
-            return true;
-        }
-        if path_in_git_dir == Path::new("HEAD") {
-            return true;
-        }
-        if path_in_git_dir == Path::new("info/exclude")
-            || path_in_git_dir == Path::new("info/sparse-checkout")
-        {
-            return true;
-        }
-
-        if let Some(fname) = path_in_git_dir.file_name().and_then(|f| f.to_str())
-            && matches!(fname, "MERGE_HEAD" | "CHERRY_PICK_HEAD" | "REVERT_HEAD")
-        {
-            return true;
-        }
-    }
-
-    false
-}
-
-fn is_ignore_definition_path(path: &Path) -> bool {
-    matches!(
-        path.file_name().and_then(|f| f.to_str()),
-        Some(".ignore") | Some(".gitignore")
-    )
-}
-
-fn watch_git_status_paths(debouncer: &mut Debouncer, git_workdir: Option<&PathBuf>) {
-    let Some(workdir) = git_workdir else {
-        return;
-    };
-
-    let git_dir = workdir.join(".git");
-    if !git_dir.is_dir() {
-        return;
-    }
-
-    // Watch .git/ non-recursively to catch top-level files:
-    // index, index.lock, HEAD, MERGE_HEAD, CHERRY_PICK_HEAD, REVERT_HEAD.
-    // We intentionally do NOT watch refs/ — individual ref updates don't
-    // affect worktree status, and a recursive watch there blows up inotify
-    // watch counts on repos with many branches/remotes/tags.
-    if let Err(e) = debouncer.watch(&git_dir, RecursiveMode::NonRecursive) {
-        warn!("Failed to watch .git directory: {}", e);
-        return;
-    }
-
-    // Watch info/ non-recursively for exclude and sparse-checkout
-    let info_dir = git_dir.join("info");
-    if info_dir.is_dir()
-        && let Err(e) = debouncer.watch(&info_dir, RecursiveMode::NonRecursive)
-    {
-        warn!("Failed to watch .git/info: {}", e);
-    }
-}

crates/fff-core/src/bigram_filter.rs

@@ -1,527 +0,0 @@
-use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering};
-
-use ahash::AHashMap;
-
-/// Maximum number of distinct bigrams tracked in the inverted index.
-/// 95 printable ASCII chars (32..=126) after lowercasing → ~70 distinct → 4900 possible.
-/// We cap at 5000 to cover all printable bigrams with margin.
-/// 5000 columns × 62.5KB (500k files) = 305MB. For 50k files: 30MB.
-const MAX_BIGRAM_COLUMNS: usize = 5000;
-
-/// Sentinel value: bigram has no allocated column.
-const NO_COLUMN: u16 = u16::MAX;
-
-/// Temporary sync dense builder for the bigram index.
-/// Builds from the many threads reading file contents in parallel
-pub struct BigramIndexBuilder {
-    // we use lookup as atomics only in the builder because it is filled by the rayon threads
-    // the actual index uses pure u16 for the allocations
-    lookup: Vec<AtomicU16>,
-    /// Per-column bitset data, lazily allocated via OnceLock.
-    col_data: Vec<AtomicU64>,
-    next_column: AtomicU16,
-    words: usize,
-    file_count: usize,
-    populated: AtomicUsize,
-}
-
-impl BigramIndexBuilder {
-    pub fn new(file_count: usize) -> Self {
-        let words = file_count.div_ceil(64);
-        let mut lookup = Vec::with_capacity(65536);
-        lookup.resize_with(65536, || AtomicU16::new(NO_COLUMN));
-        let mut col_data = Vec::with_capacity(MAX_BIGRAM_COLUMNS * words);
-        col_data.resize_with(MAX_BIGRAM_COLUMNS * words, || AtomicU64::new(0));
-        Self {
-            lookup,
-            col_data,
-            next_column: AtomicU16::new(0),
-            words,
-            file_count,
-            populated: AtomicUsize::new(0),
-        }
-    }
-
-    #[inline]
-    fn get_or_alloc_column(&self, key: u16) -> u16 {
-        let current = self.lookup[key as usize].load(Ordering::Relaxed);
-        if current != NO_COLUMN {
-            return current;
-        }
-        let new_col = self.next_column.fetch_add(1, Ordering::Relaxed);
-        if new_col >= MAX_BIGRAM_COLUMNS as u16 {
-            return NO_COLUMN;
-        }
-
-        match self.lookup[key as usize].compare_exchange(
-            NO_COLUMN,
-            new_col,
-            Ordering::Relaxed,
-            Ordering::Relaxed,
-        ) {
-            Ok(_) => new_col,
-            Err(existing) => existing,
-        }
-    }
-
-    #[inline]
-    fn column_bitset(&self, col: u16) -> &[AtomicU64] {
-        let start = col as usize * self.words;
-        &self.col_data[start..start + self.words]
-    }
-
-    pub(crate) fn add_file_content(&self, skip_builder: &Self, file_idx: usize, content: &[u8]) {
-        if content.len() < 2 {
-            return;
-        }
-
-        debug_assert!(file_idx < self.file_count);
-        let word_idx = file_idx / 64;
-        let bit_mask = 1u64 << (file_idx % 64);
-
-        // Stack-local dedup bitsets: 1024 × u64 = 8 KB each, covers all 65536 bigrams with margin
-        // have to fit in L1 cache
-        let mut seen_consec = [0u64; 1024];
-        let mut seen_skip = [0u64; 1024];
-
-        let bytes = content;
-        let len = bytes.len();
-
-        // First consecutive pair (no skip bigram possible yet).
-        let (a, b) = (bytes[0], bytes[1]);
-        if (32..=126).contains(&a) && (32..=126).contains(&b) {
-            let key = (a.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16;
-            let w = key as usize >> 6;
-            let bit = 1u64 << (key as usize & 63);
-            seen_consec[w] |= bit;
-            let col = self.get_or_alloc_column(key);
-            if col != NO_COLUMN {
-                self.column_bitset(col)[word_idx].fetch_or(bit_mask, Ordering::Relaxed);
-            }
-        }
-
-        // Main loop: consecutive (i-1, i) and skip-1 (i-2, i)
-        for i in 2..len {
-            let cur = bytes[i];
-
-            // Consecutive bigram: (bytes[i-1], bytes[i])
-            let prev = bytes[i - 1];
-            if (32..=126).contains(&prev) && (32..=126).contains(&cur) {
-                let key = (prev.to_ascii_lowercase() as u16) << 8 | cur.to_ascii_lowercase() as u16;
-                let w = key as usize >> 6;
-                let bit = 1u64 << (key as usize & 63);
-                if seen_consec[w] & bit == 0 {
-                    seen_consec[w] |= bit;
-                    let col = self.get_or_alloc_column(key);
-                    if col != NO_COLUMN {
-                        self.column_bitset(col)[word_idx].fetch_or(bit_mask, Ordering::Relaxed);
-                    }
-                }
-            }
-
-            // Skip-1 bigram: (bytes[i-2], bytes[i])
-            let skip_prev = bytes[i - 2];
-            if (32..=126).contains(&skip_prev) && (32..=126).contains(&cur) {
-                let key =
-                    (skip_prev.to_ascii_lowercase() as u16) << 8 | cur.to_ascii_lowercase() as u16;
-                let w = key as usize >> 6;
-                let bit = 1u64 << (key as usize & 63);
-                if seen_skip[w] & bit == 0 {
-                    seen_skip[w] |= bit;
-                    let col = skip_builder.get_or_alloc_column(key);
-                    if col != NO_COLUMN {
-                        skip_builder.column_bitset(col)[word_idx]
-                            .fetch_or(bit_mask, Ordering::Relaxed);
-                    }
-                }
-            }
-        }
-
-        self.populated.fetch_add(1, Ordering::Relaxed);
-        skip_builder.populated.fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn is_ready(&self) -> bool {
-        self.populated.load(Ordering::Relaxed) > 0
-    }
-
-    pub fn columns_used(&self) -> u16 {
-        self.next_column
-            .load(Ordering::Relaxed)
-            .min(MAX_BIGRAM_COLUMNS as u16)
-    }
-
-    /// Compress the dense builder into a compact `BigramFilter`.
-    ///
-    /// Retains columns where the bigram appears in ≥`min_density_pct`% (or
-    /// the default ~3.1% heuristic when `None`) and <90% of indexed files.
-    /// Sparse columns carry too little data to justify their memory;
-    /// ubiquitous columns (≥90%) are nearly all-ones and barely filter.
-    pub fn compress(self, min_density_pct: Option<u32>) -> BigramFilter {
-        let cols = self.columns_used() as usize;
-        let words = self.words;
-        let file_count = self.file_count;
-        let populated = self.populated.load(Ordering::Relaxed);
-        let dense_bytes = words * 8; // cost of one dense column
-
-        let old_lookup = self.lookup;
-        let col_data = self.col_data;
-
-        let mut lookup: Vec<u16> = vec![NO_COLUMN; 65536];
-        let mut dense_data: Vec<u64> = Vec::with_capacity(cols * words);
-        let mut dense_count: usize = 0;
-
-        for key in 0..65536usize {
-            let old_col = old_lookup[key].load(Ordering::Relaxed);
-            if old_col == NO_COLUMN || old_col as usize >= cols {
-                continue;
-            }
-
-            let col_start = old_col as usize * words;
-            let bitset = &col_data[col_start..col_start + words];
-
-            // count set bits to decide if this column is worth keeping.
-            let mut popcount = 0u32;
-            for column in bitset.iter().take(words) {
-                popcount += column.load(Ordering::Relaxed).count_ones();
-            }
-
-            // drop bigrams appearing in too few files
-            let not_to_rare = if let Some(min_pct) = min_density_pct {
-                // Percentage-based: require ≥ min_pct% of populated files.
-                populated > 0 && (popcount as usize) * 100 >= populated * min_pct as usize
-            } else {
-                // Default: popcount ≥ words × 2 (~3.1% of files).
-                (popcount as usize * 4) >= dense_bytes
-            };
-
-            if !not_to_rare {
-                continue;
-            }
-
-            // Drop ubiquitous bigrams — columns ≥90% ones carry almost no
-            // filtering power and just waste memory + AND cycles.
-            if populated > 0 && (popcount as usize) * 10 >= populated * 9 {
-                continue;
-            }
-
-            let dense_idx = dense_count as u16;
-            lookup[key] = dense_idx;
-            dense_count += 1;
-
-            for column in bitset.iter().take(words) {
-                dense_data.push(column.load(Ordering::Relaxed));
-            }
-        }
-
-        // col_data + old_lookup dropped here — single deallocation each,
-        // no fragmentation.
-
-        BigramFilter {
-            lookup,
-            dense_data,
-            dense_count,
-            words,
-            file_count,
-            populated,
-            skip_index: None,
-        }
-    }
-}
-
-unsafe impl Send for BigramIndexBuilder {}
-unsafe impl Sync for BigramIndexBuilder {}
-
-/// Inverted bigram index with optional "skip-1" extension
-/// Copmressed into bitset for minimal usage, the layout of this struct actually matters
-#[derive(Debug)]
-pub struct BigramFilter {
-    lookup: Vec<u16>,
-    /// Flat buffer of all dense column data laid out at fixed stride `words`.
-    /// Column `i` starts at `i * words`.
-    dense_data: Vec<u64>, // do not try to change this to u8 it has to be wordsize
-    dense_count: usize,
-    words: usize,
-    file_count: usize,
-    populated: usize,
-    /// Optional skip-1 bigram index (stride 2). Built from character pairs
-    /// at distance 2, e.g. "ABCDE" → (A,C),(B,D),(C,E). ANDead with the
-    /// consecutive bigram candidates during query to dramatically reduce
-    /// false positives.
-    skip_index: Option<Box<BigramFilter>>,
-}
-
-/// SIMD-friendly bitwise AND of two equal-length bitsets.
-// Auto vectorized (don't touch)
-#[inline]
-fn bitset_and(result: &mut [u64], bitset: &[u64]) {
-    result
-        .iter_mut()
-        .zip(bitset.iter())
-        .for_each(|(r, b)| *r &= *b);
-}
-
-impl BigramFilter {
-    /// AND the posting lists for all query bigrams (consecutive + skip).
-    /// Returns None if no query bigrams are tracked.
-    pub fn query(&self, pattern: &[u8]) -> Option<Vec<u64>> {
-        if pattern.len() < 2 {
-            return None;
-        }
-
-        let mut result = vec![u64::MAX; self.words];
-        if !self.file_count.is_multiple_of(64) {
-            let last = self.words - 1;
-            result[last] = (1u64 << (self.file_count % 64)) - 1;
-        }
-
-        let words = self.words;
-        let mut has_filter = false;
-
-        let mut prev = pattern[0];
-        for &b in &pattern[1..] {
-            if (32..=126).contains(&prev) && (32..=126).contains(&b) {
-                let key = (prev.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16;
-                let col = self.lookup[key as usize];
-                if col != NO_COLUMN {
-                    let offset = col as usize * words;
-                    // SAFETY: compress() guarantees offset + words <= dense_data.len()
-                    let slice = unsafe { self.dense_data.get_unchecked(offset..offset + words) };
-                    bitset_and(&mut result, slice);
-                    has_filter = true;
-                }
-            }
-            prev = b;
-        }
-
-        // strid-1 bigrams
-        if let Some(skip) = &self.skip_index
-            && pattern.len() >= 3
-            && let Some(skip_candidates) = skip.query_skip(pattern)
-        {
-            bitset_and(&mut result, &skip_candidates);
-            has_filter = true;
-        }
-
-        has_filter.then_some(result)
-    }
-
-    /// Query using stride-2 bigrams from the pattern.
-    /// For "ABCDE" queries with keys (A,C), (B,D), (C,E).
-    fn query_skip(&self, pattern: &[u8]) -> Option<Vec<u64>> {
-        let mut result = vec![u64::MAX; self.words];
-        if !self.file_count.is_multiple_of(64) {
-            let last = self.words - 1;
-            result[last] = (1u64 << (self.file_count % 64)) - 1;
-        }
-
-        let words = self.words;
-        let mut has_filter = false;
-
-        for i in 0..pattern.len().saturating_sub(2) {
-            let a = pattern[i];
-            let b = pattern[i + 2];
-            if (32..=126).contains(&a) && (32..=126).contains(&b) {
-                let key = (a.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16;
-                let col = self.lookup[key as usize];
-                if col != NO_COLUMN {
-                    let offset = col as usize * words;
-                    let slice = unsafe { self.dense_data.get_unchecked(offset..offset + words) };
-                    bitset_and(&mut result, slice);
-                    has_filter = true;
-                }
-            }
-        }
-
-        has_filter.then_some(result)
-    }
-
-    /// Attach a skip-1 bigram index for tighter candidate filtering.
-    pub fn set_skip_index(&mut self, skip: BigramFilter) {
-        self.skip_index = Some(Box::new(skip));
-    }
-
-    #[inline]
-    pub fn is_candidate(candidates: &[u64], file_idx: usize) -> bool {
-        let word = file_idx / 64;
-        let bit = file_idx % 64;
-        word < candidates.len() && candidates[word] & (1u64 << bit) != 0
-    }
-
-    pub fn count_candidates(candidates: &[u64]) -> usize {
-        candidates.iter().map(|w| w.count_ones() as usize).sum()
-    }
-
-    pub fn is_ready(&self) -> bool {
-        self.populated > 0
-    }
-
-    pub fn file_count(&self) -> usize {
-        self.file_count
-    }
-
-    pub fn columns_used(&self) -> usize {
-        self.dense_count
-    }
-
-    /// Total heap bytes used by this index (lookup + dense data + skip).
-    pub fn heap_bytes(&self) -> usize {
-        let lookup_bytes = self.lookup.len() * std::mem::size_of::<u16>();
-        let dense_bytes = self.dense_data.len() * std::mem::size_of::<u64>();
-        let skip_bytes = self.skip_index.as_ref().map_or(0, |s| s.heap_bytes());
-        lookup_bytes + dense_bytes + skip_bytes
-    }
-
-    /// Check whether a bigram key is present in this index.
-    pub fn has_key(&self, key: u16) -> bool {
-        self.lookup[key as usize] != NO_COLUMN
-    }
-
-    /// Raw lookup table (65536 entries mapping bigram key → column index).
-    pub fn lookup(&self) -> &[u16] {
-        &self.lookup
-    }
-
-    /// Flat dense bitset data at fixed stride `words`.
-    pub fn dense_data(&self) -> &[u64] {
-        &self.dense_data
-    }
-
-    /// Number of u64 words per column (= ceil(file_count / 64)).
-    pub fn words(&self) -> usize {
-        self.words
-    }
-
-    /// Number of dense columns retained after compression.
-    pub fn dense_count(&self) -> usize {
-        self.dense_count
-    }
-
-    /// Number of files that contributed content to the index.
-    pub fn populated(&self) -> usize {
-        self.populated
-    }
-
-    /// Reference to the optional skip-1 bigram sub-index.
-    pub fn skip_index(&self) -> Option<&BigramFilter> {
-        self.skip_index.as_deref()
-    }
-
-    /// Create a new bigram filter from the internal data
-    pub fn new(
-        lookup: Vec<u16>,
-        dense_data: Vec<u64>,
-        dense_count: usize,
-        words: usize,
-        file_count: usize,
-        populated: usize,
-    ) -> Self {
-        Self {
-            lookup,
-            dense_data,
-            dense_count,
-            words,
-            file_count,
-            populated,
-            skip_index: None,
-        }
-    }
-}
-
-pub fn extract_bigrams(content: &[u8]) -> Vec<u16> {
-    if content.len() < 2 {
-        return Vec::new();
-    }
-    // Use a flat bitset (65536 bits = 8 KB) for dedup — faster than HashSet.
-    let mut seen = vec![0u64; 1024]; // 1024 * 64 = 65536 bits
-    let mut bigrams = Vec::new();
-
-    let mut prev = content[0];
-    for &b in &content[1..] {
-        if (32..=126).contains(&prev) && (32..=126).contains(&b) {
-            let key = (prev.to_ascii_lowercase() as u16) << 8 | b.to_ascii_lowercase() as u16;
-            let word = key as usize / 64;
-            let bit = 1u64 << (key as usize % 64);
-            if seen[word] & bit == 0 {
-                seen[word] |= bit;
-                bigrams.push(key);
-            }
-        }
-        prev = b;
-    }
-    bigrams
-}
-
-/// Modified and added files store their own bigram sets. Deleted files are
-/// tombstoned in a bitset so they can be excluded from base query results.
-/// This overlay is updated by the background watcher on every file event
-/// and cleared when the base index is rebuilt.
-#[derive(Debug)]
-pub struct BigramOverlay {
-    /// Per-file bigram sets for files modified since the base was built.
-    /// Key = file index in the base `Vec<FileItem>`.
-    modified: AHashMap<usize, Vec<u16>>,
-
-    /// Tombstone bitset — one bit per base file. Set bits are excluded
-    /// from base query results.
-    tombstones: Vec<u64>,
-
-    /// Original files count this overlay was created for.
-    base_file_count: usize,
-}
-
-impl BigramOverlay {
-    pub(crate) fn new(base_file_count: usize) -> Self {
-        let words = base_file_count.div_ceil(64);
-        Self {
-            modified: AHashMap::new(),
-            tombstones: vec![0u64; words],
-            base_file_count,
-        }
-    }
-
-    pub(crate) fn modify_file(&mut self, file_idx: usize, content: &[u8]) {
-        self.modified.insert(file_idx, extract_bigrams(content));
-    }
-
-    pub(crate) fn delete_file(&mut self, file_idx: usize) {
-        if file_idx < self.base_file_count {
-            let word = file_idx / 64;
-            self.tombstones[word] |= 1u64 << (file_idx % 64);
-        }
-        self.modified.remove(&file_idx);
-    }
-
-    /// Return base file indices of modified files whose bigrams match ALL
-    /// of the given `pattern_bigrams`.
-    pub(crate) fn query_modified(&self, pattern_bigrams: &[u16]) -> Vec<usize> {
-        if pattern_bigrams.is_empty() {
-            return self.modified.keys().copied().collect();
-        }
-        self.modified
-            .iter()
-            .filter_map(|(&file_idx, bigrams)| {
-                pattern_bigrams
-                    .iter()
-                    .all(|pb| bigrams.contains(pb))
-                    .then_some(file_idx)
-            })
-            .collect()
-    }
-
-    /// Number of base files this overlay was created for.
-    pub(crate) fn base_file_count(&self) -> usize {
-        self.base_file_count
-    }
-
-    /// Get the tombstone bitset for clearing base candidates.
-    pub(crate) fn tombstones(&self) -> &[u64] {
-        &self.tombstones
-    }
-
-    /// Get all modified file indices (for conservative overlay merging when
-    /// we can't extract precise bigrams, e.g. regex patterns).
-    pub(crate) fn modified_indices(&self) -> Vec<usize> {
-        self.modified.keys().copied().collect()
-    }
-}

crates/fff-core/src/case_insensitive_memmem.rs

@@ -1,662 +0,0 @@
-//! SIMD-accelerated case-insensitive substring search.
-//!
-//! Implementations (fastest → simplest):
-//! - `search_packed_pair`: AVX2 packed-pair scan (two rare bytes at known offsets)
-//! - `search`:             memchr2 first-byte scan + verify
-//!
-//! The packed-pair approach mirrors what `memchr::memmem` does internally for
-//! case-sensitive search — pick two rare bytes from the needle, SIMD-scan for
-//! both simultaneously, verify candidates.  This gives quadratic selectivity
-//! over the single-byte memchr2 approach.
-
-// this is stolen from the memchr2 crate
-const BYTE_FREQUENCIES: [u8; 256] = [
-    55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00
-    42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10
-    255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20
-    208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30
-    120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185,
-    167, // 0x40 A-O
-    186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114,
-    223, // 0x50 P-_
-    151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246,
-    244, // 0x60 a-o
-    231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127,
-    27, // 0x70 p-DEL
-    212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80
-    207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90
-    118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0
-    166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0
-    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0
-];
-
-#[inline]
-fn ascii_fold_byte(b: u8) -> u8 {
-    if b.is_ascii_uppercase() { b | 0x20 } else { b }
-}
-
-/// Toggle ASCII letter case by flipping bit 5.
-/// `'n' → 'N'`, `'N' → 'n'`.
-#[inline]
-fn ascii_swap_case(b: u8) -> u8 {
-    b ^ 0x20
-}
-
-/// Effective frequency rank for a case-insensitive byte position.
-/// Takes the max of lower/upper ranks because we must scan for both.
-#[inline]
-fn case_insensitive_rank(lower: u8) -> u8 {
-    if lower.is_ascii_lowercase() {
-        let upper = ascii_swap_case(lower);
-        BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[upper as usize])
-    } else {
-        BYTE_FREQUENCIES[lower as usize]
-    }
-}
-
-/// Pick two needle positions with the rarest bytes (case-insensitive).
-/// Returns (index1, index2) where index1 <= index2.
-fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) {
-    debug_assert!(needle_lower.len() >= 2);
-
-    let mut best1 = (u8::MAX, 0usize); // (rank, position)
-    let mut best2 = (u8::MAX, 1usize);
-
-    for (i, &b) in needle_lower.iter().enumerate() {
-        let r = case_insensitive_rank(b);
-        if r < best1.0 {
-            best2 = best1;
-            best1 = (r, i);
-        } else if r < best2.0 && i != best1.1 {
-            best2 = (r, i);
-        }
-    }
-
-    let i1 = best1.1.min(best2.1);
-    let i2 = best1.1.max(best2.1);
-    (i1, i2)
-}
-
-#[inline]
-fn verify_scalar(h: *const u8, needle_lower: &[u8]) -> bool {
-    for (i, _) in needle_lower.iter().enumerate() {
-        if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] {
-            return false;
-        }
-    }
-    true
-}
-
-/// AVX2 case-insensitive verify: checks whether `needle_lower` matches
-/// the haystack bytes starting at `h`, treating ASCII uppercase as lowercase.
-///
-/// Processes 32 bytes at a time using a SIMD trick: AVX2 only has a
-/// **signed** byte compare (`cmpgt`), but we need an **unsigned** range
-/// check (`'A' <= byte <= 'Z'`).  The trick is to XOR every byte with
-/// `0x80`, which maps the unsigned range `[0, 255]` into the signed range
-/// `[-128, 127]` while preserving order.  After the flip, signed `cmpgt`
-/// gives correct unsigned comparisons.
-///
-/// Once we know which bytes are uppercase, we set bit 5 (`0x20`) on them
-/// — this converts `'A'..'Z'` to `'a'..'z'` — then compare against the
-/// pre-lowered needle.
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn verify_avx2(h: *const u8, needle_lower: &[u8]) -> bool {
-    use core::arch::x86_64::*;
-
-    let len = needle_lower.len();
-    let mut i = 0usize;
-
-    // Broadcast constants used every iteration:
-    //
-    //   flip      = 0x80 in every lane — XOR converts unsigned→signed domain
-    //   a_minus_1 = ('A' - 1) ^ 0x80  — lower bound for the range check (signed)
-    //   z_plus_1  = ('Z' + 1) ^ 0x80  — upper bound for the range check (signed)
-    //   bit20     = 0x20 in every lane — OR this onto uppercase bytes to lowercase them
-    let flip = _mm256_set1_epi8(0x80u8 as i8);
-    let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8);
-    let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8);
-    let bit20 = _mm256_set1_epi8(0x20u8 as i8);
-
-    while i + 32 <= len {
-        // Load 32 bytes from the haystack candidate position.
-        let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) };
-        // Load 32 bytes from the pre-lowercased needle.
-        let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) };
-
-        // Flip into signed domain: x = hv ^ 0x80.
-        // After this, unsigned ordering is preserved under signed compare.
-        let x = _mm256_xor_si256(hv, flip);
-
-        // ge_a[lane] = 0xFF if x[lane] > a_minus_1, i.e. hv[lane] >= 'A' (unsigned).
-        let ge_a = _mm256_cmpgt_epi8(x, a_minus_1);
-        // le_z[lane] = 0xFF if z_plus_1 > x[lane], i.e. hv[lane] <= 'Z' (unsigned).
-        let le_z = _mm256_cmpgt_epi8(z_plus_1, x);
-        // upper[lane] = 0xFF only for bytes in the range 'A'..='Z'.
-        let upper = _mm256_and_si256(ge_a, le_z);
-
-        // Case-fold: set bit 5 on uppercase bytes → converts 'A'..'Z' to 'a'..'z'.
-        // Non-letter bytes are untouched because their `upper` lane is 0x00.
-        let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20));
-
-        // Compare the folded haystack against the lowercase needle.
-        let eq = _mm256_cmpeq_epi8(folded, nv);
-        // movemask extracts the high bit of each lane into a 32-bit mask.
-        // All-equal → all high bits set → mask == 0xFFFFFFFF == -1i32.
-        if _mm256_movemask_epi8(eq) != -1i32 {
-            return false;
-        }
-
-        i += 32;
-    }
-
-    // Scalar tail: handle remaining bytes that don't fill a full 32-byte vector.
-    while i < len {
-        if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] {
-            return false;
-        }
-        i += 1;
-    }
-    true
-}
-
-// ======== NEON + dotprod (aarch64) ===========================================
-
-/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF).
-/// Bit *i* of the result corresponds to byte *i* of the input vector.
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-#[inline]
-unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 {
-    use core::arch::aarch64::*;
-
-    // AND each byte with its bit-position mask, then horizontally sum each half.
-    // Max possible sum per half = 1+2+4+8+16+32+64+128 = 255, fits in u8.
-    static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
-    let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) };
-    let masked = vandq_u8(v, bit_mask);
-    let lo = vaddv_u8(vget_low_u8(masked));
-    let hi = vaddv_u8(vget_high_u8(masked));
-    (lo as u16) | ((hi as u16) << 8)
-}
-
-/// NEON + dotprod case-insensitive verify.
-///
-/// Uses unsigned range checks (NEON has `vcge`/`vcle` for unsigned bytes
-/// no XOR-0x80 trick needed unlike AVX2) to detect uppercase ASCII, folds
-/// to lowercase, then checks equality via UDOT: XOR the folded haystack
-/// with the pre-lowered needle and dot-product the difference with itself.
-/// Any non-zero byte produces a non-zero u32 lane.
-///
-/// The UDOT instruction is emitted via inline asm because the `vdotq_u32`
-/// intrinsic is still behind an unstable feature gate on stable Rust.
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon,dotprod")]
-unsafe fn verify_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool {
-    use core::arch::aarch64::*;
-
-    let len = needle_lower.len();
-    let mut i = 0usize;
-
-    let a_val = vdupq_n_u8(b'A');
-    let z_val = vdupq_n_u8(b'Z');
-    let bit20 = vdupq_n_u8(0x20);
-
-    while i + 16 <= len {
-        let hv = unsafe { vld1q_u8(h.add(i)) };
-        let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) };
-
-        // Unsigned range check: 'A' <= byte <= 'Z'
-        let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val));
-        // Case-fold: set bit 5 on uppercase bytes → 'A'..'Z' → 'a'..'z'
-        let folded = vorrq_u8(hv, vandq_u8(upper, bit20));
-
-        // XOR with needle — all-zero iff every byte matches.
-        let xored = veorq_u8(folded, nv);
-
-        // UDOT: dot(xored, xored) sums squares of 4 consecutive byte
-        // differences into each of the 4 u32 lanes (accumulates into zero).
-        // Any non-zero byte produces a positive u32 contribution.
-        let dots: uint32x4_t;
-        let zero = vdupq_n_u32(0);
-        unsafe {
-            core::arch::asm!(
-                "udot {d:v}.4s, {a:v}.16b, {b:v}.16b",
-                d = inlateout(vreg) zero => dots,
-                a = in(vreg) xored,
-                b = in(vreg) xored,
-            );
-        }
-
-        if vmaxvq_u32(dots) != 0 {
-            return false;
-        }
-
-        i += 16;
-    }
-
-    // Scalar tail
-    while i < len {
-        if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] {
-            return false;
-        }
-        i += 1;
-    }
-    true
-}
-
-/// NEON packed-pair kernel: scan 16 haystack positions per iteration,
-/// checking two rare bytes (case-insensitive) simultaneously.
-/// Same algorithm as the AVX2 version but with 128-bit vectors.
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-unsafe fn search_packed_pair_neon(
-    haystack: &[u8],
-    needle_lower: &[u8],
-    i1: usize,
-    i2: usize,
-) -> bool {
-    use core::arch::aarch64::*;
-
-    let n = needle_lower.len();
-    let hlen = haystack.len();
-    let ptr = haystack.as_ptr();
-    let last_start = hlen - n;
-
-    let b1 = needle_lower[i1];
-    let b1_alt = if b1.is_ascii_lowercase() {
-        ascii_swap_case(b1)
-    } else {
-        b1
-    };
-    let b2 = needle_lower[i2];
-    let b2_alt = if b2.is_ascii_lowercase() {
-        ascii_swap_case(b2)
-    } else {
-        b2
-    };
-
-    let v1_lo = vdupq_n_u8(b1);
-    let v1_hi = vdupq_n_u8(b1_alt);
-    let v2_lo = vdupq_n_u8(b2);
-    let v2_hi = vdupq_n_u8(b2_alt);
-
-    let max_idx = i1.max(i2);
-    let max_offset = hlen.saturating_sub(max_idx + 16);
-    let mut offset = 0usize;
-
-    while offset <= max_offset {
-        let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) };
-        let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) };
-
-        // Case-insensitive match: OR both case variants, then AND the two positions.
-        let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi));
-        let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi));
-
-        let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) };
-
-        while mask != 0 {
-            let bit = mask.trailing_zeros() as usize;
-            let candidate = offset + bit;
-            if candidate > last_start {
-                return false;
-            }
-            if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                return true;
-            }
-            mask &= mask - 1;
-        }
-
-        offset += 16;
-    }
-
-    // Tail: remaining positions that couldn't fill a full vector.
-    if offset <= last_start {
-        let rare_pos =
-            if case_insensitive_rank(needle_lower[i1]) <= case_insensitive_rank(needle_lower[i2]) {
-                i1
-            } else {
-                i2
-            };
-        let rare_byte = needle_lower[rare_pos];
-        let tail_start = offset + rare_pos;
-        let tail_end = last_start + rare_pos + 1;
-        if tail_start < tail_end {
-            let tail_space = &haystack[tail_start..tail_end];
-            if rare_byte.is_ascii_lowercase() {
-                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            } else {
-                for pos in memchr::memchr_iter(rare_byte, tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            }
-        }
-    }
-
-    false
-}
-
-#[inline]
-unsafe fn verify_dispatch(h: *const u8, needle_lower: &[u8]) -> bool {
-    #[cfg(target_arch = "x86_64")]
-    {
-        if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") {
-            return unsafe { verify_avx2(h, needle_lower) };
-        }
-    }
-    #[cfg(target_arch = "aarch64")]
-    {
-        if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") {
-            return unsafe { verify_neon_dotprod(h, needle_lower) };
-        }
-    }
-
-    verify_scalar(h, needle_lower)
-}
-
-// ── Packed-pair search (AVX2) ───────────────────────────────────────────
-
-/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration,
-/// checking two rare bytes (case-insensitive) simultaneously.
-/// 4 cmpeq + 2 or + 1 and + 1 movemask per 32 bytes — same memory
-/// bandwidth as memchr2 but quadratic selectivity.
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn search_packed_pair_avx2(
-    haystack: &[u8],
-    needle_lower: &[u8],
-    i1: usize,
-    i2: usize,
-) -> bool {
-    use core::arch::x86_64::*;
-
-    let n = needle_lower.len();
-    let hlen = haystack.len();
-    let ptr = haystack.as_ptr();
-    let last_start = hlen - n; // last valid match-start position
-
-    let b1 = needle_lower[i1];
-    let b1_alt = if b1.is_ascii_lowercase() {
-        ascii_swap_case(b1)
-    } else {
-        b1
-    };
-    let b2 = needle_lower[i2];
-    let b2_alt = if b2.is_ascii_lowercase() {
-        ascii_swap_case(b2)
-    } else {
-        b2
-    };
-
-    let v1_lo = _mm256_set1_epi8(b1 as i8);
-    let v1_hi = _mm256_set1_epi8(b1_alt as i8);
-    let v2_lo = _mm256_set1_epi8(b2 as i8);
-    let v2_hi = _mm256_set1_epi8(b2_alt as i8);
-
-    // Main loop: process 32 candidate positions per iteration.
-    // We load from ptr+offset+i1 and ptr+offset+i2, so we need
-    // offset + max(i1,i2) + 31 < hlen.
-    let max_idx = i1.max(i2);
-    let max_offset = hlen.saturating_sub(max_idx + 32);
-    let mut offset = 0usize;
-
-    while offset <= max_offset {
-        let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) };
-        let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) };
-
-        // Case-insensitive match: OR both case variants, then AND the two positions.
-        let eq1 = _mm256_or_si256(
-            _mm256_cmpeq_epi8(chunk1, v1_lo),
-            _mm256_cmpeq_epi8(chunk1, v1_hi),
-        );
-        let eq2 = _mm256_or_si256(
-            _mm256_cmpeq_epi8(chunk2, v2_lo),
-            _mm256_cmpeq_epi8(chunk2, v2_hi),
-        );
-
-        let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32;
-
-        while mask != 0 {
-            let bit = mask.trailing_zeros() as usize;
-            let candidate = offset + bit;
-            if candidate > last_start {
-                // Past the end — no more valid positions in this or future chunks.
-                return false;
-            }
-            if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                return true;
-            }
-            mask &= mask - 1;
-        }
-
-        offset += 32;
-    }
-
-    // Tail: remaining positions that couldn't fill a full vector.
-    // Use memchr2 on the rarest byte for these last few positions.
-    if offset <= last_start {
-        let rare_pos =
-            if case_insensitive_rank(needle_lower[i1]) <= case_insensitive_rank(needle_lower[i2]) {
-                i1
-            } else {
-                i2
-            };
-        let rare_byte = needle_lower[rare_pos];
-        let tail_start = offset + rare_pos;
-        let tail_end = last_start + rare_pos + 1;
-        if tail_start < tail_end {
-            let tail_space = &haystack[tail_start..tail_end];
-            if rare_byte.is_ascii_lowercase() {
-                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            } else {
-                for pos in memchr::memchr_iter(rare_byte, tail_space) {
-                    let candidate = offset + pos;
-                    if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } {
-                        return true;
-                    }
-                }
-            }
-        }
-    }
-
-    false
-}
-
-/// Packed-pair case-insensitive substring search.
-///
-/// Selects the two rarest bytes from the needle (using the memchr byte
-/// frequency heuristic), then SIMD-scans for both at their known offsets
-/// simultaneously.  Falls back to `search` for needles shorter than 2 bytes.
-pub fn search_packed_pair(haystack: &[u8], needle_lower: &[u8]) -> bool {
-    let n = needle_lower.len();
-    if n == 0 {
-        return true;
-    }
-    if n < 2 {
-        return search(haystack, needle_lower);
-    }
-    if n > haystack.len() {
-        return false;
-    }
-
-    let (i1, i2) = select_rare_pair(needle_lower);
-
-    #[cfg(target_arch = "x86_64")]
-    {
-        if std::is_x86_feature_detected!("avx2") {
-            // Need enough haystack for at least one vector load.
-            let max_idx = i1.max(i2);
-            if haystack.len() >= max_idx + 32 {
-                return unsafe { search_packed_pair_avx2(haystack, needle_lower, i1, i2) };
-            }
-        }
-    }
-
-    #[cfg(target_arch = "aarch64")]
-    {
-        // The NEON packed-pair scan checks 16 bytes/iteration with ~7 ops,
-        // while memchr's optimized loop processes more bytes with fewer ops.
-        // Packed-pair wins when the first byte is common (lots of false
-        // positives for memchr2 that we avoid). But when the first byte is
-        // rare (z, q, x, ...) memchr2 has no false positives and its raw
-        // throughput dominates. Threshold 200 on the frequency table splits
-        // common letters (s=243, e=253, f=227) from rare ones (z=152, q=139).
-        let first_byte_rank = case_insensitive_rank(needle_lower[0]);
-        let max_idx = i1.max(i2);
-        if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 {
-            return unsafe { search_packed_pair_neon(haystack, needle_lower, i1, i2) };
-        }
-    }
-
-    // Fallback for short haystacks or non-SIMD platforms.
-    search(haystack, needle_lower)
-}
-
-// ── Original memchr2 first-byte search ──────────────────────────────────
-
-/// Case-insensitive search using memchr2 on the first byte.
-pub fn search(haystack: &[u8], needle_lower: &[u8]) -> bool {
-    let n = needle_lower.len();
-    if n == 0 {
-        return true;
-    }
-    if n > haystack.len() {
-        return false;
-    }
-
-    let search_space = &haystack[..=haystack.len() - n];
-    let first = needle_lower[0];
-
-    if first.is_ascii_lowercase() {
-        let alt = ascii_swap_case(first);
-        for pos in memchr::memchr2_iter(first, alt, search_space) {
-            if unsafe { verify_dispatch(haystack.as_ptr().add(pos), needle_lower) } {
-                return true;
-            }
-        }
-    } else {
-        for pos in memchr::memchr_iter(first, search_space) {
-            if unsafe { verify_dispatch(haystack.as_ptr().add(pos), needle_lower) } {
-                return true;
-            }
-        }
-    }
-    false
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn basic_case_insensitive() {
-        assert!(search_packed_pair(b"Hello World", b"hello"));
-        assert!(search_packed_pair(b"Hello World", b"world"));
-        assert!(search_packed_pair(b"NOMORE bugs", b"nomore"));
-        assert!(!search_packed_pair(b"Hello World", b"xyz"));
-    }
-
-    #[test]
-    fn edge_cases() {
-        assert!(search_packed_pair(b"ab", b"ab"));
-        assert!(search_packed_pair(b"AB", b"ab"));
-        assert!(!search_packed_pair(b"a", b"ab"));
-        assert!(search_packed_pair(b"anything", b""));
-        assert!(!search_packed_pair(b"", b"x"));
-    }
-
-    #[test]
-    fn packed_pair_matches_search() {
-        let haystacks: &[&[u8]] = &[
-            b"The quick brown fox jumps over the lazy dog",
-            b"int mutex_lock(struct mutex *lock) { return 0; }",
-            b"#define NOMORE_RETRIES 5\nif (nomore) return;",
-            b"abcdefghijklmnopqrstuvwxyz",
-            b"short",
-        ];
-        let needles: &[&[u8]] = &[b"fox", b"mutex", b"nomore", b"xyz", b"the", b"short", b"qr"];
-        for h in haystacks {
-            for n in needles {
-                let lower: Vec<u8> = n.iter().map(|b| b.to_ascii_lowercase()).collect();
-                assert_eq!(
-                    search_packed_pair(h, &lower),
-                    search(h, &lower),
-                    "mismatch for haystack={:?} needle={:?}",
-                    std::str::from_utf8(h),
-                    std::str::from_utf8(n),
-                );
-            }
-        }
-    }
-
-    #[test]
-    fn long_haystack_neon_path() {
-        // Haystack > 16 bytes exercises NEON packed-pair search loop
-        let haystack =
-            b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb";
-        assert!(search_packed_pair(haystack, b"this_is_a_long_needle_test"));
-        assert!(!search_packed_pair(
-            haystack,
-            b"this_is_a_long_needle_testz"
-        ));
-
-        // Needle >= 16 bytes exercises NEON dotprod verify
-        let long_needle = b"struct mutex *lock";
-        let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }";
-        assert!(search_packed_pair(haystack2, long_needle));
-
-        // All uppercase haystack, lowercase needle
-        let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-        assert!(search_packed_pair(upper_hay, b"qrstuvwxyz0123456789a"));
-        assert!(!search_packed_pair(upper_hay, b"qrstuvwxyz01234567899"));
-
-        // Needle at very end
-        let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me";
-        assert!(search_packed_pair(end_hay, b"find_me"));
-
-        // Needle at very start
-        assert!(search_packed_pair(end_hay, b"xx"));
-
-        // 1KB haystack with needle near the end
-        let mut big = vec![b'z'; 1024];
-        big[1000..1010].copy_from_slice(b"hElLo_WoRl");
-        assert!(search_packed_pair(&big, b"hello_wo"));
-        assert!(!search_packed_pair(&big, b"hello_world"));
-    }
-
-    #[test]
-    fn rare_pair_selection() {
-        // For "nomore": n=246, o=244, m=233, o=244, r=245, e=253
-        // Rarest positions should include 'm' (pos 2, rank 233)
-        let (i1, i2) = select_rare_pair(b"nomore");
-        let ranks: Vec<u8> = b"nomore"
-            .iter()
-            .map(|&b| case_insensitive_rank(b))
-            .collect();
-        let r1 = ranks[i1];
-        let r2 = ranks[i2];
-        // Both selected ranks should be <= all other ranks
-        for (i, &r) in ranks.iter().enumerate() {
-            if i != i1 && i != i2 {
-                assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal");
-            }
-        }
-    }
-}

crates/fff-core/src/constants.rs

@@ -0,0 +1,48 @@
+/// Largest file whose full content fff will touch: the default grep read cap
+/// (`GrepSearchOptions::max_file_size`) and the content-cache mmap cap
+/// (`ContentCacheBudget::max_file_size`). Binary detection also streams up to
+/// this far so nothing grep would read is left unclassified.
+pub const MAX_FFFILE_SIZE: u64 = 10 * 1024 * 1024;
+
+/// Upper bound on a file the bigram builder will build, if the file is very large there is a
+/// big probability it will only bloat the available bigrams and will anyway pop ut from the prefilter
+pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
+
+/// Total bytes the persistent content mmap cache may hold for a small repo.
+pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
+
+/// Files below one page waste the remainder when mmapped, so the cache skips
+/// them and falls back to chunked reads. Unused on Windows (no content cache)
+#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
+pub const MMAP_THRESHOLD: u64 = 16 * 1024;
+#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
+pub const MMAP_THRESHOLD: u64 = 4 * 1024;
+
+/// Watcher overflow capacity reserved after the initial scan
+pub const MAX_OVERFLOW_FILES: usize = 1024;
+
+/// Minimum delay between watcher-initiated rescans.
+pub const RESCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// Rescan delay for large indexes.
+pub const RESCAN_MIN_INTERVAL_LARGE_INDEX: std::time::Duration =
+    std::time::Duration::from_secs(5 * 60);
+
+/// Live-file count at which [`RESCAN_MIN_INTERVAL_LARGE_INDEX`] takes over.
+pub const LARGE_INDEX_FILE_COUNT: usize = 1_000_000;
+
+/// Fresh-mmap threshold: files at or above this size get mmapped directly on
+/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
+/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this
+/// constant is gated to non-Windows targets to keep `-D unused-imports` happy.
+#[cfg(target_os = "macos")]
+pub const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024;
+#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
+pub const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024;
+
+// we do not support 32kb path limit on windows
+#[cfg(target_os = "windows")]
+pub const PATH_BUF_SIZE: usize = 4096;
+
+#[cfg(not(target_os = "windows"))]
+pub const PATH_BUF_SIZE: usize = libc::PATH_MAX as usize;

crates/fff-core/src/constraints.rs

@@ -1,717 +0,0 @@
-//! Constraint-based prefiltering for search queries.
-
-use ahash::AHashSet;
-use fff_query_parser::{Constraint, GitStatusFilter};
-use smallvec::SmallVec;
-
-use crate::git::is_modified_status;
-use crate::simd_path::ArenaPtr;
-
-/// `needle` must already be lowercase.
-#[inline]
-fn contains_ascii_ci(haystack: &str, needle: &str) -> bool {
-    let h = haystack.as_bytes();
-    let n = needle.as_bytes();
-    if n.len() > h.len() {
-        return false;
-    }
-    if n.is_empty() {
-        return true;
-    }
-    let first = n[0];
-    for i in 0..=(h.len() - n.len()) {
-        if h[i].to_ascii_lowercase() == first
-            && h[i..i + n.len()]
-                .iter()
-                .zip(n)
-                .all(|(a, b)| a.to_ascii_lowercase() == *b)
-        {
-            return true;
-        }
-    }
-    false
-}
-
-const PAR_THRESHOLD: usize = 10_000;
-
-pub(crate) trait Constrainable {
-    fn write_file_name(&self, arena: ArenaPtr, out: &mut String);
-    fn git_status(&self) -> Option<git2::Status>;
-    fn write_relative_path(&self, arena: ArenaPtr, out: &mut String);
-}
-
-/// Check if a relative path ends with the given suffix at a `/` boundary (case-insensitive).
-///
-/// Returns `true` when the path equals the suffix or the character before the suffix
-/// in the path is `/`. This ensures partial directory-name matches are rejected.
-///
-/// Examples:
-/// - `path_ends_with_suffix("libswscale/input.c", "libswscale/input.c")` → true (exact)
-/// - `path_ends_with_suffix("foo/libswscale/input.c", "libswscale/input.c")` → true (suffix)
-/// - `path_ends_with_suffix("xlibswscale/input.c", "libswscale/input.c")` → false (no boundary)
-#[inline]
-pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool {
-    let path_bytes = path.as_bytes();
-    let suffix_bytes = suffix.as_bytes();
-    if path_bytes.len() < suffix_bytes.len() {
-        return false;
-    }
-
-    let start = path.len() - suffix.len();
-
-    // Must land on a char boundary — multi-byte UTF-8 can make
-    // `path.len() - suffix.len()` land inside a char.
-    if !path.is_char_boundary(start) {
-        return false;
-    }
-
-    if !path[start..].eq_ignore_ascii_case(suffix) {
-        return false;
-    }
-
-    // Exact match, or the character before is '/'.
-    // `start` is a char boundary but `start - 1` may be inside a multi-byte
-    // char, so scan backward to find the preceding ASCII byte.
-    if start == 0 {
-        return true;
-    }
-    let mut i = start;
-    while i > 0 {
-        i -= 1;
-        // ASCII bytes (0..128) are single-byte UTF-8 code units
-        if path_bytes[i] < 128 {
-            return path_bytes[i] == b'/';
-        }
-    }
-    false
-}
-
-#[inline]
-pub fn file_has_extension(file_name: &str, ext: &str) -> bool {
-    let name_bytes = file_name.as_bytes();
-    let ext_bytes = ext.as_bytes();
-    if name_bytes.len() <= ext_bytes.len() + 1 {
-        return false;
-    }
-    let start = name_bytes.len() - ext_bytes.len() - 1;
-    // `.` is ASCII (single byte), so `start` must be a char boundary.
-    // If it lands inside a multi-byte char the extension can't match.
-    if start > 0 && !file_name.is_char_boundary(start) {
-        return false;
-    }
-    name_bytes.get(start) == Some(&b'.') && name_bytes[start + 1..].eq_ignore_ascii_case(ext_bytes)
-}
-
-/// Supports multi-segment paths like "libswscale/aarch64" (consecutive components).
-#[inline]
-pub fn path_contains_segment(path: &str, segment: &str) -> bool {
-    let path_bytes = path.as_bytes();
-    let segment_bytes = segment.as_bytes();
-    let segment_len = segment_bytes.len();
-
-    // Check segment/ at start of path
-    if path_bytes.len() > segment_len
-        && path_bytes.get(segment_len) == Some(&b'/')
-        && path.is_char_boundary(segment_len)
-        && path_bytes[..segment_len].eq_ignore_ascii_case(segment_bytes)
-    {
-        return true;
-    }
-
-    // Check /segment/ anywhere using byte scanning
-    if path_bytes.len() < segment_len + 2 {
-        return false;
-    }
-
-    for i in 0..path_bytes.len().saturating_sub(segment_len + 1) {
-        if path_bytes[i] == b'/' {
-            let start = i + 1;
-            let end = start + segment_len;
-            if end < path_bytes.len()
-                && path_bytes[end] == b'/'
-                && path.is_char_boundary(start)
-                && path.is_char_boundary(end)
-                && path_bytes[start..end].eq_ignore_ascii_case(segment_bytes)
-            {
-                return true;
-            }
-        }
-    }
-    false
-}
-
-#[inline]
-#[allow(clippy::too_many_arguments)]
-fn item_matches_constraint_at_index<T: Constrainable>(
-    item: &T,
-    item_index: usize,
-    constraint: &Constraint<'_>,
-    glob_results: &[(bool, AHashSet<usize>)],
-    glob_idx: &mut usize,
-    negate: bool,
-    arena: ArenaPtr,
-    fname_buf: &mut String,
-    path_buf: &mut String,
-) -> bool {
-    let matches = match constraint {
-        Constraint::Extension(ext) => {
-            item.write_file_name(arena, fname_buf);
-            file_has_extension(fname_buf, ext)
-        }
-        Constraint::Glob(_) => {
-            let result = glob_results
-                .get(*glob_idx)
-                .map(|(is_neg, set)| {
-                    let matched = set.contains(&item_index);
-
-                    if *is_neg { !matched } else { matched }
-                })
-                .unwrap_or(true);
-            *glob_idx += 1;
-            return if negate { !result } else { result };
-        }
-        Constraint::PathSegment(segment) => {
-            item.write_relative_path(arena, path_buf);
-            path_contains_segment(path_buf, segment)
-        }
-        Constraint::FilePath(suffix) => {
-            item.write_relative_path(arena, path_buf);
-            path_ends_with_suffix(path_buf, suffix)
-        }
-        Constraint::GitStatus(status_filter) => match (item.git_status(), status_filter) {
-            (Some(status), GitStatusFilter::Modified) => is_modified_status(status),
-            (Some(status), GitStatusFilter::Untracked) => status.contains(git2::Status::WT_NEW),
-            (Some(status), GitStatusFilter::Staged) => status.intersects(
-                git2::Status::INDEX_NEW
-                    | git2::Status::INDEX_MODIFIED
-                    | git2::Status::INDEX_DELETED
-                    | git2::Status::INDEX_RENAMED
-                    | git2::Status::INDEX_TYPECHANGE,
-            ),
-            (Some(status), GitStatusFilter::Unmodified) => status.is_empty(),
-            (None, GitStatusFilter::Unmodified) => true,
-            (None, _) => false,
-        },
-        Constraint::Not(inner) => {
-            return item_matches_constraint_at_index(
-                item,
-                item_index,
-                inner,
-                glob_results,
-                glob_idx,
-                !negate,
-                arena,
-                fname_buf,
-                path_buf,
-            );
-        }
-
-        // only works with negation
-        Constraint::Text(text) => {
-            item.write_relative_path(arena, path_buf);
-            contains_ascii_ci(path_buf, text)
-        }
-
-        // Parts and Exclude are handled at a higher level
-        Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true,
-    };
-
-    if negate { !matches } else { matches }
-}
-
-/// Returns `None` if no constraints are present, `Some(filtered)` otherwise.
-/// Extension constraints use OR logic; all others use AND.
-pub(crate) fn apply_constraints<'a, T: Constrainable + Sync>(
-    items: &'a [T],
-    constraints: &[Constraint<'_>],
-    arena: ArenaPtr,
-) -> Option<Vec<&'a T>> {
-    if constraints.is_empty() {
-        return None;
-    }
-
-    // Separate extension constraints from other constraints — they use OR logic
-    let mut extensions: SmallVec<[&str; 8]> = SmallVec::new();
-    let mut other_constraints: SmallVec<[&Constraint<'_>; 8]> = SmallVec::new();
-
-    for constraint in constraints {
-        match constraint {
-            Constraint::Extension(ext) => extensions.push(ext),
-            _ => other_constraints.push(constraint),
-        }
-    }
-
-    // Only collect paths if we have glob constraints (expensive)
-    let has_globs = other_constraints
-        .iter()
-        .any(|c| matches!(c, Constraint::Glob(_) | Constraint::Not(_)));
-
-    let glob_results = if has_globs {
-        // Build a single contiguous buffer of all relative paths + offset table.
-        // One allocation for the buffer, one for offsets — NOT one String per file.
-        let mut path_buf = Vec::<u8>::new();
-        let mut offsets = Vec::<(usize, usize)>::with_capacity(items.len());
-        let mut tmp = String::with_capacity(64);
-        for item in items.iter() {
-            let start = path_buf.len();
-            item.write_relative_path(arena, &mut tmp);
-            path_buf.extend_from_slice(tmp.as_bytes());
-            offsets.push((start, path_buf.len() - start));
-        }
-        let path_refs: Vec<&str> = offsets
-            .iter()
-            .map(|&(off, len)| unsafe { std::str::from_utf8_unchecked(&path_buf[off..off + len]) })
-            .collect();
-        precompute_glob_matches(&other_constraints, &path_refs)
-    } else {
-        Vec::new()
-    };
-
-    let filtered: Vec<&T> = if items.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        items
-            .par_iter()
-            .enumerate()
-            .map_init(
-                || (String::with_capacity(64), String::with_capacity(64)),
-                |(fname_buf, path_buf), (i, item)| {
-                    if !extensions.is_empty() {
-                        item.write_file_name(arena, fname_buf);
-                        if !extensions
-                            .iter()
-                            .any(|ext| file_has_extension(fname_buf, ext))
-                        {
-                            return None;
-                        }
-                    }
-
-                    let mut glob_idx = 0;
-                    if other_constraints.iter().all(|constraint| {
-                        item_matches_constraint_at_index(
-                            item,
-                            i,
-                            constraint,
-                            &glob_results,
-                            &mut glob_idx,
-                            false,
-                            arena,
-                            fname_buf,
-                            path_buf,
-                        )
-                    }) {
-                        Some(item)
-                    } else {
-                        None
-                    }
-                },
-            )
-            .flatten()
-            .collect()
-    } else {
-        let mut fname_buf = String::with_capacity(64);
-        let mut path_buf = String::with_capacity(64);
-
-        items
-            .iter()
-            .enumerate()
-            .filter(|&(i, item)| {
-                if !extensions.is_empty() {
-                    item.write_file_name(arena, &mut fname_buf);
-                    if !extensions
-                        .iter()
-                        .any(|ext| file_has_extension(&fname_buf, ext))
-                    {
-                        return false;
-                    }
-                }
-
-                let mut glob_idx = 0;
-                other_constraints.iter().all(|constraint| {
-                    item_matches_constraint_at_index(
-                        item,
-                        i,
-                        constraint,
-                        &glob_results,
-                        &mut glob_idx,
-                        false,
-                        arena,
-                        &mut fname_buf,
-                        &mut path_buf,
-                    )
-                })
-            })
-            .map(|(_, item)| item)
-            .collect()
-    };
-
-    Some(filtered)
-}
-
-fn precompute_glob_matches<'a>(
-    constraints: &[&Constraint<'a>],
-    paths: &[&str],
-) -> Vec<(bool, AHashSet<usize>)> {
-    let mut results = Vec::new();
-    for constraint in constraints {
-        collect_glob_indices(constraint, paths, &mut results, false);
-    }
-    results
-}
-
-fn collect_glob_indices<'a>(
-    constraint: &Constraint<'a>,
-    paths: &[&str],
-    results: &mut Vec<(bool, AHashSet<usize>)>,
-    _is_negated: bool,
-) {
-    match constraint {
-        Constraint::Glob(pattern) => {
-            let indices = match_glob_pattern(pattern, paths);
-            // Negation is handled by the `negate` parameter in
-            // `item_matches_constraint_at_index`, NOT here.  Storing
-            // `is_negated=true` caused a double-negation bug when the
-            // Glob arm also applied `negate`.
-            results.push((false, indices));
-        }
-        Constraint::Not(inner) => {
-            collect_glob_indices(inner, paths, results, true);
-        }
-        _ => {}
-    }
-}
-
-/// Match a glob pattern against a list of paths, returning the set of matching indices.
-///
-/// When the `zlob` feature is enabled, delegates to `zlob::zlob_match_paths` (Zig-compiled
-/// C library, fastest). Otherwise falls back to `globset::Glob` (pure Rust).
-#[cfg(feature = "zlob")]
-fn match_glob_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
-    let Ok(Some(matches)) = zlob::zlob_match_paths(pattern, paths, zlob::ZlobFlags::RECOMMENDED)
-    else {
-        return AHashSet::new();
-    };
-
-    let matched_set: AHashSet<usize> = matches.iter().map(|s| s.as_ptr() as usize).collect();
-
-    if paths.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        paths
-            .par_iter()
-            .enumerate()
-            .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
-            .map(|(i, _)| i)
-            .collect::<Vec<_>>()
-            .into_iter()
-            .collect()
-    } else {
-        paths
-            .iter()
-            .enumerate()
-            .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
-            .map(|(i, _)| i)
-            .collect()
-    }
-}
-
-#[cfg(not(feature = "zlob"))]
-fn match_glob_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
-    let Ok(glob) = globset::Glob::new(pattern) else {
-        return AHashSet::new();
-    };
-    let matcher = glob.compile_matcher();
-
-    if paths.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        paths
-            .par_iter()
-            .enumerate()
-            .filter(|(_, p)| matcher.is_match(p))
-            .map(|(i, _)| i)
-            .collect::<Vec<_>>()
-            .into_iter()
-            .collect()
-    } else {
-        paths
-            .iter()
-            .enumerate()
-            .filter(|(_, p)| matcher.is_match(p))
-            .map(|(i, _)| i)
-            .collect()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[derive(Clone)]
-    struct TestItem {
-        relative_path: &'static str,
-        file_name: &'static str,
-    }
-
-    impl Constrainable for TestItem {
-        fn write_file_name(&self, _arena: ArenaPtr, out: &mut String) {
-            out.clear();
-            out.push_str(self.file_name);
-        }
-
-        fn write_relative_path(&self, _arena: ArenaPtr, out: &mut String) {
-            out.clear();
-            out.push_str(self.relative_path);
-        }
-
-        fn git_status(&self) -> Option<git2::Status> {
-            None
-        }
-    }
-
-    #[test]
-    fn test_file_has_extension() {
-        assert!(file_has_extension("file.rs", "rs"));
-        assert!(file_has_extension("file.RS", "rs")); // case-insensitive
-        assert!(file_has_extension("file.test.rs", "rs"));
-        assert!(file_has_extension("a.rs", "rs"));
-
-        assert!(!file_has_extension("file.tsx", "rs"));
-        assert!(!file_has_extension("rs", "rs")); // too short
-        assert!(!file_has_extension(".rs", "rs")); // just extension
-        assert!(!file_has_extension("file.rsx", "rs")); // different extension
-        assert!(!file_has_extension("filers", "rs")); // no dot
-    }
-
-    #[test]
-    fn test_path_contains_segment() {
-        // Segment at start
-        assert!(path_contains_segment("src/lib.rs", "src"));
-        assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive
-
-        // Segment in middle
-        assert!(path_contains_segment("app/src/lib.rs", "src"));
-        assert!(path_contains_segment("app/SRC/lib.rs", "src"));
-
-        // Multiple levels
-        assert!(path_contains_segment("core/workflow/src/main.rs", "src"));
-        assert!(path_contains_segment(
-            "core/workflow/src/main.rs",
-            "workflow"
-        ));
-        assert!(path_contains_segment("core/workflow/src/main.rs", "core"));
-
-        // Should not match partial segments
-        assert!(!path_contains_segment("source/lib.rs", "src"));
-        assert!(!path_contains_segment("mysrc/lib.rs", "src"));
-
-        // Should not match filename
-        assert!(!path_contains_segment("lib/src", "src"));
-
-        // Multi-segment constraints
-        assert!(path_contains_segment(
-            "libswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "foo/libswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "foo/LibSwscale/AArch64/input.S",
-            "libswscale/aarch64"
-        )); // case-insensitive
-        assert!(!path_contains_segment(
-            "xlibswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        )); // partial match at start
-        assert!(!path_contains_segment(
-            "foo/libswscale/aarch64x/input.S",
-            "libswscale/aarch64"
-        )); // partial match at end
-        assert!(path_contains_segment(
-            "crates/fff-core/src/grep.rs",
-            "fff-core/src"
-        ));
-
-        // Edge cases
-        assert!(!path_contains_segment("", "src"));
-        assert!(!path_contains_segment("src", "src")); // no trailing slash
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix() {
-        // Exact match
-        assert!(path_ends_with_suffix(
-            "libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Suffix match at / boundary
-        assert!(path_ends_with_suffix(
-            "foo/libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Deep nesting
-        assert!(path_ends_with_suffix(
-            "a/b/c/libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // No boundary — partial directory name
-        assert!(!path_ends_with_suffix(
-            "xlibswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Case insensitive
-        assert!(path_ends_with_suffix(
-            "foo/LibSwscale/Input.C",
-            "libswscale/input.c"
-        ));
-
-        // Single file name
-        assert!(path_ends_with_suffix("input.c", "input.c"));
-        assert!(!path_ends_with_suffix("xinput.c", "input.c"));
-
-        // Suffix longer than path
-        assert!(!path_ends_with_suffix("input.c", "foo/input.c"));
-
-        // Simple path
-        assert!(path_ends_with_suffix("src/main.rs", "src/main.rs"));
-        assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs"));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix_does_not_panic_on_unicode_suffix() {
-        assert!(!path_ends_with_suffix("유니코드_파일_테스트.csv", "트.c"));
-        assert!(path_ends_with_suffix(
-            "data/유니코드_파일_테스트.csv",
-            "유니코드_파일_테스트.csv"
-        ));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix_unicode_apostrophe_mismatch() {
-        assert!(!path_ends_with_suffix(
-            "dir/\u{2019}bar/file.txt",
-            "'bar/file.txt"
-        ));
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix_unicode_space_mismatch() {
-        assert!(!path_ends_with_suffix(
-            "dir/\u{202f}am/file.txt",
-            " am/file.txt"
-        ));
-    }
-
-    #[test]
-    fn test_path_contains_segment_does_not_panic_on_unicode_segment() {
-        assert!(!path_contains_segment("문서/notes.txt", "문x"));
-        assert!(path_contains_segment("프로젝트/문서/notes.txt", "문서"));
-    }
-
-    #[test]
-    fn test_path_contains_segment_unicode_no_panic() {
-        assert!(!path_contains_segment(
-            "Library/Cloud/Project\u{2019}s Folder/books.ttl",
-            "Project's Folder"
-        ));
-    }
-
-    #[test]
-    fn test_file_has_extension_unicode_no_panic() {
-        assert!(!file_has_extension("cat\u{00e9}.rs", "s"));
-    }
-
-    #[test]
-    fn test_file_has_extension_unicode_filename() {
-        assert!(file_has_extension("운영-가이드.md", "md"));
-        assert!(file_has_extension("테스트.csv", "csv"));
-        assert!(!file_has_extension("테스트.csv", "md"));
-    }
-
-    #[test]
-    fn test_apply_constraints_file_path_with_unicode_suffix() {
-        let arena_ptr = ArenaPtr(std::ptr::null());
-
-        let item = TestItem {
-            relative_path: "data/유니코드_파일_테스트.csv",
-            file_name: "유니코드_파일_테스트.csv",
-        };
-
-        let exact = [Constraint::FilePath("유니코드_파일_테스트.csv")];
-        let mismatch = [Constraint::FilePath("트.c")];
-
-        let exact_items = [item.clone()];
-        let exact_matches =
-            apply_constraints(&exact_items, &exact, arena_ptr).expect("constraints applied");
-        assert_eq!(exact_matches.len(), 1);
-
-        let mismatch_items = [item];
-        let mismatch_matches =
-            apply_constraints(&mismatch_items, &mismatch, arena_ptr).expect("constraints applied");
-        assert!(mismatch_matches.is_empty());
-    }
-
-    #[test]
-    fn test_unicode_path_no_panic_real_korean_cases() {
-        // Real Korean paths that caused panics
-        let path1 = "Downloads/(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv";
-        let path2 = "hermes-agent-lecture-materials/세부_커리큘럼_최종.csv";
-        let path3 = "projects/fastcampus-hermes-agent-curriculum/chapters/part-02-Hermes-설치-및-기본-사용/section-02-doctor로-설치-상태-검증/research/03-fix가-자동-수정하는-것과-못하는-것.md";
-
-        // These must not panic regardless of segment/suffix used
-        assert!(!path_contains_segment(path1, "작성"));
-        assert!(!path_ends_with_suffix(path1, "작성.csv"));
-        assert!(!path_contains_segment(path2, "최종"));
-        assert!(!path_ends_with_suffix(path2, "최종.csv"));
-        assert!(!path_contains_segment(path3, "수정"));
-        assert!(!path_ends_with_suffix(path3, "것.md"));
-
-        // Positive cases should still work
-        assert!(path_contains_segment(
-            path2,
-            "hermes-agent-lecture-materials"
-        ));
-        assert!(path_ends_with_suffix(
-            path1,
-            "(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv"
-        ));
-        assert!(path_ends_with_suffix(path2, "세부_커리큘럼_최종.csv"));
-    }
-
-    #[test]
-    fn test_negated_glob_excludes_matching_files() {
-        let arena_ptr = ArenaPtr(std::ptr::null());
-
-        let items = vec![
-            TestItem {
-                relative_path: "src/main.rs",
-                file_name: "main.rs",
-            },
-            TestItem {
-                relative_path: "src/lib.ts",
-                file_name: "lib.ts",
-            },
-            TestItem {
-                relative_path: "include/fff.h",
-                file_name: "fff.h",
-            },
-        ];
-
-        // Not(Glob("**/*.rs")) should exclude .rs files
-        let constraints = vec![Constraint::Not(Box::new(Constraint::Glob("**/*.rs")))];
-        let result = apply_constraints(&items, &constraints, arena_ptr).unwrap();
-        let paths: Vec<&str> = result.iter().map(|i| i.relative_path).collect();
-        assert!(
-            !paths.contains(&"src/main.rs"),
-            "rs file should be excluded"
-        );
-        assert!(paths.contains(&"src/lib.ts"), "ts file should be included");
-        assert!(
-            paths.contains(&"include/fff.h"),
-            "h file should be included"
-        );
-    }
-}

crates/fff-core/src/dbs/db_healthcheck.rs

@@ -9,16 +9,23 @@ pub struct DbHealth {
     pub disk_size: u64,
     /// Entry counts by table name
     pub entry_counts: Vec<(&'static str, u64)>,
+    /// Set to `false` if can not acquire the write lock
+    pub healthy: bool,
 }
 
 pub trait DbHealthChecker {
     fn get_env(&self) -> &heed::Env;
+    fn is_healthy(&self) -> bool;
+    /// Entries per database, each group has a static string label
     fn count_entries(&self) -> Result<Vec<(&'static str, u64)>>;
 
+    /// Health summary of the database, returns summary struct
     fn get_health(&self) -> Result<DbHealth> {
         let env = self.get_env();
 
-        let size = env.real_disk_size().map_err(crate::error::Error::EnvOpen)?;
+        let size = env
+            .real_disk_size()
+            .map_err(crate::error::Error::GenericDbError)?;
         let path = env.path().to_string_lossy().to_string();
         let entry_counts = self.count_entries()?;
 
@@ -26,6 +33,7 @@ pub trait DbHealthChecker {
             path,
             disk_size: size,
             entry_counts,
+            healthy: self.is_healthy(),
         })
     }
 }

crates/fff-core/src/dbs/env_pool.rs

@@ -0,0 +1,250 @@
+use heed::{Env, EnvOpenOptions};
+use std::collections::HashMap;
+use std::fs;
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, LazyLock, Mutex, MutexGuard, PoisonError, Weak};
+use std::thread;
+use std::time::Duration;
+
+use crate::error::{Error, Result};
+use crate::lmdb::DbHealth;
+
+pub(crate) struct EnvSpec {
+    pub label: &'static str,
+    pub map_size: usize,
+    pub max_dbs: u32,
+    pub size_cap_bytes: u64,
+}
+
+pub(crate) struct PooledEnv {
+    env: Env,
+    key: PathBuf,
+    /// lmdb's env spec label
+    label: &'static str,
+    map_size: usize,
+    max_dbs: u32,
+    health: DbHealth,
+    gc_started: AtomicBool,
+    dbi_lock: Mutex<()>,
+}
+
+impl Drop for PooledEnv {
+    fn drop(&mut self) {
+        let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+        // Only remove a dead entry: begin_exclusive_destroy may have removed ours.
+        if pool.get(&self.key).is_some_and(|w| w.strong_count() == 0) {
+            pool.remove(&self.key);
+        }
+        // heed closes the env right after this body; a concurrent reopen of the
+        // same path rides out that gap via env_closing_event in get_or_open.
+    }
+}
+
+// Cloneable handle to a process-shared LMDB env, derefs to `heed::Env`.
+#[derive(Clone)]
+pub(crate) struct SharedEnv(Arc<PooledEnv>);
+
+impl Deref for SharedEnv {
+    type Target = Env;
+    fn deref(&self) -> &Env {
+        &self.0.env
+    }
+}
+
+impl std::fmt::Debug for SharedEnv {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("SharedEnv").field(&self.0.env).finish()
+    }
+}
+
+impl SharedEnv {
+    pub(crate) fn get_or_open(db_path: &Path, spec: &EnvSpec) -> Result<Self> {
+        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+        let path = fs::canonicalize(db_path).map_err(|e| Error::EnvOpen {
+            db: spec.label,
+            source: heed::Error::Io(e),
+        })?;
+
+        let mut close_waits = 0u32;
+        let mut transient_retries = 0u32;
+
+        loop {
+            let mut open_failed = false;
+
+            {
+                let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+                if let Some(existing) = pool.get(&path).and_then(Weak::upgrade) {
+                    drop(pool);
+                    if existing.label != spec.label
+                        || existing.map_size != spec.map_size
+                        || existing.max_dbs != spec.max_dbs
+                    {
+                        return Err(Error::EnvSpecMismatch {
+                            path,
+                            open_as: existing.label,
+                            requested_as: spec.label,
+                        });
+                    }
+                    return Ok(Self(existing));
+                }
+
+                erase_if_oversized(&path, spec);
+                let result = unsafe {
+                    let mut opts = EnvOpenOptions::new();
+                    opts.map_size(spec.map_size);
+                    if spec.max_dbs > 0 {
+                        opts.max_dbs(spec.max_dbs);
+                    }
+                    opts.open(&path)
+                };
+
+                match result {
+                    Ok(env) => {
+                        let entry = Arc::new(PooledEnv {
+                            env,
+                            key: path.clone(),
+                            label: spec.label,
+                            map_size: spec.map_size,
+                            max_dbs: spec.max_dbs,
+                            health: DbHealth::new(),
+                            gc_started: AtomicBool::new(false),
+                            dbi_lock: Mutex::new(()),
+                        });
+                        pool.insert(path.clone(), Arc::downgrade(&entry));
+                        drop(pool);
+                        let shared = Self(entry);
+
+                        match shared.clear_stale_readers() {
+                            Ok(cleared_count) if cleared_count > 0 => {
+                                tracing::info!(
+                                    cleared_count,
+                                    db = spec.label,
+                                    "reclaimed stale LMDB reader slots at open"
+                                );
+                            }
+                            Ok(_) => {}
+                            Err(e) => {
+                                tracing::debug!("clear_stale_readers at open failed: {e}")
+                            }
+                        }
+
+                        return Ok(shared);
+                    }
+                    Err(heed::Error::EnvAlreadyOpened) => open_failed = true,
+                    // special handling cause we know this happens randomly
+                    Err(e)
+                        if is_transient_env_open_error(&e)
+                            && transient_retries < MAX_TRANSIENT_RETRIES =>
+                    {
+                        transient_retries += 1;
+                        tracing::debug!(
+                            path = %path.display(),
+                            transient_retries,
+                            error = ?e,
+                            "transient LMDB env open error, retrying"
+                        );
+                    }
+                    Err(e) => {
+                        return Err(Error::EnvOpen {
+                            db: spec.label,
+                            source: e,
+                        });
+                    }
+                }
+            }
+
+            if open_failed {
+                close_waits += 1;
+                if close_waits > MAX_CLOSE_WAITS {
+                    return Err(Error::EnvOpen {
+                        db: spec.label,
+                        source: heed::Error::EnvAlreadyOpened,
+                    });
+                }
+
+                match heed::env_closing_event(&path) {
+                    Some(event) => {
+                        event.wait_timeout(CLOSE_WAIT);
+                    }
+                    None => thread::sleep(Duration::from_millis(2)),
+                }
+            } else {
+                thread::sleep(TRANSIENT_RETRY_SLEEP);
+            }
+        }
+    }
+
+    pub(crate) fn health(&self) -> &DbHealth {
+        &self.0.health
+    }
+
+    // First caller wins: GC runs once per opened env, not once per tracker.
+    pub(crate) fn try_start_gc(&self) -> bool {
+        !self.0.gc_started.swap(true, Ordering::AcqRel)
+    }
+
+    // LMDB forbids mdb_dbi_open from concurrent txns in the same process.
+    pub(crate) fn lock_dbi_open(&self) -> MutexGuard<'_, ()> {
+        self.0
+            .dbi_lock
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+    }
+
+    pub(crate) fn destroy(&self) -> Result<Option<heed::EnvClosingEvent>> {
+        let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+        let holders = Arc::strong_count(&self.0);
+
+        if holders > 1 {
+            return Err(Error::DbInUse {
+                db: self.0.label,
+                path: self.0.key.clone(),
+                holders: holders - 1,
+            });
+        }
+
+        pool.remove(&self.0.key);
+        Ok(heed::env_closing_event(&self.0.key))
+    }
+}
+
+static POOL: LazyLock<Mutex<HashMap<PathBuf, Weak<PooledEnv>>>> = LazyLock::new(Mutex::default);
+
+const CLOSE_WAIT: Duration = Duration::from_millis(100);
+const MAX_CLOSE_WAITS: u32 = 100;
+const TRANSIENT_RETRY_SLEEP: Duration = Duration::from_millis(50);
+const MAX_TRANSIENT_RETRIES: u32 = 8;
+
+// Concurrent mdb_env_open calls on the same path can race on macOS
+// this is for some reason fixable by simple retry of the open
+fn is_transient_env_open_error(err: &heed::Error) -> bool {
+    match err {
+        heed::Error::Io(io) => matches!(
+            io.kind(),
+            std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound
+        ),
+        _ => false,
+    }
+}
+
+fn erase_if_oversized(db_path: &Path, spec: &EnvSpec) {
+    let data = db_path.join("data.mdb");
+    let Ok(meta) = fs::metadata(&data) else {
+        return;
+    };
+
+    if meta.len() <= spec.size_cap_bytes {
+        return;
+    }
+
+    tracing::error!(
+        path = %db_path.display(),
+        size = meta.len(),
+        cap = spec.size_cap_bytes,
+        "LMDB db exceeds size cap, erasing"
+    );
+    let _ = fs::remove_file(&data);
+    let _ = fs::remove_file(db_path.join("lock.mdb"));
+}

crates/fff-core/src/dbs/frecency.rs

@@ -1,21 +1,18 @@
-use crate::db_healthcheck::DbHealthChecker;
+use super::db_healthcheck::DbHealthChecker;
+use super::env_pool::SharedEnv;
 use crate::error::{Error, Result};
 use crate::file_picker::FFFMode;
 use crate::git::is_modified_status;
-use crate::shared::SharedFrecency;
-use heed::{Database, Env, EnvOpenOptions};
-use heed::{
-    EnvFlags,
-    types::{Bytes, SerdeBincode},
-};
-use std::fs;
-use std::path::PathBuf;
+use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
+use heed::Database;
+use heed::types::{Bytes, SerdeBincode};
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::{collections::VecDeque, path::Path};
 
 const DECAY_CONSTANT: f64 = 0.0693; // ln(2)/10 for 10-day half-life
 const SECONDS_PER_DAY: f64 = 86400.0;
 const MAX_HISTORY_DAYS: f64 = 30.0; // Only consider accesses within 30 days
+const MAX_TIMESTAMPS_PER_FILE: usize = 128;
 
 // AI mode: faster decay since AI sessions are shorter and more intense
 const AI_DECAY_CONSTANT: f64 = 0.231; // ln(2)/3 for 3-day half-life
@@ -23,8 +20,9 @@ const AI_MAX_HISTORY_DAYS: f64 = 7.0; // Only consider accesses within 7 days
 
 #[derive(Debug)]
 pub struct FrecencyTracker {
-    env: Env,
+    env: SharedEnv,
     db: Database<Bytes, SerdeBincode<VecDeque<u64>>>,
+    health: DbHealth,
 }
 
 const MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [
@@ -49,240 +47,113 @@ impl DbHealthChecker for FrecencyTracker {
         &self.env
     }
 
+    fn is_healthy(&self) -> bool {
+        self.health.is_healthy()
+    }
+
     fn count_entries(&self) -> Result<Vec<(&'static str, u64)>> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
-        let count = self.db.len(&rtxn).map_err(Error::DbRead)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count = self.db.len(&rtxn).map_err(|source| Error::DbRead {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok(vec![("absolute_frecency_entries", count)])
     }
 }
 
+impl LmdbStore for FrecencyTracker {
+    const LABEL: &'static str = "frecency";
+    // 10 MiB hard ceiling. Owner's db after years of use is ~560 KiB, so this
+    // leaves ~18× headroom while capping runaway growth (see GH issue #437).
+    const MAP_SIZE: usize = 10 * 1024 * 1024;
+    const MAX_DBS: u32 = 0;
+    // Nuke the db when it exceeds 8 MiB on disk — leaves a small margin under
+    // MAP_SIZE so we don't hit MDB_MAP_FULL before the open-time erase fires.
+    const SIZE_CAP_BYTES: u64 = 12 * 1024 * 1024;
+
+    fn shared_env(&self) -> &SharedEnv {
+        &self.env
+    }
+
+    fn health(&self) -> &DbHealth {
+        &self.health
+    }
+
+    fn purge_stale_data(env: &SharedEnv) -> Result<()> {
+        let (deleted, pruned) = Self::purge_stale_entries(env)?;
+        if deleted > 0 || pruned > 0 {
+            tracing::info!(deleted, pruned, "Frecency GC purged entries");
+        }
+
+        Ok(())
+    }
+}
+
 impl FrecencyTracker {
     /// Returns the on-disk path of the LMDB environment directory.
     pub fn db_path(&self) -> &Path {
         self.env.path()
     }
 
-    pub fn new(db_path: impl AsRef<Path>, use_unsafe_no_lock: bool) -> Result<Self> {
+    pub fn open(db_path: impl AsRef<Path>) -> Result<Self> {
         let db_path = db_path.as_ref();
-        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+        let (env, health) = Self::open_env(db_path)?;
 
-        let env = unsafe {
-            let mut opts = EnvOpenOptions::new();
-            opts.map_size(24 * 1024 * 1024); // 24 MiB
-            if use_unsafe_no_lock {
-                opts.flags(EnvFlags::NO_LOCK | EnvFlags::NO_SYNC | EnvFlags::NO_META_SYNC);
-            }
-            opts.open(db_path).map_err(Error::EnvOpen)?
-        };
-        env.clear_stale_readers()
-            .map_err(Error::DbClearStaleReaders)?;
-
-        // Try read-only open first — avoids blocking on the LMDB write lock
-        // when another process (Neovim, another fff-mcp) already has it.
-        // Only fall back to create_database (which needs a write txn) if the
-        // database doesn't exist yet.
-        let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
-        let maybe_db: Option<Database<Bytes, SerdeBincode<VecDeque<u64>>>> =
-            env.open_database(&rtxn, None).map_err(Error::DbOpen)?;
-
-        drop(rtxn);
-
-        let db = match maybe_db {
-            Some(db) => db,
-            None => {
-                // First time: create the database (requires write lock).
-                let mut wtxn = env.write_txn().map_err(Error::DbStartWriteTxn)?;
-                let db = env
-                    .create_database(&mut wtxn, None)
-                    .map_err(Error::DbCreate)?;
-                wtxn.commit().map_err(Error::DbCommit)?;
-                db
-            }
-        };
-
-        Ok(FrecencyTracker {
-            db,
-            env: env.clone(),
-        })
+        let db = Self::open_database_safe(&env, None)?;
+        Ok(FrecencyTracker { db, env, health })
     }
 
-    /// Spawns a background thread to purge stale frecency entries and compact the database.
-    /// Run it once in a while to purge old pages and keep DB file size reasonable.
-    ///
-    /// It's okay to not join this thread since it acquires locks for the db access
-    ///
-    /// ```
-    /// use fff_search::frecency::FrecencyTracker;
-    /// use fff_search::SharedFrecency;
-    /// let shared_frecency: SharedFrecency = Default::default();
-    /// let _ = FrecencyTracker::spawn_gc(shared_frecency, "/path/to/frecency_db".into(), true).ok();
-    /// ```
-    pub fn spawn_gc(
-        shared: SharedFrecency,
-        db_path: String,
-        use_unsafe_no_lock: bool,
-    ) -> Result<std::thread::JoinHandle<()>> {
-        Ok(std::thread::Builder::new()
-            .name("fff-frecency-gc".into())
-            .spawn(move || Self::run_frecency_gc(shared, db_path, use_unsafe_no_lock))?)
-    }
-
-    #[tracing::instrument(skip(shared), fields(db_path = %db_path))]
-    fn run_frecency_gc(shared: SharedFrecency, db_path: String, use_unsafe_no_lock: bool) {
-        let start = std::time::Instant::now();
-        let data_path = PathBuf::from(&db_path).join("data.mdb");
-
-        // Phase 1: Purge stale entries.
-        // The RwLock protects the Option<FrecencyTracker> (not the DB itself),
-        // so a read lock is sufficient — LMDB handles its own write serialization.
-        let (deleted, pruned) = {
-            let guard = match shared.read() {
-                Ok(g) => g,
-                Err(e) => {
-                    tracing::debug!("Failed to acquire read lock: {e}");
-                    return;
-                }
-            };
-            let Some(ref tracker) = *guard else {
-                return;
-            };
-            match tracker.purge_stale_entries() {
-                Ok(result) => result,
-                Err(e) => {
-                    tracing::debug!("Purge failed: {e}");
-                    return;
-                }
-            }
-        };
-
-        if deleted > 0 || pruned > 0 {
-            tracing::info!(deleted, pruned, elapsed = ?start.elapsed(), "Frecency GC purged entries");
-        }
-
-        // Compact if we purged entries OR the file has significant freelist bloat
-        let file_size = fs::metadata(&data_path).map(|m| m.len()).unwrap_or(0);
-        if deleted == 0 && pruned == 0 && file_size <= 512 * 1024 {
-            return;
-        }
-
-        // Phase 2: Manual compaction under a single write lock
-        let mut guard = match shared.write() {
-            Ok(g) => g,
-            Err(e) => {
-                tracing::debug!("Failed to acquire write lock: {e}");
-                return;
-            }
-        };
-
-        // Read all entries from current env
-        let entries: Vec<(Vec<u8>, VecDeque<u64>)> = match guard.as_ref() {
-            Some(tracker) => {
-                let rtxn = match tracker.env.read_txn() {
-                    Ok(t) => t,
-                    Err(e) => {
-                        tracing::debug!("Compaction read_txn failed: {e}");
-                        return;
-                    }
-                };
-                let iter = match tracker.db.iter(&rtxn) {
-                    Ok(i) => i,
-                    Err(e) => {
-                        tracing::debug!("Compaction iter failed: {e}");
-                        return;
-                    }
-                };
-                let mut entries = Vec::new();
-                let mut read_errors = 0u32;
-                for result in iter {
-                    match result {
-                        Ok((key, value)) => entries.push((key.to_vec(), value)),
-                        Err(_) => read_errors += 1,
-                    }
-                }
-                if read_errors > 0 {
-                    tracing::warn!(
-                        read_errors,
-                        "Skipped corrupted entries during compaction read"
-                    );
-                }
-                entries
-            }
-            None => return,
-        };
-
-        // Drop old tracker, delete files, create fresh env, write back
-        *guard = None;
-
-        let lock_path = PathBuf::from(&db_path).join("lock.mdb");
-        let _ = fs::remove_file(&data_path);
-        let _ = fs::remove_file(&lock_path);
-
-        let tracker = match FrecencyTracker::new(&db_path, use_unsafe_no_lock) {
-            Ok(t) => t,
-            Err(e) => {
-                tracing::error!("Compaction reopen failed, frecency disabled: {e}");
-                return;
-            }
-        };
-
-        let write_result = (|| -> std::result::Result<(), heed::Error> {
-            let mut wtxn = tracker.env.write_txn()?;
-            for (key, value) in &entries {
-                tracker.db.put(&mut wtxn, key.as_slice(), value)?;
-            }
-            wtxn.commit()?;
-            Ok(())
-        })();
-
-        match write_result {
-            Ok(()) => {
-                let new_size = fs::metadata(&data_path).map(|m| m.len()).unwrap_or(0);
-                *guard = Some(tracker);
-                tracing::debug!(
-                    entries = entries.len(),
-                    old_size = file_size,
-                    new_size,
-                    elapsed = ?start.elapsed(),
-                    "Frecency DB compacted"
-                );
-            }
-            Err(e) => {
-                tracing::error!("Compaction write failed, frecency data may be incomplete: {e}");
-                *guard = Some(tracker);
-            }
-        }
+    #[deprecated(
+        since = "0.7.0",
+        note = "LMDB unsafe no-lock mode is no longer supported; use `FrecencyTracker::open` instead. \
+                The `_use_unsafe_no_lock` argument is ignored."
+    )]
+    pub fn new(db_path: impl AsRef<Path>, _use_unsafe_no_lock: bool) -> Result<Self> {
+        Self::open(db_path)
     }
 
     /// Removes entries where all timestamps are older than MAX_HISTORY_DAYS,
     /// and prunes stale timestamps from entries that still have recent ones.
     /// Returns (deleted_count, pruned_count).
-    fn purge_stale_entries(&self) -> Result<(usize, usize)> {
-        let now = self.get_now();
+    fn purge_stale_entries(env: &SharedEnv) -> Result<(usize, usize)> {
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
         let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64);
 
-        // Collect entries to delete or update
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let db: Database<Bytes, SerdeBincode<VecDeque<u64>>> = Self::open_database_safe(env, None)?;
+
+        let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
+            db: Self::LABEL,
+            source,
+        })?;
         let mut to_delete: Vec<Vec<u8>> = Vec::new();
         let mut to_update: Vec<(Vec<u8>, VecDeque<u64>)> = Vec::new();
 
-        let iter = self.db.iter(&rtxn).map_err(Error::DbRead)?;
+        let iter = db.iter(&rtxn).map_err(|source| Error::DbRead {
+            db: Self::LABEL,
+            source,
+        })?;
         for result in iter {
-            let (key, accesses) = result.map_err(Error::DbRead)?;
+            let (key, accesses) = result.map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
 
-            // Timestamps are chronologically ordered (oldest at front).
-            // Find the first timestamp that is still within the retention window.
+            // Timestamps chronologically ordered (oldest at front).
             let fresh_start = accesses.iter().position(|&ts| ts >= cutoff_time);
             match fresh_start {
-                None => {
-                    // All timestamps are stale — delete the entire entry
-                    to_delete.push(key.to_vec());
-                }
-                Some(0) => {
-                    // All timestamps are fresh — nothing to do
-                }
+                None => to_delete.push(key.to_vec()),
+                Some(0) => {}
                 Some(start) => {
-                    // Some timestamps are stale — keep only the fresh ones
                     let pruned: VecDeque<u64> = accesses.iter().skip(start).copied().collect();
                     to_update.push((key.to_vec(), pruned));
                 }
@@ -294,26 +165,56 @@ impl FrecencyTracker {
             return Ok((0, 0));
         }
 
-        // Apply all changes in a single write transaction
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = env.write_txn().map_err(|source| Error::DbStartWriteTxn {
+            db: Self::LABEL,
+            source,
+        })?;
+
         for key in &to_delete {
-            self.db.delete(&mut wtxn, key).map_err(Error::DbWrite)?;
+            db.delete(&mut wtxn, key).map_err(|source| Error::DbWrite {
+                db: Self::LABEL,
+                source,
+            })?;
         }
+
         for (key, accesses) in &to_update {
-            self.db
-                .put(&mut wtxn, key, accesses)
-                .map_err(Error::DbWrite)?;
+            db.put(&mut wtxn, key, accesses)
+                .map_err(|source| Error::DbWrite {
+                    db: Self::LABEL,
+                    source,
+                })?;
         }
-        wtxn.commit().map_err(Error::DbCommit)?;
+        wtxn.commit().map_err(|source| Error::DbCommit {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok((to_delete.len(), to_update.len()))
     }
 
     fn get_accesses(&self, path: &Path) -> Result<Option<VecDeque<u64>>> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
-
         let key_hash = Self::path_to_hash_bytes(path)?;
-        self.db.get(&rtxn, &key_hash).map_err(Error::DbRead)
+
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        let result = self
+            .db
+            .get(&rtxn, &key_hash)
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        rtxn.commit().map_err(|source| Error::DbCommit {
+            db: Self::LABEL,
+            source,
+        })?;
+
+        Ok(result)
     }
 
     fn get_now(&self) -> u64 {
@@ -324,6 +225,16 @@ impl FrecencyTracker {
     }
 
     fn path_to_hash_bytes(path: &Path) -> Result<[u8; 32]> {
+        // On Windows, resolve to the canonical form (short-name/case/symlink)
+        // so the same file always hashes to one key regardless of how the
+        // caller spelled it. Falls back to the raw path when the file no
+        // longer exists (e.g. watcher delete events), so the op is never
+        // dropped. No-op on other platforms.
+        #[cfg(windows)]
+        let canonical: Option<std::path::PathBuf> = crate::path_utils::canonicalize(path).ok();
+        #[cfg(windows)]
+        let path: &Path = canonical.as_deref().unwrap_or(path);
+
         let Some(key) = path.to_str() else {
             return Err(Error::InvalidPath(path.to_path_buf()));
         };
@@ -339,16 +250,22 @@ impl FrecencyTracker {
         Ok(last.map(|ts| self.get_now().saturating_sub(ts)))
     }
 
-    pub fn track_access(&self, path: &Path) -> Result<()> {
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+    /// Number of tracked access for file path
+    pub fn access_count(&self, path: &Path) -> Result<usize> {
+        Ok(self.get_accesses(path)?.map_or(0, |a| a.len()))
+    }
 
+    pub fn track_access(&self, path: &Path) -> Result<()> {
         let key_hash = Self::path_to_hash_bytes(path)?;
         let mut accesses = self.get_accesses(path)?.unwrap_or_default();
 
         let now = self.get_now();
         let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64);
+
+        // Drop stale timestamps from the front while also enforcing the
+        // per-file cap. Reserves one slot for the `push_back` below.
         while let Some(&front_time) = accesses.front() {
-            if front_time < cutoff_time {
+            if front_time < cutoff_time || accesses.len() >= MAX_TIMESTAMPS_PER_FILE {
                 accesses.pop_front();
             } else {
                 break;
@@ -358,13 +275,43 @@ impl FrecencyTracker {
         accesses.push_back(now);
         tracing::debug!(?path, accesses = accesses.len(), "Tracking access");
 
-        self.db
-            .put(&mut wtxn, &key_hash, &accesses)
-            .map_err(Error::DbWrite)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        if let Err(e) = self.db.put(&mut wtxn, &key_hash, &accesses) {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on put");
+                tracing::error!(
+                    ?path,
+                    "Frecency DB hit MDB_MAP_FULL; dropping write — db will be \
+                     erased on next open via LmdbStore::erase_if_oversized"
+                );
+                return Ok(());
+            }
+            return Err(Error::DbWrite {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
-        wtxn.commit().map_err(Error::DbCommit)?;
-
-        Ok(())
+        wtxn.commit()
+            .inspect_err(|e| {
+                if is_map_full(e) {
+                    self.health.mark_unhealthy("MDB_MAP_FULL on commit");
+                    tracing::error!(
+                        ?path,
+                        "Frecency DB hit MDB_MAP_FULL on commit; dropping write"
+                    );
+                }
+            })
+            .map_err(|source| Error::DbCommit {
+                db: Self::LABEL,
+                source,
+            })
     }
 
     pub fn get_access_score(&self, file_path: &Path, mode: FFFMode) -> i64 {
@@ -464,6 +411,15 @@ mod tests {
     use super::*;
     use crate::file_picker::FFFMode;
 
+    // A path that doesn't exist on disk must still hash (canonicalize fails on
+    // Windows → falls back to the raw string), so watcher delete events and
+    // raced files never drop their frecency op.
+    #[test]
+    fn hashes_nonexistent_path_without_error() {
+        let missing = Path::new("/this/path/definitely/does/not/exist/frecency_test_xyz");
+        assert!(FrecencyTracker::path_to_hash_bytes(missing).is_ok());
+    }
+
     fn calculate_test_frecency_score(access_timestamps: &[u64], current_time: u64) -> i64 {
         let mut total_frecency = 0.0;
 
@@ -533,7 +489,7 @@ mod tests {
     fn test_modification_score_interpolation() {
         let temp_dir = std::env::temp_dir().join("fff_test_interpolation");
         let _ = std::fs::remove_dir_all(&temp_dir);
-        let tracker = FrecencyTracker::new(temp_dir.to_str().unwrap(), true).unwrap();
+        let tracker = FrecencyTracker::open(temp_dir.to_str().unwrap()).unwrap();
 
         let current_time = tracker.get_now();
         let git_status = Some(git2::Status::WT_MODIFIED);

crates/fff-core/src/dbs/lmdb.rs

@@ -0,0 +1,197 @@
+use heed::{Database, Env};
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::thread;
+
+use super::env_pool::{EnvSpec, SharedEnv};
+use crate::error::{Error, Result};
+
+pub(crate) fn is_map_full(err: &heed::Error) -> bool {
+    matches!(err, heed::Error::Mdb(heed::MdbError::MapFull))
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DbHealthState {
+    Pending = 0,
+    Healthy = 1,
+    Degraded = 2,
+}
+
+impl DbHealthState {
+    fn from_u8(v: u8) -> Self {
+        debug_assert!(v <= 2);
+
+        match v {
+            0 => Self::Pending,
+            1 => Self::Healthy,
+            _ => Self::Degraded,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct DbHealth(Arc<AtomicU8>);
+
+impl DbHealth {
+    pub(crate) fn new() -> Self {
+        Self(Arc::new(AtomicU8::new(DbHealthState::Pending as u8)))
+    }
+
+    pub(crate) fn is_healthy(&self) -> bool {
+        // Pending counts as unhealthy: if the GC thread never flipped to
+        // Healthy, something's wrong (deadlocked clear_stale_readers, stuck
+        // writer mutex, etc.) and we want that surfaced to the user.
+        DbHealthState::from_u8(self.0.load(Ordering::Acquire)) == DbHealthState::Healthy
+    }
+
+    pub(crate) fn mark_healthy(&self) {
+        let _ = self.0.compare_exchange(
+            DbHealthState::Pending as u8,
+            DbHealthState::Healthy as u8,
+            Ordering::AcqRel,
+            Ordering::Acquire,
+        );
+    }
+
+    pub(crate) fn mark_unhealthy(&self, reason: &'static str) {
+        let prev = self.0.swap(DbHealthState::Degraded as u8, Ordering::AcqRel);
+        if DbHealthState::from_u8(prev) != DbHealthState::Degraded {
+            tracing::error!(reason, "LMDB tracker marked unhealthy");
+        }
+    }
+}
+
+/// Spawns a background thread that is ensuring that the environment that was previously
+/// open is safe, accessible and doesn't have a corrupted lock.md file. If it does this thread will
+/// hang indefinitely but we will have the information that the database is in failure mode
+pub(crate) fn spawn_lmdb_gc<T: LmdbStore>(shared: Arc<RwLock<Option<T>>>) {
+    let thread_shared = shared.clone();
+    let spawn_result = thread::Builder::new()
+        .name("fff-lmdb-gc".into())
+        .spawn(move || {
+            // Holding a read guard blocks `destroy` / re-init's write
+            // guard until this thread finishes — natural serialization.
+            let guard = match thread_shared.read() {
+                Ok(g) => g,
+                Err(e) => {
+                    tracing::debug!("gc: read lock poisoned: {e}");
+                    return;
+                }
+            };
+            let Some(ref tracker) = *guard else {
+                return; // destroyed before we started
+            };
+            // Trackers attaching to an already-pooled env must not repeat the
+            // GC; the first opener's run flips the shared health flag.
+            if !tracker.shared_env().try_start_gc() {
+                return;
+            }
+
+            if let Err(e) = T::purge_stale_data(tracker.shared_env()) {
+                tracing::debug!("purge_stale_data failed: {e}");
+            }
+
+            tracker.health().mark_healthy();
+        });
+
+    if let Err(e) = spawn_result {
+        tracing::debug!(?e, "failed to spawn fff-lmdb-gc thread");
+        // No thread = mark healthy now so healthcheck isn't stuck Pending.
+        if let Ok(guard) = shared.read()
+            && let Some(ref tracker) = *guard
+        {
+            tracker.health().mark_healthy();
+        }
+    }
+}
+
+pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
+    /// Short label used to defferintiate different instances of this trait
+    const LABEL: &'static str;
+    /// LMDB map size in bytes. Must be a multiple of the OS page size.
+    const MAP_SIZE: usize;
+    /// Number of named sub-databases. `0` for single-db envs.
+    const MAX_DBS: u32;
+    /// Hard cap on `data.mdb` size.
+    const SIZE_CAP_BYTES: u64;
+
+    /// Borrow the pooled env handle shared by every tracker of this path.
+    fn shared_env(&self) -> &SharedEnv;
+    /// Borrow the health flag from the tracker.
+    fn health(&self) -> &DbHealth;
+
+    /// Borrow the raw heed env.
+    fn env(&self) -> &Env {
+        self.shared_env()
+    }
+
+    /// Override to purge stale rows, compact, etc. Default no-op. Runs on
+    /// the GC thread while a read lock is held against the shared handle,
+    /// so destroy / re-init naturally wait for it.
+    fn purge_stale_data(_env: &SharedEnv) -> Result<()> {
+        Ok(())
+    }
+
+    /// Open (or join) the process-shared LMDB env for `db_path`. The health
+    /// flag is per-env: the GC of the first opener flips it for everyone.
+    #[tracing::instrument]
+    fn open_env(db_path: &Path) -> Result<(SharedEnv, DbHealth)> {
+        let shared = SharedEnv::get_or_open(
+            db_path,
+            &EnvSpec {
+                label: Self::LABEL,
+                map_size: Self::MAP_SIZE,
+                max_dbs: Self::MAX_DBS,
+                size_cap_bytes: Self::SIZE_CAP_BYTES,
+            },
+        )?;
+        let health = shared.health().clone();
+        Ok((shared, health))
+    }
+
+    /// Open or create a database without blocking on the LMDB writer mutex
+    /// when the database already exists.
+    fn open_database_safe<KC, DC>(env: &SharedEnv, name: Option<&str>) -> Result<Database<KC, DC>>
+    where
+        KC: 'static,
+        DC: 'static,
+    {
+        let db = Self::LABEL;
+        // mdb_dbi_open must not run from concurrent txns in this process.
+        let _dbi_guard = env.lock_dbi_open();
+
+        let rtxn = env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn { db, source })?;
+        let maybe_db: Option<Database<KC, DC>> = env
+            .open_database(&rtxn, name)
+            .map_err(|source| Error::DbOpen { db, source })?;
+
+        // do not drop the DB here
+        rtxn.commit()
+            .map_err(|source| Error::DbCommit { db, source })?;
+
+        match maybe_db {
+            Some(handle) => Ok(handle),
+            None => {
+                // First time: create the database (requires write lock).
+                // unfortunately this CAN be deadlocking and this is what we see happens
+                // if the other part of the code is segfaulting, so the only rule to prevent this
+                // write the good code mf, okay?
+                let mut wtxn = env
+                    .write_txn()
+                    .map_err(|source| Error::DbStartWriteTxn { db, source })?;
+                let handle = env
+                    .create_database(&mut wtxn, name)
+                    .map_err(|source| Error::DbCreate { db, source })?;
+
+                wtxn.commit()
+                    .map_err(|source| Error::DbCommit { db, source })?;
+                Ok(handle)
+            }
+        }
+    }
+}

crates/fff-core/src/dbs/mod.rs

@@ -0,0 +1,11 @@
+pub(crate) mod env_pool;
+pub(crate) mod lmdb;
+
+pub mod db_healthcheck;
+pub use db_healthcheck::{DbHealth, DbHealthChecker};
+
+pub mod frecency;
+pub use frecency::*;
+
+pub mod query_tracker;
+pub use query_tracker::*;

crates/fff-core/src/dbs/query_tracker.rs

@@ -1,11 +1,11 @@
-use crate::db_healthcheck::DbHealthChecker;
+use super::db_healthcheck::DbHealthChecker;
+use super::env_pool::SharedEnv;
 use crate::error::Error;
-use heed::types::Bytes;
-use heed::{Database, Env, EnvOpenOptions};
-use heed::{EnvFlags, types::SerdeBincode};
+use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
+use heed::types::{Bytes, SerdeBincode};
+use heed::{Database, Env};
 use serde::{Deserialize, Serialize};
 use std::collections::VecDeque;
-use std::fs;
 use std::path::{Path, PathBuf};
 use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -28,13 +28,14 @@ struct HistoryEntry {
 
 #[derive(Debug)]
 pub struct QueryTracker {
-    env: Env,
+    env: SharedEnv,
     // Database for (project_path, query) -> QueryMatchEntry mappings
     query_file_db: Database<Bytes, SerdeBincode<QueryMatchEntry>>,
     // Database for project_path -> VecDeque<HistoryEntry> mappings (file picker)
     query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
     // Database for project_path -> VecDeque<HistoryEntry> mappings (grep)
     grep_query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
+    health: DbHealth,
 }
 
 impl DbHealthChecker for QueryTracker {
@@ -42,15 +43,40 @@ impl DbHealthChecker for QueryTracker {
         &self.env
     }
 
-    fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+    fn is_healthy(&self) -> bool {
+        self.health.is_healthy()
+    }
 
-        let count_queries = self.query_file_db.len(&rtxn).map_err(Error::DbRead)?;
-        let count_histories = self.query_history_db.len(&rtxn).map_err(Error::DbRead)?;
-        let count_grep_histories = self
-            .grep_query_history_db
+    fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+
+        let count_queries = self
+            .query_file_db
             .len(&rtxn)
-            .map_err(Error::DbRead)?;
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count_histories = self
+            .query_history_db
+            .len(&rtxn)
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count_grep_histories =
+            self.grep_query_history_db
+                .len(&rtxn)
+                .map_err(|source| Error::DbRead {
+                    db: Self::LABEL,
+                    source,
+                })?;
 
         Ok(vec![
             ("query_file_entries", count_queries),
@@ -60,52 +86,54 @@ impl DbHealthChecker for QueryTracker {
     }
 }
 
+impl LmdbStore for QueryTracker {
+    const LABEL: &'static str = "query";
+    // 10 MiB hard ceiling. Same reasoning as FrecencyTracker (GH issue #437).
+    const MAP_SIZE: usize = 10 * 1024 * 1024;
+    const MAX_DBS: u32 = 16;
+    const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
+
+    fn shared_env(&self) -> &SharedEnv {
+        &self.env
+    }
+
+    fn health(&self) -> &DbHealth {
+        &self.health
+    }
+}
+
 impl QueryTracker {
     /// Returns the on-disk path of the LMDB environment directory.
     pub fn db_path(&self) -> &Path {
         self.env.path()
     }
 
-    pub fn new(db_path: impl AsRef<Path>, use_unsafe_no_lock: bool) -> Result<Self, Error> {
+    pub fn open(db_path: impl AsRef<Path>) -> Result<Self, Error> {
         let db_path = db_path.as_ref();
-        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+        let (env, health) = Self::open_env(db_path)?;
 
-        let env = unsafe {
-            let mut opts = EnvOpenOptions::new();
-            opts.map_size(10 * 1024 * 1024); // 100 MiB
-            opts.max_dbs(16); // Allow up to 16 databases per environment
-            if use_unsafe_no_lock {
-                opts.flags(EnvFlags::NO_LOCK | EnvFlags::NO_SYNC | EnvFlags::NO_META_SYNC);
-            }
-            opts.open(db_path).map_err(Error::EnvOpen)?
-        };
-
-        env.clear_stale_readers()
-            .map_err(Error::DbClearStaleReaders)?;
-
-        let mut wtxn = env.write_txn().map_err(Error::DbStartWriteTxn)?;
-
-        // Create two named databases
-        let query_file_db = env
-            .create_database(&mut wtxn, Some("query_file_associations"))
-            .map_err(Error::DbCreate)?;
-        let query_history_db = env
-            .create_database(&mut wtxn, Some("query_history"))
-            .map_err(Error::DbCreate)?;
-        let grep_query_history_db = env
-            .create_database(&mut wtxn, Some("grep_query_history"))
-            .map_err(Error::DbCreate)?;
-
-        wtxn.commit().map_err(Error::DbCommit)?;
+        let query_file_db = Self::open_database_safe(&env, Some("query_file_associations"))?;
+        let query_history_db = Self::open_database_safe(&env, Some("query_history"))?;
+        let grep_query_history_db = Self::open_database_safe(&env, Some("grep_query_history"))?;
 
         Ok(QueryTracker {
             env,
             query_file_db,
             query_history_db,
             grep_query_history_db,
+            health,
         })
     }
 
+    #[deprecated(
+        since = "0.7.0",
+        note = "LMDB unsafe no-lock mode is no longer supported; use `QueryTracker::open` instead. \
+                The `_use_unsafe_no_lock` argument is ignored."
+    )]
+    pub fn new(db_path: impl AsRef<Path>, _use_unsafe_no_lock: bool) -> Result<Self, Error> {
+        Self::open(db_path)
+    }
+
     fn get_now(&self) -> u64 {
         SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -144,7 +172,10 @@ impl QueryTracker {
     ) -> Result<(), Error> {
         let mut history = db
             .get(wtxn, project_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_default();
 
         history.push_back(HistoryEntry {
@@ -156,7 +187,10 @@ impl QueryTracker {
         }
 
         db.put(wtxn, project_key, &history)
-            .map_err(Error::DbWrite)?;
+            .map_err(|source| Error::DbWrite {
+                db: Self::LABEL,
+                source,
+            })?;
         Ok(())
     }
 
@@ -168,11 +202,17 @@ impl QueryTracker {
         project_key: &[u8; 32],
         offset: usize,
     ) -> Result<Option<String>, Error> {
-        let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
+            db: Self::LABEL,
+            source,
+        })?;
 
         let mut history = db
             .get(&rtxn, project_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_default();
 
         // history is FIFO, last element is most recent
@@ -195,12 +235,21 @@ impl QueryTracker {
         let file_path_buf = file_path.to_path_buf();
 
         let query_key = Self::create_query_key(project_path, query)?;
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         let mut entry = self
             .query_file_db
             .get(&wtxn, &query_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_else(|| QueryMatchEntry {
                 file_path: file_path_buf.clone(),
                 open_count: 0,
@@ -230,15 +279,50 @@ impl QueryTracker {
 
         entry.last_opened = now;
 
-        self.query_file_db
-            .put(&mut wtxn, &query_key, &entry)
-            .map_err(Error::DbWrite)?;
+        if let Err(e) = self.query_file_db.put(&mut wtxn, &query_key, &entry) {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on put");
+                tracing::error!(
+                    ?query,
+                    "Query tracker DB hit MDB_MAP_FULL; dropping write — db will \
+                     be erased on next open"
+                );
+                return Ok(());
+            }
+            return Err(Error::DbWrite {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
         // Update query history database
         let project_key = Self::create_project_key(project_path)?;
-        Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now)?;
+        if let Err(e) =
+            Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now)
+        {
+            if let Error::DbWrite {
+                source: ref inner, ..
+            } = e
+                && is_map_full(inner)
+            {
+                self.health.mark_unhealthy("MDB_MAP_FULL on history append");
+                tracing::error!(?query, "Query tracker DB map full while appending history");
+                return Ok(());
+            }
+            return Err(e);
+        }
 
-        wtxn.commit().map_err(Error::DbCommit)?;
+        if let Err(e) = wtxn.commit() {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on commit");
+                tracing::error!(?query, "Query tracker DB map full on commit");
+                return Ok(());
+            }
+            return Err(Error::DbCommit {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
         tracing::debug!(?query, ?file_path, "Tracked query completion");
         Ok(())
@@ -251,12 +335,21 @@ impl QueryTracker {
         min_combo_count: u32,
     ) -> Result<Option<QueryMatchEntry>, Error> {
         let query_key = Self::create_query_key(project_path, query)?;
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         let last_match = self
             .query_file_db
             .get(&rtxn, &query_key)
-            .map_err(Error::DbRead)?;
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
 
         Ok(last_match.filter(|entry| entry.open_count >= min_combo_count))
     }
@@ -270,13 +363,21 @@ impl QueryTracker {
     ) -> Result<i32, Error> {
         let query_key = Self::create_query_key(project_path, query)?;
         tracing::debug!(?query_key, "HASH");
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         match self
             .query_file_db
             .get(&rtxn, &query_key)
-            .map_err(Error::DbRead)?
-        {
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })? {
             Some(entry) => {
                 // Check if the file path matches and return boost
                 if entry.file_path == file_path && entry.open_count >= 2 {
@@ -305,17 +406,45 @@ impl QueryTracker {
     pub fn track_grep_query(&mut self, query: &str, project_path: &Path) -> Result<(), Error> {
         let now = self.get_now();
         let project_key = Self::create_project_key(project_path)?;
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
-        Self::append_to_history(
+        if let Err(e) = Self::append_to_history(
             &self.grep_query_history_db,
             &mut wtxn,
             &project_key,
             query,
             now,
-        )?;
+        ) {
+            if let Error::DbWrite {
+                source: ref inner, ..
+            } = e
+                && is_map_full(inner)
+            {
+                self.health
+                    .mark_unhealthy("MDB_MAP_FULL on grep history append");
+                tracing::error!(?query, "Grep query history DB map full; dropping write");
+                return Ok(());
+            }
+            return Err(e);
+        }
 
-        wtxn.commit().map_err(Error::DbCommit)?;
+        if let Err(e) = wtxn.commit() {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on commit");
+                tracing::error!(?query, "Grep query history DB map full on commit");
+                return Ok(());
+            }
+            return Err(Error::DbCommit {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
         tracing::debug!(?query, "Tracked grep query");
         Ok(())
@@ -343,7 +472,7 @@ mod tests {
         let temp_dir = env::temp_dir().join("fff_test_query_tracking_new");
         let _ = std::fs::remove_dir_all(&temp_dir);
 
-        let mut tracker = QueryTracker::new(temp_dir.to_str().unwrap(), true).unwrap();
+        let mut tracker = QueryTracker::open(temp_dir.to_str().unwrap()).unwrap();
 
         let project_path = PathBuf::from("/test/project");
         let file_path = PathBuf::from("/test/project/src/main.rs");

crates/fff-core/src/error.rs

@@ -8,7 +8,7 @@ pub enum Error {
     #[error("Invalid path {0}")]
     InvalidPath(std::path::PathBuf),
     #[error(
-        "Can not start fff at the file system root {0} — pass a project or at least home directory instead"
+        "Can not run certain FFF features in a file system root or home directories. Consider smaller per-project directories."
     )]
     FilesystemRoot(std::path::PathBuf),
     #[error("File picker not initialized")]
@@ -26,26 +26,79 @@ pub enum Error {
         path: std::path::PathBuf,
         source: std::io::Error,
     },
-    #[error("Failed to open frecency database env: {0}")]
-    EnvOpen(#[source] heed::Error),
-    #[error("Failed to create frecency database: {0}")]
-    DbCreate(#[source] heed::Error),
-    #[error("Failed to open frecency database: {0}")]
-    DbOpen(#[source] heed::Error),
-    #[error("Failed to clear stale readers for frecency database: {0}")]
-    DbClearStaleReaders(#[source] heed::Error),
+    #[error("Something is wrong with the local db instance: {0}")]
+    GenericDbError(#[from] heed::Error),
+    #[error("Failed to open {db} database env: {source}")]
+    EnvOpen {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error(
+        "LMDB env at {path} is already open as the '{open_as}' database with different options; requested by '{requested_as}'. Use a distinct path per database."
+    )]
+    EnvSpecMismatch {
+        path: std::path::PathBuf,
+        open_as: &'static str,
+        requested_as: &'static str,
+    },
+    #[error(
+        "The {db} database at {path} is still used by {holders} other tracker(s) in this process"
+    )]
+    DbInUse {
+        db: &'static str,
+        path: std::path::PathBuf,
+        holders: usize,
+    },
+    #[error("Failed to create {db} database: {source}")]
+    DbCreate {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to open {db} database: {source}")]
+    DbOpen {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to clear stale readers for {db} database: {source}")]
+    DbClearStaleReaders {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
 
-    #[error("Failed to start read transaction for frecency database: {0}")]
-    DbStartReadTxn(#[source] heed::Error),
-    #[error("Failed to start write transaction for frecency database: {0}")]
-    DbStartWriteTxn(#[source] heed::Error),
-
-    #[error("Failed to read from frecency database: {0}")]
-    DbRead(#[source] heed::Error),
-    #[error("Failed to write to frecency database: {0}")]
-    DbWrite(#[source] heed::Error),
-    #[error("Failed to commit write transaction to frecency database: {0}")]
-    DbCommit(#[source] heed::Error),
+    #[error("Failed to start read transaction for {db} database: {source}")]
+    DbStartReadTxn {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to start write transaction for {db} database: {source}")]
+    DbStartWriteTxn {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to read from {db} database: {source}")]
+    DbRead {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to write to {db} database: {source}")]
+    DbWrite {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to commit write transaction to {db} database: {source}")]
+    DbCommit {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
     #[error("Failed to start file system watcher: {0}")]
     FileSystemWatch(#[from] notify::Error),
 
@@ -54,6 +107,24 @@ pub enum Error {
 
     #[error("libgit2 error occurred: {0}")]
     Git(#[from] git2::Error),
+
+    #[error("Filesystem walk failed: {0}")]
+    WalkFailed(String),
+
+    #[error("Invalid glob pattern '{pattern}': {reason}")]
+    InvalidGlobPattern { pattern: String, reason: String },
+
+    #[error("File system watching is disabled for this picker")]
+    WatcherDisabled,
+
+    #[error("File system watcher is not ready")]
+    WatcherNotReady,
+
+    #[error("Indexed base path changed while creating the watch subscription")]
+    WatchBaseChanged,
+
+    #[error("Failed to start watch callback dispatcher: {0}")]
+    WatchDispatcherStart(#[source] std::io::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;

crates/fff-core/src/git.rs

@@ -1,20 +1,40 @@
 use crate::error::Result;
+use ahash::AHashMap;
 use git2::{Repository, Status, StatusOptions};
 use std::{
     fmt::Debug,
     path::{Path, PathBuf},
 };
-use tracing::debug;
 
-/// Represents a cache of a single git status query, if there is no
-/// status aka file is clear but it was specifically requested to updated
-/// the status is `None` otherwise contains only actual file statuses.
-#[derive(Debug, Clone)]
-pub struct GitStatusCache(Vec<(PathBuf, Status)>);
+pub(crate) fn default_status_options() -> StatusOptions {
+    let mut opts = StatusOptions::new();
+    opts.include_untracked(true)
+        .recurse_untracked_dirs(true)
+        .include_unmodified(true)
+        .exclude_submodules(true);
+    opts
+}
+
+/// Status options for the initial scan / rescan.
+///
+/// Skips `include_unmodified` because every `FileItem` starts with
+/// `git_status: None` (== clean), so a missing cache entry already means
+/// "clean" — no need to ask libgit2 to enumerate every tracked path.
+/// Saves seconds on huge dirty trees (e.g. chromium with 400k+ entries).
+pub(crate) fn initial_scan_status_options() -> StatusOptions {
+    let mut opts = StatusOptions::new();
+    opts.include_untracked(true)
+        .recurse_untracked_dirs(true)
+        .exclude_submodules(true);
+    opts
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct GitStatusCache(AHashMap<PathBuf, Status>);
 
 impl IntoIterator for GitStatusCache {
     type Item = (PathBuf, Status);
-    type IntoIter = std::vec::IntoIter<Self::Item>;
+    type IntoIter = <AHashMap<PathBuf, Status> as IntoIterator>::IntoIter;
 
     fn into_iter(self) -> Self::IntoIter {
         self.0.into_iter()
@@ -26,25 +46,27 @@ impl GitStatusCache {
         self.0.len()
     }
 
+    #[inline]
     pub fn lookup_status(&self, full_path: &Path) -> Option<Status> {
-        self.0
-            .binary_search_by(|(path, _)| path.as_path().cmp(full_path))
-            .ok()
-            .and_then(|idx| self.0.get(idx).map(|(_, status)| *status))
+        self.0.get(full_path).copied()
     }
 
     #[tracing::instrument(skip(repo, status_options))]
     fn read_status_impl(repo: &Repository, status_options: &mut StatusOptions) -> Result<Self> {
         let statuses = repo.statuses(Some(status_options))?;
         let Some(repo_path) = repo.workdir() else {
-            return Ok(Self(vec![])); // repo is bare
+            return Ok(Self(AHashMap::new())); // repo is bare
         };
 
-        let mut entries = Vec::with_capacity(statuses.len());
+        let repo_path = crate::path_utils::normalize(repo_path.to_path_buf());
+
+        let mut entries = AHashMap::with_capacity(statuses.len());
         for entry in &statuses {
-            if let Some(entry_path) = entry.path() {
-                let full_path = repo_path.join(entry_path);
-                entries.push((full_path, entry.status()));
+            if let Ok(entry_path) = entry.path() {
+                // libgit2 returns entry paths with forward slashes on every platform
+                // fff stores native paths - meaning we have forward slash issue on windows
+                let full_path = crate::path_utils::normalize(repo_path.join(entry_path));
+                entries.insert(full_path, entry.status());
             }
         }
 
@@ -70,46 +92,38 @@ impl GitStatusCache {
         }
     }
 
-    #[tracing::instrument(skip(repo), level = tracing::Level::DEBUG)]
+    #[tracing::instrument(skip(repo), fields(paths_count = paths.len()), level = tracing::Level::DEBUG)]
     pub fn git_status_for_paths<TPath: AsRef<Path> + Debug>(
         repo: &Repository,
         paths: &[TPath],
     ) -> Result<Self> {
         if paths.is_empty() {
-            return Ok(Self(vec![]));
+            return Ok(Self(AHashMap::new()));
         }
 
         let Some(workdir) = repo.workdir() else {
-            return Ok(Self(vec![]));
+            return Ok(Self(AHashMap::new()));
         };
+        let workdir = crate::path_utils::normalize(workdir.to_path_buf());
 
         // git pathspec is pretty slow and requires to walk the whole directory
         // so for a single file which is the most general use case we query directly the file
         if paths.len() == 1 {
             let full_path = paths[0].as_ref();
-            let relative_path = full_path.strip_prefix(workdir)?;
+            let relative_path = full_path.strip_prefix(&workdir)?;
             let status = repo.status_file(relative_path)?;
 
-            return Ok(Self(vec![(full_path.to_path_buf(), status)]));
+            let mut map = AHashMap::with_capacity(1);
+            map.insert(full_path.to_path_buf(), status);
+            return Ok(Self(map));
         }
 
-        let mut status_options = StatusOptions::new();
-        status_options
-            .include_untracked(true)
-            .recurse_untracked_dirs(true)
-            // when reading partial status it's important to include all files requested
-            .include_unmodified(true);
-
+        let mut status_options = default_status_options();
         for path in paths {
-            status_options.pathspec(path.as_ref().strip_prefix(workdir)?);
+            status_options.pathspec(path.as_ref().strip_prefix(&workdir)?);
         }
 
         let git_status_cache = Self::read_status_impl(repo, &mut status_options)?;
-        debug!(
-            status_len = git_status_cache.statuses_len(),
-            "Multiple files git status"
-        );
-
         Ok(git_status_cache)
     }
 }
@@ -157,3 +171,83 @@ pub fn format_git_status_opt(status: Option<Status>) -> Option<&'static str> {
 pub fn format_git_status(status: Option<Status>) -> &'static str {
     format_git_status_opt(status).unwrap_or("unknown")
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use std::process::Command;
+    use tempfile::TempDir;
+
+    fn git(dir: &Path, args: &[&str]) {
+        let out = Command::new("git")
+            .args(args)
+            .current_dir(dir)
+            .env("GIT_AUTHOR_NAME", "t")
+            .env("GIT_AUTHOR_EMAIL", "t@t")
+            .env("GIT_COMMITTER_NAME", "t")
+            .env("GIT_COMMITTER_EMAIL", "t@t")
+            .output()
+            .unwrap();
+        assert!(out.status.success(), "git {args:?} failed");
+    }
+
+    /// Regression: on case-insensitive filesystems libgit2 returns
+    /// statuses in a case-insensitive order. Our previous sorted-`Vec` +
+    /// `binary_search_by(Path::cmp)` lookup silently missed entries
+    /// because `Path::cmp` is byte-wise.
+    ///
+    /// This test uses deliberately mixed-case filenames so the two
+    /// orderings disagree, then checks every lookup succeeds.
+    #[test]
+    fn lookup_is_case_exact_regardless_of_libgit2_sort_order() {
+        let tmp = TempDir::new().unwrap();
+        // `std::fs::canonicalize` on Windows adds a `\\?\` UNC prefix that
+        // libgit2's workdir string lacks. Use dunce so both sides match.
+        let base = crate::path_utils::canonicalize(tmp.path()).unwrap();
+
+        // Mixed-case names that sort differently under byte-wise vs
+        // case-insensitive comparators.
+        let names = [
+            "README.md",
+            "a_lower.rs",
+            "Z_upper.rs",
+            "mixed_Case.txt",
+            "nested/Inner_File.rs",
+        ];
+        for n in &names {
+            let p = base.join(n);
+            fs::create_dir_all(p.parent().unwrap()).unwrap();
+            fs::write(&p, format!("// {n}\n")).unwrap();
+        }
+
+        git(&base, &["init", "-b", "main"]);
+        git(&base, &["add", "-A"]);
+        git(&base, &["commit", "-m", "seed", "--no-gpg-sign"]);
+
+        // Modify every file so they all end up in the status output as
+        // WT_MODIFIED — guarantees a non-trivial map we have to look up.
+        for n in &names {
+            let p = base.join(n);
+            fs::write(&p, format!("// {n}\n// edit\n")).unwrap();
+        }
+
+        let repo = Repository::open(&base).unwrap();
+        let paths: Vec<PathBuf> = names.iter().map(|n| base.join(n)).collect();
+        let cache = GitStatusCache::git_status_for_paths(&repo, &paths).unwrap();
+
+        for (n, abs) in names.iter().zip(paths.iter()) {
+            let status = cache.lookup_status(abs);
+            assert!(
+                status.is_some(),
+                "lookup for {n} returned None; cache holds {} entries",
+                cache.statuses_len(),
+            );
+            assert!(
+                status.unwrap().contains(Status::WT_MODIFIED),
+                "expected WT_MODIFIED for {n}, got {:?}",
+                status
+            );
+        }
+    }
+}

crates/fff-core/src/git_status_worker.rs

@@ -0,0 +1,120 @@
+use crate::shared::{SharedFrecency, WeakFilePicker};
+use ahash::AHashSet;
+use parking_lot::{Condvar, Mutex};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+// we don't really need a queue here
+#[derive(Default)]
+struct Pending {
+    paths: AHashSet<PathBuf>,
+    full_rescan: bool,
+    shutdown: bool,
+}
+
+impl Pending {
+    fn has_work(&self) -> bool {
+        self.full_rescan || !self.paths.is_empty()
+    }
+}
+
+/// Condvar based queue that is used for batch processing events
+pub(crate) struct GitStatusWorker {
+    state: Mutex<Pending>,
+    cv: Condvar,
+    consumer_spawned: AtomicBool,
+}
+
+impl GitStatusWorker {
+    pub(crate) fn new() -> Arc<Self> {
+        Arc::new(Self {
+            state: Mutex::new(Pending::default()),
+            cv: Condvar::new(),
+            consumer_spawned: AtomicBool::new(false),
+        })
+    }
+
+    pub(crate) fn spawn_once(
+        self: &Arc<Self>,
+        weak_picker: WeakFilePicker,
+        frecency: SharedFrecency,
+    ) {
+        if self
+            .consumer_spawned
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+        {
+            Self::spawn_consumer(Arc::clone(self), weak_picker, frecency);
+        }
+    }
+
+    pub(crate) fn enqueue_paths<I>(&self, paths: I)
+    where
+        I: IntoIterator<Item = PathBuf>,
+    {
+        let mut guard = self.state.lock();
+        guard.paths.extend(paths);
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    pub(crate) fn request_full_rescan(&self) {
+        let mut guard = self.state.lock();
+        guard.full_rescan = true;
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    pub(crate) fn signal_shutdown(&self) {
+        let mut guard = self.state.lock();
+        guard.shutdown = true;
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    fn wait_and_take(&self) -> Option<Pending> {
+        let mut guard = self.state.lock();
+        while !guard.shutdown && !guard.has_work() {
+            self.cv.wait(&mut guard);
+        }
+        if guard.shutdown {
+            return None;
+        }
+        Some(std::mem::take(&mut *guard))
+    }
+
+    // the problem: git status update can take a lot of time especially on big repositories
+    // and there is unpredictable wait time on the lock file if huge commit is going so we have to
+    // spawn a separate thread to guartee that notify handler is unlocked even if git update takes a
+    // lot of time on every event burst (pretty cheap as this thread is going to sleep 99.9% of time)
+    fn spawn_consumer(
+        mailbox: Arc<GitStatusWorker>,
+        weak_picker: WeakFilePicker,
+        frecency: SharedFrecency,
+    ) {
+        let _ = std::thread::Builder::new()
+            .name("fff-git-status".into())
+            .spawn(move || {
+                while let Some(work) = mailbox.wait_and_take() {
+                    let Some(picker) = weak_picker.upgrade() else {
+                        break;
+                    };
+
+                    if work.full_rescan {
+                        if let Err(e) = picker.refresh_git_status(&frecency) {
+                            tracing::error!("git-status worker: full rescan failed: {e:?}");
+                        }
+                    } else if !work.paths.is_empty() {
+                        let paths: Vec<PathBuf> = work.paths.into_iter().collect();
+                        if let Err(e) = picker.update_git_status_for_paths(&paths, &frecency) {
+                            tracing::error!("git-status worker: path update failed: {e:?}");
+                        }
+                    }
+                }
+
+                tracing::info!("git-status worker stopped");
+            })
+            .inspect_err(|err| tracing::error!(?err, "Failed to spawn git status worker"));
+    }
+}

crates/fff-core/src/grep/classify.rs

@@ -0,0 +1,131 @@
+//! Definition and import line classification (vibe coded POC)
+//!
+//! Byte-level heuristics that tag a matched line as a code definition
+//! (`struct`, `fn`, `class`, …) or an import/use statement. Used to
+//! rank/annotate grep results for AI/MCP consumers. Gated behind the
+//! `definitions` feature since only such consumers need it.
+
+/// Detect if a line looks like a code definition (struct, fn, class, etc.)
+pub fn is_definition_line(line: &str) -> bool {
+    let s = line.trim_start().as_bytes();
+    let s = skip_modifiers(s);
+    is_definition_keyword(s)
+}
+
+/// Modifier keywords that can precede a definition keyword.
+/// Each must be followed by whitespace to be consumed.
+const MODIFIERS: &[&[u8]] = &[
+    b"pub",
+    b"export",
+    b"default",
+    b"async",
+    b"abstract",
+    b"unsafe",
+    b"static",
+    b"protected",
+    b"private",
+    b"public",
+];
+
+/// Definition keywords to detect.
+const DEF_KEYWORDS: &[&[u8]] = &[
+    b"struct",
+    b"fn",
+    b"enum",
+    b"trait",
+    b"impl",
+    b"class",
+    b"interface",
+    b"function",
+    b"def",
+    b"func",
+    b"type",
+    b"module",
+    b"object",
+];
+
+/// Skip zero or more modifier keywords (including `pub(crate)` style visibility).
+fn skip_modifiers(mut s: &[u8]) -> &[u8] {
+    loop {
+        // Handle `pub(...)` — e.g. `pub(crate)`, `pub(super)`
+        if s.starts_with(b"pub(")
+            && let Some(end) = s.iter().position(|&b| b == b')')
+        {
+            s = skip_ws(&s[end + 1..]);
+            continue;
+        }
+        let mut matched = false;
+        for &kw in MODIFIERS {
+            if s.starts_with(kw) {
+                let rest = &s[kw.len()..];
+                if rest.first().is_some_and(|b| b.is_ascii_whitespace()) {
+                    s = skip_ws(rest);
+                    matched = true;
+                    break;
+                }
+            }
+        }
+        if !matched {
+            return s;
+        }
+    }
+}
+
+/// Check if `s` starts with a definition keyword followed by a word boundary.
+fn is_definition_keyword(s: &[u8]) -> bool {
+    for &kw in DEF_KEYWORDS {
+        if s.starts_with(kw) {
+            let after = s.get(kw.len());
+            // Word boundary: end of input, or next byte is not alphanumeric/underscore
+            if after.is_none_or(|b| !b.is_ascii_alphanumeric() && *b != b'_') {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Skip ASCII whitespace.
+#[inline]
+fn skip_ws(s: &[u8]) -> &[u8] {
+    let n = s
+        .iter()
+        .position(|b| !b.is_ascii_whitespace())
+        .unwrap_or(s.len());
+    &s[n..]
+}
+
+/// Detect import/use lines — lower value than definitions or usages.
+///
+/// Checks if the line (after leading whitespace) starts with a common
+/// import statement prefix. Pure byte-level checks, no regex.
+pub fn is_import_line(line: &str) -> bool {
+    let s = line.trim_start().as_bytes();
+    s.starts_with(b"import ")
+        || s.starts_with(b"import\t")
+        || (s.starts_with(b"from ") && s.get(5).is_some_and(|&b| b == b'\'' || b == b'"'))
+        || s.starts_with(b"use ")
+        || s.starts_with(b"use\t")
+        || starts_with_require(s)
+        || starts_with_include(s)
+}
+
+/// Match `require(` or `require (`.
+#[inline]
+fn starts_with_require(s: &[u8]) -> bool {
+    if !s.starts_with(b"require") {
+        return false;
+    }
+    let rest = &s[b"require".len()..];
+    rest.first() == Some(&b'(') || (rest.first() == Some(&b' ') && rest.get(1) == Some(&b'('))
+}
+
+/// Match `# include ` (with optional spaces after `#`).
+#[inline]
+fn starts_with_include(s: &[u8]) -> bool {
+    if s.first() != Some(&b'#') {
+        return false;
+    }
+    let rest = skip_ws(&s[1..]);
+    rest.starts_with(b"include ") || rest.starts_with(b"include\t")
+}

crates/fff-core/src/grep/fuzzy_grep.rs

@@ -0,0 +1,358 @@
+use crate::simd_path::ArenaPtr;
+use crate::types::{ContentCacheBudget, FileItem, MmapSlot};
+use fff_grep::lines::LineStep;
+use rayon::prelude::*;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use super::sink::{
+    char_indices_to_byte_offsets, classify_definition, strip_line_terminators,
+    truncate_display_bytes,
+};
+use super::types::{GrepMatch, GrepResult, GrepSearchOptions};
+
+#[allow(clippy::too_many_arguments)]
+pub(super) fn fuzzy_grep_search<'a>(
+    grep_text: &str,
+    files_to_search: &[&'a FileItem],
+    options: &GrepSearchOptions,
+    total_files: usize,
+    filtered_file_count: usize,
+    case_insensitive: bool,
+    budget: &ContentCacheBudget,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> GrepResult<'a> {
+    // max_typos controls how many *needle* characters can be unmatched.
+    // A transposition (e.g. "shcema" -> "schema") costs ~1 typo with
+    // default gap penalties. We scale max_typos by needle length:
+    //   1-2 chars -> 0 typos (exact subsequence only)
+    //   3-5 chars -> 1 typo
+    //   6+  chars -> 2 typos
+    // Cap at 2: higher values (3+) let the SIMD prefilter pass lines
+    // missing key characters entirely (e.g. query "flvencodeX" matching
+    // lines without 'l' or 'v'). Quality comes from the post-match filters.
+    let max_typos = (grep_text.len() / 3).min(2);
+    let scoring = neo_frizbee::Scoring {
+        // Use default gap penalties. Higher values (e.g. 20) cause
+        // smith-waterman to prefer *dropping needle chars* over paying
+        // gap costs, which inflates the typo count and breaks
+        // transposition matching ("shcema" -> "schema" becomes 3 typos instead of 1)
+        exact_match_bonus: 100,
+        // gap_open_penalty: 4,
+        // gap_extend_penalty: 2,
+        prefix_bonus: 0,
+        capitalization_bonus: if case_insensitive { 0 } else { 4 },
+        ..neo_frizbee::Scoring::default()
+    };
+
+    let matcher = neo_frizbee::Matcher::new(
+        grep_text,
+        &neo_frizbee::Config {
+            // Use the real max_typos so frizbee's SIMD prefilter actually rejects non-matching lines (~2 SIMD instructions per line vs full SW scoring).
+            max_typos: Some(max_typos as u16),
+            sort: false,
+            scoring,
+            ..Default::default()
+        },
+    );
+
+    // Minimum score threshold: 50% of a perfect contiguous match.
+    // With default scoring (match_score=12, matching_case_bonus=4 = 16/char),
+    // a transposition costs ~5 from a gap, keeping the score well above 50%
+    let perfect_score = (grep_text.len() as u16) * 16;
+    let min_score = (perfect_score * 50) / 100;
+
+    // Target identifiers are often longer than the query due to delimiters
+    // (e.g. query "flvencodepicture" -> "ff_flv_encode_picture_header" from ffmpeg)
+    // Allow 3x needle length to accommodate underscore/dot-separated names
+    let max_match_span = grep_text.len() * 3;
+    let needle_len = grep_text.len();
+
+    // Each delimiter (_, .) in the target creates a gap. A typical C/Rust
+    // identifier like "ff_flv_encode_picture_header" has 4-5 underscores.
+    // Scale generously so delimiter gaps don't reject valid matches.
+    let max_gaps = (needle_len / 3).max(2);
+
+    // If a file doesn't contain enough distinct needle characters just skip it
+    let needle_bytes = grep_text.as_bytes();
+    let mut unique_needle_chars: Vec<u8> = Vec::new();
+    for &b in needle_bytes {
+        let lo = b.to_ascii_lowercase();
+        let hi = b.to_ascii_uppercase();
+        if !unique_needle_chars.contains(&lo) {
+            unique_needle_chars.push(lo);
+        }
+        if lo != hi && !unique_needle_chars.contains(&hi) {
+            unique_needle_chars.push(hi);
+        }
+    }
+
+    // How many distinct needle chars must appear in the file.
+    // With max_typos allowed, we need at least (unique_count - max_typos)
+    let unique_count = {
+        let mut seen = [false; 256];
+        for &b in needle_bytes {
+            seen[b.to_ascii_lowercase() as usize] = true;
+        }
+        seen.iter().filter(|&&v| v).count()
+    };
+    let min_chars_required = unique_count.saturating_sub(max_typos);
+
+    let time_budget = if options.time_budget_ms > 0 {
+        Some(std::time::Duration::from_millis(options.time_budget_ms))
+    } else {
+        None
+    };
+    let search_start = std::time::Instant::now();
+    let budget_exceeded = AtomicBool::new(false);
+    let max_matches_per_file = options.max_matches_per_file;
+
+    // for fuzzy match we need a bit smarter chunking as the amount of work we have to perform is
+    // exponentially larger than the original grep (and the nature of work is heavier), so in short we have to
+    // understand if the approximate index prefilter got us a lot of candidates or not
+    //
+    // if we have a few candidates -> likely we have a lot of matches, so verify the check faster
+    // if we have a lot of candidates -> rely on a larger chunk pipelining more parallel lines at once
+    let page_limit = options.page_limit;
+    let base_chunk = rayon::current_num_threads() * 4;
+    let prefilter_strong = total_files > 0 && files_to_search.len() * 2 < total_files;
+    let max_chunk = if prefilter_strong {
+        base_chunk
+    } else {
+        (base_chunk * 256).max(8 * 1024)
+    };
+
+    let growth = if prefilter_strong { 1 } else { 2 };
+    let mut chunk_size = base_chunk;
+    let mut chunk_start = 0;
+    let mut running_matches = 0usize;
+    let mut per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = Vec::new();
+
+    while chunk_start < files_to_search.len() {
+        let chunk_end = (chunk_start + chunk_size).min(files_to_search.len());
+        let chunk = &files_to_search[chunk_start..chunk_end];
+        let chunk_offset = chunk_start;
+        chunk_start = chunk_end;
+        chunk_size = (chunk_size * growth).min(max_chunk);
+
+        // Parallel phase with `map_init`: each rayon worker thread clones the
+        // matcher once and gets a reusable read buffer + mmap slot. Buffer holds
+        // small files, slot holds fresh mmap for cache-miss files ≥ FRESH_MMAP_THRESHOLD.
+        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
+            .par_iter()
+            .enumerate()
+            .map_init(
+                || {
+                    (
+                        matcher.clone(),
+                        Vec::with_capacity(64 * 1024),
+                        MmapSlot::default(),
+                    )
+                },
+                |(matcher, buf, mmap_slot), (local_idx, file)| {
+                    if abort_signal.load(Ordering::Relaxed) {
+                        budget_exceeded.store(true, Ordering::Relaxed);
+                        return None;
+                    }
+
+                    if let Some(budget) = time_budget
+                        && search_start.elapsed() > budget
+                    {
+                        budget_exceeded.store(true, Ordering::Relaxed);
+                        return None;
+                    }
+
+                    let file_arena = if file.is_overflow() {
+                        overflow_arena
+                    } else {
+                        arena
+                    };
+
+                    let file_bytes =
+                        file.get_content_for_search(buf, mmap_slot, file_arena, base_path, budget)?;
+
+                    if min_chars_required > 0 {
+                        let mut chars_found = 0usize;
+                        for &ch in &unique_needle_chars {
+                            if memchr::memchr(ch, file_bytes).is_some() {
+                                chars_found += 1;
+                                if chars_found >= min_chars_required {
+                                    break;
+                                }
+                            }
+                        }
+                        if chars_found < min_chars_required {
+                            return None;
+                        }
+                    }
+
+                    // Validate the whole file as UTF-8 once upfront. Source code
+                    // files are virtually always valid UTF-8; this single check
+                    // replaces per-line from_utf8 calls (~8% of fuzzy grep time)
+                    let file_is_utf8 = std::str::from_utf8(file_bytes).is_ok();
+
+                    let mut stepper = LineStep::new(b'\n', 0, file_bytes.len());
+                    let estimated_lines = (file_bytes.len() / 40).max(64);
+                    let mut file_lines: Vec<&str> = Vec::with_capacity(estimated_lines);
+                    let mut line_meta: Vec<(u64, u64)> = Vec::with_capacity(estimated_lines);
+
+                    let mut line_number: u64 = 1;
+                    while let Some(line_match) = stepper.next_match(file_bytes) {
+                        let byte_offset = line_match.start() as u64;
+                        let trimmed = strip_line_terminators(&file_bytes[line_match]);
+
+                        if !trimmed.is_empty() {
+                            // we know for sure that the file is UTF-8 at this point
+                            let line_str = if file_is_utf8 {
+                                unsafe { std::str::from_utf8_unchecked(trimmed) }
+                            } else if let Ok(s) = std::str::from_utf8(trimmed) {
+                                s
+                            } else {
+                                line_number += 1;
+                                continue;
+                            };
+                            file_lines.push(line_str);
+                            line_meta.push((line_number, byte_offset));
+                        }
+
+                        line_number += 1;
+                    }
+
+                    if file_lines.is_empty() {
+                        return None;
+                    }
+
+                    // Single-pass: score + indices in one Smith-Waterman run per line (not parallel)
+                    let matches_with_indices = matcher.match_list_indices(&file_lines);
+                    let mut file_matches: Vec<GrepMatch> = Vec::new();
+
+                    for mut match_indices in matches_with_indices {
+                        if match_indices.score < min_score {
+                            continue;
+                        }
+
+                        let idx = match_indices.index as usize;
+                        let raw_line = file_lines[idx];
+
+                        let truncated = truncate_display_bytes(raw_line.as_bytes());
+                        let display_line = if truncated.len() < raw_line.len() {
+                            // SAFETY: truncate_display_bytes preserves UTF-8 char boundaries
+                            &raw_line[..truncated.len()]
+                        } else {
+                            raw_line
+                        };
+
+                        // If the line was truncated, re-compute indices on the shorter string.
+                        if display_line.len() < raw_line.len() {
+                            let Some(re_indices) = matcher
+                                .match_list_indices(&[display_line])
+                                .into_iter()
+                                .next()
+                            else {
+                                continue;
+                            };
+                            match_indices = re_indices;
+                        }
+
+                        match_indices.indices.sort_unstable();
+
+                        // Minimum matched chars: at least (needle_len - max_typos)
+                        // characters must appear. This is consistent with the typo
+                        // budget: each typo can drop one needle char from the alignment.
+                        let min_matched = needle_len.saturating_sub(max_typos).max(1);
+                        if match_indices.indices.len() < min_matched {
+                            continue;
+                        }
+
+                        let indices = &match_indices.indices;
+
+                        if let (Some(&first), Some(&last)) = (indices.first(), indices.last()) {
+                            // reject widely scattered matches
+                            let span = last - first + 1;
+                            if span > max_match_span {
+                                continue;
+                            }
+
+                            // Density check: matched chars / span must be dense enough.
+                            // Relaxed for perfect subsequence matches (all needle chars
+                            // present), slightly relaxed for typo matches to handle
+                            // delimiter-heavy targets
+                            // (e.g. "ff_flv_encode_picture_header" has span inflated by underscores w/ density ~68%)
+                            let density = (indices.len() * 100) / span;
+                            let min_density = if indices.len() >= needle_len {
+                                45 // Perfect subsequence relaxed (delimiters inflate span)
+                            } else {
+                                65 // Has typos filter out a long string
+                            };
+                            if density < min_density {
+                                continue;
+                            }
+
+                            // Gap count check: count discontinuities in the indices
+                            let gap_count = indices.windows(2).filter(|w| w[1] != w[0] + 1).count();
+                            if gap_count > max_gaps {
+                                continue;
+                            }
+                        }
+
+                        let (ln, bo) = line_meta[idx];
+                        let match_byte_offsets =
+                            char_indices_to_byte_offsets(display_line, &match_indices.indices);
+                        let col = match_byte_offsets
+                            .first()
+                            .map(|r| r.0 as usize)
+                            .unwrap_or(0);
+
+                        file_matches.push(GrepMatch {
+                            file_index: 0,
+                            line_number: ln,
+                            col,
+                            byte_offset: bo,
+                            is_definition: classify_definition(
+                                options.classify_definitions,
+                                display_line,
+                            ),
+                            line_content: display_line.to_string(),
+                            match_byte_offsets,
+                            fuzzy_score: Some(match_indices.score),
+                            context_before: Vec::new(),
+                            context_after: Vec::new(),
+                        });
+
+                        if max_matches_per_file != 0 && file_matches.len() >= max_matches_per_file {
+                            break;
+                        }
+                    }
+
+                    if file_matches.is_empty() {
+                        return None;
+                    }
+
+                    Some((chunk_offset + local_idx, *file, file_matches))
+                },
+            )
+            .flatten()
+            .collect();
+
+        for result in chunk_results {
+            running_matches += result.2.len();
+            per_file_results.push(result);
+        }
+
+        if running_matches >= page_limit || budget_exceeded.load(Ordering::Relaxed) {
+            break;
+        }
+    }
+
+    GrepResult::collect(
+        per_file_results,
+        files_to_search.len(),
+        options,
+        total_files,
+        filtered_file_count,
+        budget_exceeded.load(Ordering::Relaxed),
+    )
+}

crates/fff-core/src/grep/grep.rs

@@ -0,0 +1,706 @@
+use super::prefilter::prefilter_with_filepath_retry;
+use super::regex::{RegexMatcher, RegexSink, build_regex};
+use super::sink::{SinkState, debug_assert_newline_terminator};
+use super::types::{GrepMatch, GrepMode, GrepResult, GrepSearchOptions};
+use crate::index::{
+    BigramFilter, BigramOverlay, bigram_boundary, fuzzy_candidates, literal_candidates,
+    regex_candidates,
+};
+use crate::simd_string_utils::memmem;
+use crate::types::{ContentCacheBudget, FileItem, FileSliceExt, MmapSlot};
+use fff_grep::{
+    Searcher, SearcherBuilder, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use fff_query_parser::{FFFQuery, GrepConfig, QueryParser};
+use rayon::prelude::*;
+use smallvec::SmallVec;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use tracing::Level;
+
+#[allow(clippy::large_enum_variant)]
+pub(super) enum NeedleFinder<'a> {
+    CaseSensitive(memchr::memmem::Finder<'a>),
+    /// Pre-lowered needle bytes for the SIMD case-insensitive search.
+    CaseInsensitive(&'a [u8]),
+}
+
+impl<'a> NeedleFinder<'a> {
+    fn new(needle: &'a [u8], case_insensitive: bool) -> Self {
+        if case_insensitive {
+            Self::CaseInsensitive(needle)
+        } else {
+            Self::CaseSensitive(memchr::memmem::Finder::new(needle))
+        }
+    }
+
+    #[inline]
+    fn find(&self, haystack: &[u8]) -> Option<usize> {
+        match self {
+            Self::CaseSensitive(finder) => finder.find(haystack),
+            Self::CaseInsensitive(needle_lower) => memmem::find(haystack, needle_lower),
+        }
+    }
+
+    #[inline]
+    fn needle(&self) -> &[u8] {
+        match self {
+            Self::CaseSensitive(finder) => finder.needle(),
+            Self::CaseInsensitive(needle_lower) => needle_lower,
+        }
+    }
+
+    /// Compare `haystack` against a slice of the needle with the same case
+    /// semantics as `find`.
+    #[inline]
+    fn eq_fold(&self, haystack: &[u8], needle_seg: &[u8]) -> bool {
+        match self {
+            Self::CaseSensitive(_) => haystack == needle_seg,
+            Self::CaseInsensitive(_) => {
+                haystack.len() == needle_seg.len() && memmem::find(haystack, needle_seg) == Some(0)
+            }
+        }
+    }
+
+    /// Collect highlight spans for every needle occurrence within a line.
+    /// The case branch is resolved once per line, not once per occurrence.
+    #[inline]
+    fn for_each_occurrence(&self, haystack: &[u8], mut on_match: impl FnMut(usize)) {
+        match self {
+            Self::CaseSensitive(finder) => {
+                let mut start_pos = 0usize;
+                while let Some(pos) = finder.find(&haystack[start_pos..]) {
+                    on_match(start_pos + pos);
+                    start_pos += pos + 1;
+                }
+            }
+            Self::CaseInsensitive(needle_lower) => {
+                let mut start_pos = 0usize;
+                while let Some(pos) = memmem::find(&haystack[start_pos..], needle_lower) {
+                    on_match(start_pos + pos);
+                    start_pos += pos + 1;
+                }
+            }
+        }
+    }
+}
+
+struct PlainTextMatcher<'a> {
+    finder: &'a NeedleFinder<'a>,
+}
+
+impl Matcher for PlainTextMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>, NoError> {
+        let hay = &haystack[at..];
+        let needle_len = self.finder.needle().len();
+
+        Ok(self
+            .finder
+            .find(hay)
+            .map(|pos| Match::new(at + pos, at + pos + needle_len)))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        Some(fff_grep::LineTerminator::byte(b'\n'))
+    }
+}
+
+struct PlainTextSink<'r> {
+    state: SinkState,
+    finder: &'r NeedleFinder<'r>,
+    pattern_len: u32,
+    multiline_segment_len: Option<usize>,
+}
+
+impl Sink for PlainTextSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(
+        &mut self,
+        searcher: &Searcher,
+        sink_match: &SinkMatch<'_>,
+    ) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = sink_match.bytes();
+        let (display_bytes, display_len, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, sink_match);
+
+        let line_content = String::from_utf8_lossy(display_bytes).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        if let Some(seg_len) = self.multiline_segment_len {
+            // Multiline needle: the match starts on this line, so the needle's
+            // first segment must be a suffix of the line. Highlight that suffix.
+            let seg = &self.finder.needle()[..seg_len];
+            if !seg.is_empty()
+                && display_bytes.len() >= seg.len()
+                && self
+                    .finder
+                    .eq_fold(&display_bytes[display_bytes.len() - seg.len()..], seg)
+            {
+                col = display_bytes.len() - seg.len();
+                match_byte_offsets.push((col as u32, display_len));
+            }
+        } else {
+            let pattern_len = self.pattern_len;
+            self.finder.for_each_occurrence(display_bytes, |pos| {
+                let abs_start = pos as u32;
+                let abs_end = (abs_start + pattern_len).min(display_len);
+                if first {
+                    col = pos;
+                    first = false;
+                }
+                match_byte_offsets.push((abs_start, abs_end));
+            });
+        }
+
+        let (context_before, context_after) = self.state.extract_context(sink_match);
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+/// Perform a grep search across all indexed files.
+///
+/// When `query` is empty, returns git-modified/untracked files sorted by
+/// frecency for the "welcome state" UI.
+#[tracing::instrument(skip_all, fields(file_count = files.len()))]
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn grep_search<'a>(
+    files: &'a [FileItem],
+    query: &FFFQuery<'_>,
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let result = grep_search_parsed(
+        files,
+        query,
+        options,
+        budget,
+        bigram_index,
+        bigram_overlay,
+        abort_signal,
+        base_path,
+        arena,
+        overflow_arena,
+    );
+
+    // Constraint parsing can swallow tokens the user meant literally (e.g. `!=`
+    // becoming an exclusion). If the constrained search scanned everything and
+    // found nothing, retry the whole raw query as literal text. This also holds
+    // for later pages: an empty full scan at offset 0 stays empty at any offset,
+    // so paging offsets consistently index the literal search's file list.
+    let full_scan_empty = result.matches.is_empty() && result.next_file_offset == 0;
+    if !full_scan_empty || query.constraints.is_empty() || abort_signal.load(Ordering::Relaxed) {
+        return result;
+    }
+
+    let raw = query.raw_query.trim();
+    if raw.is_empty() {
+        return result;
+    }
+
+    // Keep any explicit FilePath scope (AI mode `path/to/file.ext` prefix) so the
+    // fallback can't leak matches outside the file the user pinned. Only the
+    // swallowed operator/glob tokens are dropped. See issue #756.
+    let scoped_constraints: fff_query_parser::ConstraintVec<'_> = query
+        .constraints
+        .iter()
+        .filter(|c| matches!(c, fff_query_parser::Constraint::FilePath(_)))
+        .cloned()
+        .collect();
+
+    let literal_query = FFFQuery {
+        raw_query: query.raw_query,
+        constraints: scoped_constraints,
+        fuzzy_query: fff_query_parser::FuzzyQuery::Text(raw),
+        location: None,
+    };
+
+    let mut fallback = grep_search_parsed(
+        files,
+        &literal_query,
+        options,
+        budget,
+        bigram_index,
+        bigram_overlay,
+        abort_signal,
+        base_path,
+        arena,
+        overflow_arena,
+    );
+
+    if fallback.matches.is_empty() {
+        result
+    } else {
+        fallback.literal_fallback = true;
+        fallback
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn grep_search_parsed<'a>(
+    files: &'a [FileItem],
+    query: &FFFQuery<'_>,
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let total_files = files.live_count();
+    let constraints_from_query = &query.constraints[..];
+
+    let grep_text = extract_grep_text(query);
+    if grep_text.is_empty() {
+        return GrepResult::empty(total_files, total_files);
+    }
+
+    let case_insensitive = if options.smart_case {
+        !grep_text.chars().any(|c| c.is_uppercase())
+    } else {
+        false
+    };
+
+    let base_count = bigram_boundary(bigram_overlay, files.len());
+
+    let mut regex_fallback_error: Option<String> = None;
+    let regex = match options.mode {
+        GrepMode::PlainText => None,
+        GrepMode::Fuzzy => {
+            let bigram_candidates = fuzzy_candidates(bigram_index, bigram_overlay, &grep_text);
+
+            let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
+                files,
+                constraints_from_query,
+                bigram_candidates.as_deref(),
+                base_count,
+                options,
+                arena,
+                overflow_arena,
+            );
+
+            if files_to_search.is_empty() {
+                return GrepResult::empty(total_files, filtered_file_count);
+            }
+
+            return super::fuzzy_grep::fuzzy_grep_search(
+                &grep_text,
+                &files_to_search,
+                options,
+                total_files,
+                filtered_file_count,
+                case_insensitive,
+                budget,
+                abort_signal,
+                base_path,
+                arena,
+                overflow_arena,
+            );
+        }
+        GrepMode::Regex => build_regex(&grep_text, options.smart_case)
+            .inspect_err(|err| {
+                tracing::warn!("Regex compilation failed for {}. Error {}", grep_text, err);
+
+                regex_fallback_error = Some(err.to_string());
+            })
+            .ok(),
+    };
+
+    let (multiline_segment_len, effective_pattern) = match replace_newline_escapes(&grep_text) {
+        Some((replaced, first_newline_pos)) => (Some(first_newline_pos), replaced),
+        None => (None, grep_text),
+    };
+
+    let is_multiline = multiline_segment_len.is_some();
+
+    // when there is multiple line requested automatically expand the context to include all the lines
+    let after_context = if is_multiline && regex.is_none() && options.after_context == 0 {
+        effective_pattern.bytes().filter(|&b| b == b'\n').count()
+    } else {
+        options.after_context
+    };
+
+    let finder_pattern: Vec<u8> = if case_insensitive {
+        effective_pattern.as_bytes().to_ascii_lowercase()
+    } else {
+        effective_pattern.as_bytes().to_vec()
+    };
+    let finder = NeedleFinder::new(&finder_pattern, case_insensitive);
+    let pattern_len = finder_pattern.len() as u32;
+
+    // PlainText (or regex-fallback-to-plain): literal bigram query.
+    // Regex: decompose the pattern HIR into an AND/OR bigram query tree.
+    let bigram_candidates = if regex.is_none() {
+        literal_candidates(bigram_index, bigram_overlay, &[&effective_pattern])
+    } else {
+        regex_candidates(bigram_index, bigram_overlay, &effective_pattern)
+    };
+
+    let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
+        files,
+        constraints_from_query,
+        bigram_candidates.as_deref(),
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if files_to_search.is_empty() {
+        return GrepResult::empty(total_files, filtered_file_count);
+    }
+
+    // `PlainTextMatcher` is used by the grep-searcher engine for line detection.
+    // `PlainTextSink` / `RegexSink` handle highlight extraction independently via ripgrep create
+    let plain_matcher = PlainTextMatcher { finder: &finder };
+
+    let searcher = {
+        let mut b = SearcherBuilder::new();
+        b.line_number(true).multi_line(is_multiline);
+        b
+    }
+    .build();
+
+    let should_prefilter = regex.is_none();
+    let mut result = perform_grep(
+        &files_to_search,
+        options,
+        &GrepContext {
+            total_files,
+            filtered_file_count,
+            budget,
+            base_path,
+            arena,
+            overflow_arena,
+            prefilter: should_prefilter.then_some(&finder),
+            abort_signal,
+        },
+        // The single sink-selection point: every mode's matcher/sink pairing
+        // is decided here based on the compiled pattern.
+        |file_bytes: &[u8], max_matches: usize| {
+            let state = SinkState {
+                file_index: 0,
+                matches: Vec::with_capacity(4),
+                max_matches,
+                before_context: options.before_context,
+                after_context,
+                classify_definitions: options.classify_definitions,
+            };
+
+            match regex {
+                Some(ref re) => {
+                    let regex_matcher = RegexMatcher {
+                        regex: re,
+                        is_multiline,
+                    };
+                    let mut sink = RegexSink { state, re };
+                    if let Err(e) = searcher.search_slice(&regex_matcher, file_bytes, &mut sink) {
+                        tracing::error!(error = %e, "Grep (regex) search failed");
+                    }
+                    sink.state.matches
+                }
+                None => {
+                    let mut sink = PlainTextSink {
+                        state,
+                        finder: &finder,
+                        pattern_len,
+                        multiline_segment_len,
+                    };
+                    if let Err(e) = searcher.search_slice(&plain_matcher, file_bytes, &mut sink) {
+                        tracing::error!(error = %e, "Grep (plain text) search failed");
+                    }
+                    sink.state.matches
+                }
+            }
+        },
+    );
+    result.regex_fallback_error = regex_fallback_error;
+    result
+}
+
+/// Replace unescaped `\n` escapes with real newlines in a single pass.
+///
+/// Returns `Some((replaced, first_newline_pos))` when the pattern contained at
+/// least one real `\n` escape (the user wants multiline search), where
+/// `first_newline_pos` is the byte offset of the first inserted newline in the
+/// replaced string. Returns `None` when nothing had to be replaced: `\\n` is
+/// preserved as-is (escaped backslash + literal `n`, e.g. `\\nvim-data`).
+pub(super) fn replace_newline_escapes(text: &str) -> Option<(String, usize)> {
+    let bytes = text.as_bytes();
+    let mut result = Vec::with_capacity(bytes.len());
+    let mut first_newline_pos: Option<usize> = None;
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] == b'\\' && i + 1 < bytes.len() {
+            if bytes[i + 1] == b'n' {
+                // Odd number of consecutive backslashes before 'n' -> real \n escape
+                let mut backslash_count = 1;
+                while backslash_count <= i && bytes[i - backslash_count] == b'\\' {
+                    backslash_count += 1;
+                }
+                if backslash_count % 2 == 1 {
+                    first_newline_pos.get_or_insert(result.len());
+                    result.push(b'\n');
+                    i += 2;
+                    continue;
+                }
+            }
+            result.push(bytes[i]);
+            i += 1;
+        } else {
+            result.push(bytes[i]);
+            i += 1;
+        }
+    }
+
+    let first_newline_pos = first_newline_pos?;
+    let replaced = String::from_utf8(result).unwrap_or_else(|_| text.to_string());
+    Some((replaced, first_newline_pos))
+}
+
+pub fn parse_grep_query(query: &str) -> FFFQuery<'_> {
+    let parser = QueryParser::new(GrepConfig);
+    parser.parse(query)
+}
+
+/// Extract the grep pattern text from the parsed query: all non-constraint
+/// tokens joined with spaces, e.g. `"name = *.rs someth"` -> `"name = someth"`
+/// with constraint `Extension("rs")`.
+fn extract_grep_text(query: &FFFQuery<'_>) -> String {
+    if !matches!(query.fuzzy_query, fff_query_parser::FuzzyQuery::Empty) {
+        return query.grep_text();
+    }
+
+    // if constraint-only or empty query we use raw_query for backslash-escape handling
+    let t = query.raw_query.trim();
+    if t.starts_with('\\') && t.len() > 1 {
+        let suffix = &t[1..];
+        let parser = QueryParser::new(GrepConfig);
+        if !parser.parse(suffix).constraints.is_empty() {
+            return suffix.to_string();
+        }
+    }
+    t.to_string()
+}
+
+#[derive(Clone, Copy)]
+pub(super) struct GrepContext<'a, 'b> {
+    pub(super) total_files: usize,
+    pub(super) filtered_file_count: usize,
+    pub(super) budget: &'a ContentCacheBudget,
+    pub(super) base_path: &'a Path,
+    pub(super) arena: crate::simd_path::ArenaPtr,
+    pub(super) overflow_arena: crate::simd_path::ArenaPtr,
+    pub(super) prefilter: Option<&'a NeedleFinder<'b>>,
+    pub(super) abort_signal: &'a AtomicBool,
+}
+
+impl GrepContext<'_, '_> {
+    #[inline]
+    fn arena_for_file(&self, file: &FileItem) -> crate::simd_path::ArenaPtr {
+        if file.is_overflow() {
+            self.overflow_arena
+        } else {
+            self.arena
+        }
+    }
+}
+
+#[tracing::instrument(
+    skip_all,
+    level = Level::DEBUG,
+    fields(prefiltered_count = files_to_search.len())
+)]
+pub(super) fn perform_grep<'a, F>(
+    files_to_search: &[&'a FileItem],
+    options: &GrepSearchOptions,
+    ctx: &GrepContext<'_, '_>,
+    search_file: F,
+) -> GrepResult<'a>
+where
+    F: Fn(&[u8], usize) -> Vec<GrepMatch> + Sync,
+{
+    let time_budget = if options.time_budget_ms > 0 {
+        Some(std::time::Duration::from_millis(options.time_budget_ms))
+    } else {
+        None
+    };
+
+    let search_start = std::time::Instant::now();
+    let page_limit = options.page_limit;
+    let budget_exceeded = AtomicBool::new(false);
+
+    let mut result_files: Vec<&'a FileItem> = Vec::new();
+    let mut all_matches: Vec<GrepMatch> = Vec::new();
+    let mut files_consumed: usize = 0;
+    let mut page_filled = false;
+
+    // Each chunk is a rayon barrier. A flat small chunk over 500k files = ~7800
+    // barriers; x2 growth makes it logarithmic. But a too-aggressive growth
+    // over-scans: when a page fills mid-chunk, the whole submitted chunk still
+    // runs.
+    //
+    // So only grow when the prefilter is weak (large candidate set);
+    // when bigram cut the set in half, keep fixed small chunks for cheap page-fill termination.
+    let base_chunk = rayon::current_num_threads() * 4;
+    let prefilter_strong = ctx.total_files > 0 && files_to_search.len() * 2 < ctx.total_files;
+    let max_chunk = if prefilter_strong {
+        base_chunk
+    } else {
+        (base_chunk * 256).max(8 * 1024)
+    };
+    let growth = if prefilter_strong { 1 } else { 2 };
+    let mut chunk_size = base_chunk;
+    let mut chunk_start = 0;
+
+    while chunk_start < files_to_search.len() {
+        let chunk_end = (chunk_start + chunk_size).min(files_to_search.len());
+        let chunk = &files_to_search[chunk_start..chunk_end];
+        chunk_start = chunk_end;
+        chunk_size = (chunk_size * growth).min(max_chunk);
+        let chunk_offset = files_consumed;
+
+        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
+            .par_iter()
+            .enumerate()
+            .map_init(
+                // tested it out a few times, this is just fine for rayon worker in this specific
+                // case it doesn't reallocate this many times and it is actually faster than using
+                // scoped threads with a predefined local scratch buffers because of spawn cost
+                || (Vec::with_capacity(64 * 1024), MmapSlot::default()),
+                |(buf, mmap_slot), (local_idx, file)| {
+                    // perform all the atomic machinery on every 8th
+                    if local_idx % 8 == 0 {
+                        let mut need_abort = ctx.abort_signal.load(Ordering::Relaxed);
+                        if !need_abort
+                            && let Some(budget) = time_budget
+                            && all_matches.len() > 1
+                            && search_start.elapsed() > budget
+                        {
+                            need_abort = true;
+                        }
+
+                        if need_abort {
+                            budget_exceeded.store(true, Ordering::Relaxed);
+                            return None;
+                        }
+                    }
+
+                    let content = file.get_content_for_search(
+                        buf,
+                        mmap_slot,
+                        ctx.arena_for_file(file),
+                        ctx.base_path,
+                        ctx.budget,
+                    )?;
+
+                    // Fast whole-file memmem check before entering the
+                    // grep-searcher machinery. Skips Vec alloc, Searcher
+                    // setup, and line-splitting for files that can't match.
+                    if let Some(pf) = ctx.prefilter
+                        && pf.find(content).is_none()
+                    {
+                        return None;
+                    }
+
+                    let file_matches = search_file(content, options.max_matches_per_file);
+
+                    if file_matches.is_empty() {
+                        return None;
+                    }
+
+                    Some((chunk_offset + local_idx, *file, file_matches))
+                },
+            )
+            .flatten()
+            .collect();
+
+        // Every file in the chunk was visited by rayon (matched or not).
+        files_consumed = chunk_offset + chunk.len();
+
+        // Flatten this chunk's results into the accumulator.
+        for (batch_idx, file, file_matches) in chunk_results {
+            let file_result_idx = result_files.len();
+            result_files.push(file);
+
+            for mut m in file_matches {
+                m.file_index = file_result_idx;
+                if options.trim_whitespace {
+                    m.trim_leading_whitespace();
+                }
+                all_matches.push(m);
+            }
+
+            if all_matches.len() >= page_limit {
+                // Tighten files_consumed to the file that tipped us over so
+                // the next page resumes right after it.
+                files_consumed = batch_idx + 1;
+                page_filled = true;
+                break;
+            }
+        }
+
+        if page_filled || budget_exceeded.load(Ordering::Relaxed) {
+            break;
+        }
+    }
+
+    // If no file had any match, we searched the entire slice.
+    if result_files.is_empty() {
+        files_consumed = files_to_search.len();
+    }
+
+    let has_more = budget_exceeded.load(Ordering::Relaxed)
+        || (page_filled && files_consumed < files_to_search.len());
+
+    let next_file_offset = if has_more {
+        options.file_offset + files_consumed
+    } else {
+        0
+    };
+
+    GrepResult {
+        matches: all_matches,
+        files_with_matches: result_files.len(),
+        files: result_files,
+        total_files_searched: files_consumed,
+        total_files: ctx.total_files,
+        filtered_file_count: ctx.filtered_file_count,
+        next_file_offset,
+        regex_fallback_error: None,
+        literal_fallback: false,
+    }
+}

crates/fff-core/src/grep/grep_tests.rs

@@ -0,0 +1,518 @@
+use super::grep::replace_newline_escapes;
+use super::*;
+
+use crate::file_picker::{FilePicker, FilePickerOptions};
+use crate::index::BigramIndexBuilder;
+use std::io::Write;
+use std::sync::atomic::AtomicBool;
+
+#[test]
+fn test_replace_newline_escapes() {
+    // Single \n → multiline: replaced with a real newline at byte 3
+    assert_eq!(
+        replace_newline_escapes("foo\\nbar"),
+        Some(("foo\nbar".to_string(), 3))
+    );
+    // \\n → escaped backslash + literal n, NOT multiline
+    // (this is what the user types when grepping Rust source with `\\nvim`)
+    assert_eq!(replace_newline_escapes("foo\\\\nvim-data"), None);
+    // Real-world: source file has literal \\AppData\\Local\\nvim-data
+    // (double backslash in the file, so user types double backslash)
+    assert_eq!(
+        replace_newline_escapes(r#"format!("{}\\AppData\\Local\\nvim-data","#),
+        None
+    );
+    // No \n at all
+    assert_eq!(replace_newline_escapes("hello world"), None);
+    // \\\\n → even number of backslashes before n → NOT multiline
+    assert_eq!(replace_newline_escapes("foo\\\\\\\\nbar"), None);
+    // \\\n → 3 backslashes: first two pair up, third + n = \n → multiline,
+    // newline lands after "foo" + 2 kept backslashes = byte 5
+    assert_eq!(
+        replace_newline_escapes("foo\\\\\\nbar"),
+        Some(("foo\\\\\nbar".to_string(), 5))
+    );
+    // Position is for the FIRST newline when there are several
+    assert_eq!(
+        replace_newline_escapes("a\\nb\\nc"),
+        Some(("a\nb\nc".to_string(), 1))
+    );
+}
+
+#[test]
+fn test_fuzzy_typo_scoring() {
+    // Mirror the config from fuzzy_grep_search
+    let needle = "schema";
+    let max_typos = (needle.len() / 3).min(2); // 2
+    let config = neo_frizbee::Config {
+        max_typos: Some(max_typos as u16),
+        sort: false,
+        scoring: neo_frizbee::Scoring {
+            exact_match_bonus: 100,
+            ..neo_frizbee::Scoring::default()
+        },
+        ..Default::default()
+    };
+    let min_matched = needle.len().saturating_sub(1).max(1); // 5
+    let max_match_span = needle.len() + 4; // 10
+
+    // Helper: check if a match would pass our post-filters
+    let passes = |n: &str, h: &str| -> bool {
+        let Some(mut mi) = neo_frizbee::match_list_indices(n, &[h], &config)
+            .into_iter()
+            .next()
+        else {
+            return false;
+        };
+        // upstream returns indices in reverse order, sort ascending
+        mi.indices.sort_unstable();
+        if mi.indices.len() < min_matched {
+            return false;
+        }
+        if let (Some(&first), Some(&last)) = (mi.indices.first(), mi.indices.last()) {
+            let span = last - first + 1;
+            if span > max_match_span {
+                return false;
+            }
+            let density = (mi.indices.len() * 100) / span;
+            if density < 70 {
+                return false;
+            }
+        }
+        true
+    };
+
+    // Exact match: must pass
+    assert!(passes("schema", "schema"));
+    // Exact in longer line: must pass
+    assert!(passes("schema", "  schema: String,"));
+    // In identifier: must pass
+    assert!(passes("schema", "pub fn validate_schema() {}"));
+    // Transposition: must pass
+    assert!(passes("shcema", "schema"));
+    // Partial "ema" only line: must NOT pass
+    assert!(!passes("schema", "it has ema in it"));
+    // Completely unrelated: must NOT pass
+    assert!(!passes("schema", "hello world foo bar"));
+}
+
+#[test]
+fn test_multi_grep_search() {
+    use crate::file_picker::{FilePicker, FilePickerOptions};
+    use std::io::Write;
+
+    let dir = tempfile::tempdir().unwrap();
+
+    // File 1: has "GrepMode" and "GrepMatch"
+    {
+        let mut f = std::fs::File::create(dir.path().join("grep.rs")).unwrap();
+        writeln!(f, "pub enum GrepMode {{").unwrap();
+        writeln!(f, "    PlainText,").unwrap();
+        writeln!(f, "    Regex,").unwrap();
+        writeln!(f, "}}").unwrap();
+        writeln!(f, "pub struct GrepMatch {{").unwrap();
+        writeln!(f, "    pub line_number: u64,").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    // File 2: has "PlainTextMatcher" only
+    {
+        let mut f = std::fs::File::create(dir.path().join("matcher.rs")).unwrap();
+        writeln!(f, "struct PlainTextMatcher {{").unwrap();
+        writeln!(f, "    needle: Vec<u8>,").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    // File 3: no matches
+    {
+        let mut f = std::fs::File::create(dir.path().join("other.rs")).unwrap();
+        writeln!(f, "fn main() {{").unwrap();
+        writeln!(f, "    println!(\"hello\");").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: dir.path().to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let files = picker.get_files();
+    let arena = picker.arena_base_ptr();
+
+    let options = super::GrepSearchOptions {
+        max_file_size: MAX_FFFILE_SIZE,
+        max_matches_per_file: 0,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 100,
+        mode: super::GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: None,
+    };
+    let no_cancel = AtomicBool::new(false);
+
+    // Test with 3 patterns
+    let result = super::multi_grep_search(
+        files,
+        &["GrepMode", "GrepMatch", "PlainTextMatcher"],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+
+    assert!(
+        result.matches.len() >= 3,
+        "Expected at least 3 matches, got {}",
+        result.matches.len()
+    );
+
+    let has_grep_mode = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("GrepMode"));
+    let has_grep_match = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("GrepMatch"));
+    let has_plain_text_matcher = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("PlainTextMatcher"));
+
+    assert!(has_grep_mode, "Should find GrepMode");
+    assert!(has_grep_match, "Should find GrepMatch");
+    assert!(has_plain_text_matcher, "Should find PlainTextMatcher");
+
+    assert_eq!(result.files.len(), 2, "Should match exactly 2 files");
+
+    // Test with single pattern
+    let result2 = super::multi_grep_search(
+        files,
+        &["PlainTextMatcher"],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+    assert_eq!(
+        result2.matches.len(),
+        1,
+        "Single pattern should find 1 match"
+    );
+
+    // Test with empty patterns
+    let result3 = super::multi_grep_search(
+        files,
+        &[],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+    assert_eq!(
+        result3.matches.len(),
+        0,
+        "Empty patterns should find nothing"
+    );
+}
+
+/// E2E: multiline grep (`\n` in query) and escaped-backslash literals (`\\n`)
+/// through the full picker.grep pipeline, in both PlainText and Regex modes.
+#[test]
+fn test_grep_multiline_and_escaped_newline_e2e() {
+    let dir = tempfile::tempdir().unwrap();
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+
+    // Content spanning two lines: "hello unicorn\nrainbow world"
+    {
+        let mut f = std::fs::File::create(base.join("multi.txt")).unwrap();
+        writeln!(f, "hello unicorn").unwrap();
+        writeln!(f, "rainbow world").unwrap();
+    }
+    // Content with a literal double backslash before "nvim": `C:\\Users\\nvim-data`
+    {
+        let mut f = std::fs::File::create(base.join("winpath.rs")).unwrap();
+        writeln!(f, "let p = \"C:\\\\Users\\\\nvim-data\";").unwrap();
+    }
+    {
+        let mut f = std::fs::File::create(base.join("noise.txt")).unwrap();
+        writeln!(f, "nothing interesting here").unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let options = crate::GrepSearchOptions {
+        page_limit: 100,
+        max_matches_per_file: 0,
+        ..Default::default()
+    };
+
+    // 1. PlainText + `\n`: needle becomes a real newline, match spans two lines
+    let query = super::parse_grep_query("unicorn\\nrainbow");
+    let result = picker.grep(&query, &options);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "multiline plaintext should match multi.txt"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "multi.txt");
+    let m = &result.matches[0];
+    assert_eq!(m.line_content, "hello unicorn");
+    // Auto after-context: the rest of the matched span is returned
+    assert_eq!(m.context_after, vec!["rainbow world"]);
+    // First needle segment highlighted as the line suffix
+    assert_eq!(m.match_byte_offsets.as_slice(), &[(6, 13)]);
+    assert_eq!(m.col, 6);
+
+    // 2. PlainText + `\\n`: literal backslash + n, must NOT be mangled to newline
+    let query = super::parse_grep_query("\\\\nvim-data");
+    let result = picker.grep(&query, &options);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "escaped backslash should match winpath.rs literally"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "winpath.rs");
+    assert!(result.matches[0].line_content.contains("\\\\nvim-data"));
+    assert!(result.matches[0].context_after.is_empty());
+
+    // 3. Regex + `\n`: goes through the MultiLine searcher strategy
+    let regex_options = super::GrepSearchOptions {
+        mode: super::GrepMode::Regex,
+        ..options.clone()
+    };
+    let query = super::parse_grep_query("unicorn\\nrainbow");
+    let result = picker.grep(&query, &regex_options);
+    assert!(result.regex_fallback_error.is_none());
+    assert_eq!(
+        result.files.len(),
+        1,
+        "multiline regex should match multi.txt"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "multi.txt");
+    let m = &result.matches[0];
+    // Blob is normalized: single-line content + remaining lines as context
+    assert_eq!(m.line_content, "hello unicorn");
+    assert_eq!(m.context_after, vec!["rainbow world"]);
+    // Highlight clamped to the visible first line
+    assert_eq!(m.match_byte_offsets.as_slice(), &[(6, 13)]);
+}
+
+/// Regression test for issue #407: Live grep returns duplicate results
+/// when the bigram candidate bitset has trailing bits set beyond
+/// `base_file_count`. The bitset is rounded up to a multiple of 64 bits
+/// so any trailing bit that happens to be set (e.g. from overlay data)
+/// would previously map to an overflow file index, which was then also
+/// unconditionally appended by the overflow loop, producing duplicates.
+#[test]
+fn test_grep_no_duplicates_with_overflow_trailing_bits() {
+    let dir = tempfile::tempdir().unwrap();
+    // Match the picker's internal dunce-canonicalize so paths passed to
+    // on_create_or_modify resolve back to the same base_path on Windows.
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+
+    // Five base files: only three contain the pattern "unicorn".
+    // We need some files WITHOUT the pattern so the bigrams for
+    // "unicorn" aren't treated as ubiquitous (≥90% of files) and
+    // dropped from the index during compress().
+    let base_contents: &[(&str, &str)] = &[
+        ("a.txt", "hello unicorn world"),
+        ("b.txt", "another unicorn line"),
+        ("c.txt", "one more unicorn here"),
+        ("d.txt", "nothing special in here"),
+        ("e.txt", "just some random content"),
+    ];
+    for (name, content) in base_contents {
+        let mut f = std::fs::File::create(base.join(name)).unwrap();
+        writeln!(f, "{}", content).unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+    assert_eq!(picker.get_files().len(), 5);
+
+    // Manually build a bigram index over the 5 base files.
+    let base_count = 5usize;
+    let consec_builder = BigramIndexBuilder::new(base_count);
+    let skip_builder = BigramIndexBuilder::new(base_count);
+    for (i, (_, content)) in base_contents.iter().enumerate() {
+        consec_builder.add_file_content(&skip_builder, i, content.as_bytes());
+    }
+    let mut index = consec_builder.compress(Some(0));
+    index.set_skip_index(skip_builder.compress(Some(0)));
+    picker.set_bigram_index(index);
+
+    // Add three overflow files (new after the bigram index was built),
+    // all containing "unicorn".
+    for name in ["f.txt", "g.txt", "h.txt"] {
+        let path = base.join(name);
+        let mut f = std::fs::File::create(&path).unwrap();
+        writeln!(f, "overflow unicorn entry").unwrap();
+        drop(f);
+        picker.handle_create_or_modify(&path);
+    }
+    assert_eq!(picker.get_files().len(), 8);
+
+    // Inject a trailing bit into the overlay at a file index that
+    // corresponds to an overflow file (i.e. >= base_file_count=5 but
+    // < bitset_word_size=64). Without the fix, the bigram-candidate
+    // merge would set this bit in the bitset, and the bitset loop would
+    // push files[6] while the overflow loop also appends files[5..]
+    // which includes files[6], producing a duplicate.
+    let overflow_rel = "g.txt"; // middle overflow file
+    let overflow_abs = picker
+        .get_files()
+        .iter()
+        .position(|f| f.relative_path(&picker) == overflow_rel)
+        .expect("overflow file should be present");
+    assert!(overflow_abs >= base_count);
+    assert!(
+        overflow_abs < 64,
+        "index must fit in the single bitset word"
+    );
+
+    if let Some(overlay) = picker.bigram_overlay() {
+        overlay
+            .write()
+            .modify_file(overflow_abs, b"overflow unicorn entry");
+    }
+
+    // Run a grep for "unicorn": six files match
+    // (a, b, c in base + f, g, h in overflow).
+    let query = super::parse_grep_query("unicorn");
+    let options = super::GrepSearchOptions {
+        max_file_size: MAX_FFFILE_SIZE,
+        max_matches_per_file: 0,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 100,
+        mode: super::GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: Some(std::sync::Arc::new(AtomicBool::new(false))),
+    };
+    let result = picker.grep(&query, &options);
+
+    // Collect the matched relative paths via the returned files list.
+    let mut paths: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(&picker))
+        .collect();
+    paths.sort();
+
+    // Every file (base + overflow) should match exactly once.
+    let mut dedup = paths.clone();
+    dedup.dedup();
+    assert_eq!(
+        dedup, paths,
+        "grep must not return duplicate results (issue #407): {:?}",
+        paths
+    );
+    assert_eq!(
+        paths,
+        vec!["a.txt", "b.txt", "c.txt", "f.txt", "g.txt", "h.txt"],
+    );
+
+    // And the match count must equal the number of files (one line per
+    // file). A duplicate entry in files_to_search would double-count
+    // matches for the duplicated file.
+    assert_eq!(
+        result.matches.len(),
+        6,
+        "expected exactly one match per file, got {}",
+        result.matches.len()
+    );
+}
+
+/// Issue #756: an AI-mode regex query with an inline FilePath scope and
+/// top-level alternation. The regex fragments are swallowed as bogus Glob
+/// constraints, the constrained search finds nothing, and the literal/regex
+/// fallback must NOT drop the FilePath scope — otherwise the `|` branch leaks
+/// matches into files outside the pinned path.
+#[test]
+fn regex_fallback_keeps_file_path_scope_issue_756() {
+    use fff_query_parser::{AiGrepConfig, QueryParser};
+    let dir = tempfile::tempdir().unwrap();
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+    std::fs::create_dir(base.join("scope")).unwrap();
+    std::fs::write(
+        base.join("scope").join("target.css"),
+        "/* ---------- target ---------- */\n",
+    )
+    .unwrap();
+    std::fs::write(
+        base.join("outside.css"),
+        "/* ---------- outside ---------- */\n",
+    )
+    .unwrap();
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let options = crate::GrepSearchOptions {
+        mode: super::GrepMode::Regex,
+        smart_case: true,
+        max_matches_per_file: 80,
+        page_limit: 100,
+        ..Default::default()
+    };
+
+    let raw = r"scope/target.css ^/\* |^\s*/\* ----------";
+    let query = QueryParser::new(AiGrepConfig).parse(raw);
+    let result = picker.grep(&query, &options);
+    let mut paths: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(&picker))
+        .collect();
+    paths.sort();
+
+    assert_eq!(
+        paths,
+        vec!["scope/target.css"],
+        "regex fallback must not leak outside the FilePath scope"
+    );
+}

crates/fff-core/src/grep/mod.rs

@@ -0,0 +1,27 @@
+//! Live grep. `grep.rs` implements the main plain-text path, the parallel
+//! scan engine, and the `grep_search` entry point that picks the matcher/sink
+//! for every mode in one place; `regex`, `multi_pattern`, and `fuzzy_grep`
+//! hold the mode-specific machinery on top of the shared `prefilter`/`sink`.
+
+#[allow(clippy::module_inception)]
+mod grep;
+pub use grep::*;
+
+mod fuzzy_grep;
+mod multi_pattern;
+mod prefilter;
+mod regex;
+mod sink;
+mod types;
+
+#[cfg(feature = "definitions")]
+mod classify;
+#[cfg(feature = "definitions")]
+pub use classify::*;
+
+pub(crate) use multi_pattern::multi_grep_search;
+pub use regex::has_regex_metacharacters;
+pub use types::*;
+
+#[cfg(test)]
+mod grep_tests;

crates/fff-core/src/grep/multi_pattern.rs

@@ -0,0 +1,191 @@
+use super::grep::{GrepContext, perform_grep};
+use super::prefilter::prefilter_files;
+use super::sink::{SinkState, debug_assert_newline_terminator};
+use super::types::{GrepResult, GrepSearchOptions};
+use crate::index::{BigramFilter, BigramOverlay, bigram_boundary, literal_candidates};
+use crate::types::{ContentCacheBudget, FileItem, FileSliceExt};
+use aho_corasick::AhoCorasick;
+use fff_grep::{
+    Searcher, SearcherBuilder, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use smallvec::SmallVec;
+use std::path::Path;
+use std::sync::atomic::AtomicBool;
+
+/// A `grep_matcher::Matcher` backed by Aho-Corasick for multi-pattern search.
+///
+/// Finds the first occurrence of any pattern starting at the given offset.
+/// Always reports `\n` as the line terminator for the fast candidate-line path.
+struct AhoCorasickMatcher<'a> {
+    ac: &'a AhoCorasick,
+}
+
+impl Matcher for AhoCorasickMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> std::result::Result<Option<Match>, NoError> {
+        let hay = &haystack[at..];
+        let found: Option<aho_corasick::Match> = self.ac.find(hay);
+        Ok(found.map(|m| Match::new(at + m.start(), at + m.end())))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        Some(fff_grep::LineTerminator::byte(b'\n'))
+    }
+}
+
+/// Sink for Aho-Corasick multi-pattern mode.
+///
+/// Collects all pattern match positions on each matched line for highlighting.
+struct AhoCorasickSink<'a> {
+    state: SinkState,
+    ac: &'a AhoCorasick,
+}
+
+impl Sink for AhoCorasickSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(&mut self, searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = mat.bytes();
+        let (display_bytes, display_len, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, mat);
+
+        let line_content = String::from_utf8_lossy(display_bytes).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        for m in self.ac.find_iter(display_bytes as &[u8]) {
+            let abs_start = m.start() as u32;
+            let abs_end = (m.end() as u32).min(display_len);
+            if first {
+                col = abs_start as usize;
+                first = false;
+            }
+            match_byte_offsets.push((abs_start, abs_end));
+        }
+
+        let (context_before, context_after) = self.state.extract_context(mat);
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+/// Multi-pattern OR search using Aho-Corasick.
+///
+/// Builds a single automaton from all patterns and searches each file in one
+/// pass. This is significantly faster than regex alternation for literal text
+/// searches because Aho-Corasick uses SIMD-accelerated multi-needle matching.
+///
+/// Returns the same `GrepResult` type as `grep_search`.
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn multi_grep_search<'a>(
+    files: &'a [FileItem],
+    patterns: &[&str],
+    constraints: &[fff_query_parser::Constraint<'_>],
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let total_files = files.live_count();
+
+    if patterns.is_empty() || patterns.iter().all(|p| p.is_empty()) {
+        return GrepResult::empty(total_files, total_files);
+    }
+
+    let bigram_candidates = literal_candidates(bigram_index, bigram_overlay, patterns);
+    let base_file_count = bigram_boundary(bigram_overlay, files.len());
+
+    // Constraints are separate from patterns, so a miss must not broaden the search.
+    let (files_to_search, filtered_file_count) = prefilter_files(
+        files,
+        constraints,
+        bigram_candidates.as_deref(),
+        base_file_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if files_to_search.is_empty() {
+        return GrepResult::empty(total_files, filtered_file_count);
+    }
+
+    // Smart case: case-insensitive when all patterns are lowercase
+    let case_insensitive = if options.smart_case {
+        !patterns.iter().any(|p| p.chars().any(|c| c.is_uppercase()))
+    } else {
+        false
+    };
+
+    let ac = aho_corasick::AhoCorasickBuilder::new()
+        .ascii_case_insensitive(case_insensitive)
+        .build(patterns)
+        .expect("Aho-Corasick build should not fail for literal patterns");
+
+    let searcher = {
+        let mut b = SearcherBuilder::new();
+        b.line_number(true);
+        b
+    }
+    .build();
+
+    let ac_matcher = AhoCorasickMatcher { ac: &ac };
+    perform_grep(
+        &files_to_search,
+        options,
+        &GrepContext {
+            total_files,
+            filtered_file_count,
+            budget,
+            base_path,
+            arena,
+            overflow_arena,
+            prefilter: None, // no memmem prefilter for multi-pattern search
+            abort_signal,
+        },
+        |file_bytes: &[u8], max_matches: usize| {
+            let state = SinkState {
+                file_index: 0,
+                matches: Vec::with_capacity(4),
+                max_matches,
+                before_context: options.before_context,
+                after_context: options.after_context,
+                classify_definitions: options.classify_definitions,
+            };
+
+            let mut sink = AhoCorasickSink { state, ac: &ac };
+
+            if let Err(e) = searcher.search_slice(&ac_matcher, file_bytes, &mut sink) {
+                tracing::error!(error = %e, "Grep (aho-corasick multi) search failed");
+            }
+
+            sink.state.matches
+        },
+    )
+}

crates/fff-core/src/grep/prefilter.rs

@@ -0,0 +1,204 @@
+use super::types::GrepSearchOptions;
+use crate::index::BigramFilter;
+use crate::index::constraints::{ConstraintPlan, ConstraintsBuffers};
+use crate::sort_buffer::sort_with_buffer;
+use crate::types::FileItem;
+use fff_query_parser::Constraint;
+
+/// Prefilter with a FilePath-constraint fallback: when constraints yield 0
+/// files and the query had FilePath constraints, retry without them (the path
+/// token was likely part of the search text).
+#[allow(clippy::too_many_arguments)]
+pub(super) fn prefilter_with_filepath_retry<'a>(
+    files: &'a [FileItem],
+    constraints: &[Constraint<'_>],
+    bigram_candidates: Option<&[u64]>,
+    base_count: usize,
+    options: &GrepSearchOptions,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> (Vec<&'a FileItem>, usize) {
+    let (files_to_search, filtered_file_count) = prefilter_files(
+        files,
+        constraints,
+        bigram_candidates,
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if !files_to_search.is_empty() {
+        return (files_to_search, filtered_file_count);
+    }
+
+    let Some(stripped) = strip_file_path_constraint_if_present(constraints) else {
+        return (files_to_search, filtered_file_count);
+    };
+
+    prefilter_files(
+        files,
+        &stripped,
+        bigram_candidates,
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    )
+}
+
+/// Single pass prefilter that doesn't involve file reading
+/// allocates only amount of memory required for storing references of the FileItems have to be
+/// opened for grepping unaviodably, in the worst case allocates N * <word> memory if no prefilter needed
+pub(crate) fn prefilter_files<'a>(
+    files: &'a [FileItem],
+    constraints: &[Constraint<'_>],
+    bigram_candidates: Option<&[u64]>,
+    base_count: usize,
+    options: &GrepSearchOptions,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> (Vec<&'a FileItem>, usize) {
+    let max_file_size = options.max_file_size;
+    let plan = if constraints.is_empty() {
+        None
+    } else {
+        Some(ConstraintPlan::build(
+            constraints,
+            files,
+            arena,
+            overflow_arena,
+        ))
+    };
+
+    let mut scratch = ConstraintsBuffers::new();
+
+    #[inline(always)]
+    fn basic_prefilter(file: &FileItem, max: u64) -> bool {
+        !file.is_deleted() && !file.is_binary() && file.size > 0 && file.size <= max
+    }
+
+    // squeeze as much prefilters into a single loop as possible
+    let mut prefiltered: Vec<&FileItem> = match bigram_candidates {
+        Some(candidates) => {
+            let boundary = base_count.min(files.len());
+            let (indexed, tail) = files.split_at(boundary);
+
+            let cap = BigramFilter::count_candidates(candidates) + tail.len();
+            let mut out: Vec<&FileItem> = Vec::with_capacity(cap);
+
+            let full_words = boundary / 64;
+            let last_word_bits = boundary % 64;
+
+            // we need this because we already had a regression of the wrong bit
+            // has been set for the very last word based on the overlay, it's pretty cheap
+            macro_rules! evaluate_bigram_match_word {
+                ($word:expr, $base:expr) => {{
+                    let mut bits: u64 = $word;
+                    while bits != 0 {
+                        let bit = bits.trailing_zeros() as usize;
+                        let file_idx = $base + bit;
+                        bits &= bits - 1;
+
+                        let f = unsafe { indexed.get_unchecked(file_idx) };
+                        if !basic_prefilter(f, max_file_size) {
+                            continue;
+                        }
+                        if let Some(plan) = plan.as_ref()
+                            && !plan.matches(f, file_idx, arena, overflow_arena, &mut scratch)
+                        {
+                            continue;
+                        }
+                        out.push(f);
+                    }
+                }};
+            }
+
+            // Full words: every set bit guaranteed `< boundary`.
+            for (word_idx, &word) in candidates.iter().take(full_words).enumerate() {
+                if word != 0 {
+                    evaluate_bigram_match_word!(word, word_idx * 64);
+                }
+            }
+
+            // Last partial word: mask bits past `boundary` once at word load.
+            if last_word_bits != 0 {
+                // this will get only (mod 64) bits from the last word guarantee that it's 0 padded
+                let last_mask: u64 = (1u64 << last_word_bits) - 1;
+                let word = candidates[full_words] & last_mask;
+                if word != 0 {
+                    evaluate_bigram_match_word!(word, full_words * 64);
+                }
+            }
+
+            // Sequential processing for non-bigrammable files: they are always in the end
+            for (offset, f) in tail.iter().enumerate() {
+                if !basic_prefilter(f, max_file_size) {
+                    continue;
+                }
+                if let Some(ref p) = plan
+                    && !p.matches(f, boundary + offset, arena, overflow_arena, &mut scratch)
+                {
+                    continue;
+                }
+                out.push(f);
+            }
+
+            out
+        }
+        // this will be executed if there is no bigram, in the worst case it will allocate
+        // whole array of files but probability in the real repo of NO preflter working is so
+        // low that we just ignore that, usually there would be at least a few files excluded
+        None => {
+            let mut out: Vec<&FileItem> = Vec::new();
+            for (idx, f) in files.iter().enumerate() {
+                if !basic_prefilter(f, max_file_size) {
+                    continue;
+                }
+                if let Some(ref p) = plan
+                    && !p.matches(f, idx, arena, overflow_arena, &mut scratch)
+                {
+                    continue;
+                }
+                out.push(f);
+            }
+            out
+        }
+    };
+
+    let total_count = prefiltered.len();
+
+    sort_with_buffer(&mut prefiltered, |a, b| {
+        b.total_frecency_score()
+            .cmp(&a.total_frecency_score())
+            .then(b.modified.cmp(&a.modified))
+    });
+
+    if options.file_offset > 0 && options.file_offset < total_count {
+        let paginated = prefiltered.split_off(options.file_offset);
+        (paginated, total_count)
+    } else if options.file_offset >= total_count {
+        (Vec::new(), total_count)
+    } else {
+        (prefiltered, total_count)
+    }
+}
+
+fn strip_file_path_constraint_if_present<'a>(
+    constraints: &[Constraint<'a>],
+) -> Option<fff_query_parser::ConstraintVec<'a>> {
+    if !constraints
+        .iter()
+        .any(|c| matches!(c, Constraint::FilePath(_)))
+    {
+        return None;
+    }
+
+    let filtered: fff_query_parser::ConstraintVec<'a> = constraints
+        .iter()
+        .filter(|c| !matches!(c, Constraint::FilePath(_)))
+        .cloned()
+        .collect();
+
+    Some(filtered)
+}

crates/fff-core/src/grep/regex.rs

@@ -0,0 +1,130 @@
+use super::sink::{SinkState, debug_assert_newline_terminator, split_multiline_blob};
+use fff_grep::{
+    Searcher, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use smallvec::SmallVec;
+
+pub fn has_regex_metacharacters(text: &str) -> bool {
+    regex::escape(text) != text
+}
+
+pub(super) fn build_regex(pattern: &str, smart_case: bool) -> Result<regex::bytes::Regex, String> {
+    if pattern.is_empty() {
+        return Err("empty pattern".to_string());
+    }
+
+    let regex_pattern = if pattern.contains("\\n") {
+        pattern.replace("\\n", "\n")
+    } else {
+        pattern.to_string()
+    };
+
+    let case_insensitive = if smart_case {
+        !pattern.chars().any(|c| c.is_uppercase())
+    } else {
+        false
+    };
+
+    regex::bytes::RegexBuilder::new(&regex_pattern)
+        .case_insensitive(case_insensitive)
+        .multi_line(true)
+        .unicode(false)
+        .build()
+        .map_err(|e| e.to_string())
+}
+
+pub(super) struct RegexMatcher<'r> {
+    pub(super) regex: &'r regex::bytes::Regex,
+    pub(super) is_multiline: bool,
+}
+
+impl Matcher for RegexMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>, NoError> {
+        Ok(self
+            .regex
+            .find_at(haystack, at)
+            .map(|m| Match::new(m.start(), m.end())))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        if self.is_multiline {
+            None
+        } else {
+            Some(fff_grep::LineTerminator::byte(b'\n'))
+        }
+    }
+}
+
+pub(super) struct RegexSink<'r> {
+    pub(super) state: SinkState,
+    pub(super) re: &'r regex::bytes::Regex,
+}
+
+impl Sink for RegexSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(
+        &mut self,
+        searcher: &Searcher,
+        sink_match: &SinkMatch<'_>,
+    ) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = sink_match.bytes();
+        let (display_bytes, _, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, sink_match);
+
+        // MultiLine strategy hands over all matched lines as one blob: keep
+        // `line_content` single-line, the remaining lines become after-context.
+        let (first_line, extra_after) = split_multiline_blob(display_bytes);
+        let first_len = first_line.len() as u32;
+        let line_content = String::from_utf8_lossy(first_line).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        for m in self.re.find_iter(display_bytes) {
+            let abs_start = m.start() as u32;
+            if abs_start >= first_len {
+                continue; // highlight only spans visible in the first line
+            }
+            let abs_end = (m.end() as u32).min(first_len);
+            if first {
+                col = abs_start as usize;
+                first = false;
+            }
+            match_byte_offsets.push((abs_start, abs_end));
+        }
+
+        let (context_before, context_after) = self.state.extract_context(sink_match);
+        let context_after = if extra_after.is_empty() {
+            context_after
+        } else {
+            let mut combined = extra_after;
+            combined.extend(context_after);
+            combined
+        };
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}

crates/fff-core/src/grep/sink.rs

@@ -0,0 +1,242 @@
+use super::types::GrepMatch;
+use fff_grep::{Searcher, SinkMatch};
+use smallvec::SmallVec;
+
+/// Maximum bytes of a matched line to keep for display. Prevents minified
+/// JS or huge single-line files from blowing up memory.
+pub(super) const MAX_LINE_DISPLAY_LEN: usize = 512;
+
+#[cfg(feature = "definitions")]
+#[inline]
+pub(super) fn classify_definition(enabled: bool, line: &str) -> bool {
+    enabled && super::classify::is_definition_line(line)
+}
+
+#[cfg(not(feature = "definitions"))]
+#[inline]
+pub(super) fn classify_definition(_enabled: bool, _line: &str) -> bool {
+    false
+}
+
+#[inline]
+pub(super) fn debug_assert_newline_terminator(searcher: &Searcher) {
+    debug_assert_eq!(
+        searcher.line_terminator(),
+        fff_grep::LineTerminator::byte(b'\n'),
+        "sink helpers assume \\n line terminators (see module invariant)"
+    );
+}
+
+#[inline]
+pub(super) fn strip_line_terminators(bytes: &[u8]) -> &[u8] {
+    let mut len = bytes.len();
+    while len > 0 && matches!(bytes[len - 1], b'\n' | b'\r') {
+        len -= 1;
+    }
+    &bytes[..len]
+}
+
+pub(super) struct SinkState {
+    pub(super) file_index: usize,
+    pub(super) matches: Vec<GrepMatch>,
+    pub(super) max_matches: usize,
+    pub(super) before_context: usize,
+    pub(super) after_context: usize,
+    pub(super) classify_definitions: bool,
+}
+
+impl SinkState {
+    #[inline]
+    pub(super) fn prepare_line<'a>(
+        line_bytes: &'a [u8],
+        mat: &SinkMatch<'_>,
+    ) -> (&'a [u8], u32, u64, u64) {
+        let line_number = mat.line_number().unwrap_or(0);
+        let byte_offset = mat.absolute_byte_offset();
+
+        // Trim trailing newline/CR directly on bytes to avoid UTF-8 conversion.
+        let trimmed_bytes = strip_line_terminators(line_bytes);
+
+        // Truncate for display (floor to a char boundary).
+        let display_bytes = truncate_display_bytes(trimmed_bytes);
+
+        let display_len = display_bytes.len() as u32;
+        (display_bytes, display_len, line_number, byte_offset)
+    }
+
+    #[inline]
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn push_match(
+        &mut self,
+        line_number: u64,
+        col: usize,
+        byte_offset: u64,
+        line_content: String,
+        match_byte_offsets: SmallVec<[(u32, u32); 4]>,
+        context_before: Vec<String>,
+        context_after: Vec<String>,
+    ) {
+        let is_definition = classify_definition(self.classify_definitions, &line_content);
+        self.matches.push(GrepMatch {
+            file_index: self.file_index,
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            fuzzy_score: None,
+            is_definition,
+            context_before,
+            context_after,
+        });
+    }
+
+    /// Extract context lines from the full buffer around a matched region.
+    pub(super) fn extract_context(&self, mat: &SinkMatch<'_>) -> (Vec<String>, Vec<String>) {
+        if self.before_context == 0 && self.after_context == 0 {
+            return (Vec::new(), Vec::new());
+        }
+
+        let buffer = mat.buffer();
+        let range = mat.bytes_range_in_buffer();
+
+        let mut before = Vec::new();
+        if self.before_context > 0 && range.start > 0 {
+            // Walk backward from the start of the match line to find preceding lines
+            let mut pos = range.start;
+            let mut lines_found = 0;
+            while lines_found < self.before_context && pos > 0 {
+                // Skip the newline just before our current position
+                pos -= 1;
+                // Find the previous newline
+                let line_start = match memchr::memrchr(b'\n', &buffer[..pos]) {
+                    Some(nl) => nl + 1,
+                    None => 0,
+                };
+                let line = &buffer[line_start..pos];
+                // Trim trailing \r
+                let line = if line.last() == Some(&b'\r') {
+                    &line[..line.len() - 1]
+                } else {
+                    line
+                };
+                let truncated = truncate_display_bytes(line);
+                before.push(String::from_utf8_lossy(truncated).into_owned());
+                pos = line_start;
+                lines_found += 1;
+            }
+            before.reverse();
+        }
+
+        let mut after = Vec::new();
+        if self.after_context > 0 && range.end < buffer.len() {
+            let mut pos = range.end;
+            let mut lines_found = 0;
+            while lines_found < self.after_context && pos < buffer.len() {
+                // Find the next newline
+                let line_end = match memchr::memchr(b'\n', &buffer[pos..]) {
+                    Some(nl) => pos + nl,
+                    None => buffer.len(),
+                };
+                let line = &buffer[pos..line_end];
+                // Trim trailing \r
+                let line = if line.last() == Some(&b'\r') {
+                    &line[..line.len() - 1]
+                } else {
+                    line
+                };
+                let truncated = truncate_display_bytes(line);
+                after.push(String::from_utf8_lossy(truncated).into_owned());
+                pos = if line_end < buffer.len() {
+                    line_end + 1 // skip past \n
+                } else {
+                    buffer.len()
+                };
+                lines_found += 1;
+            }
+        }
+
+        (before, after)
+    }
+}
+
+/// Truncate a byte slice for display, respecting UTF-8 char boundaries.
+#[inline]
+pub(super) fn truncate_display_bytes(bytes: &[u8]) -> &[u8] {
+    if bytes.len() <= MAX_LINE_DISPLAY_LEN {
+        bytes
+    } else {
+        let mut end = MAX_LINE_DISPLAY_LEN;
+        while end > 0 && !is_utf8_char_boundary(bytes[end]) {
+            end -= 1;
+        }
+        &bytes[..end]
+    }
+}
+
+/// Split a multiline match blob (from the MultiLine searcher strategy) into
+/// the first line and the remaining lines so `line_content` stays single-line.
+pub(super) fn split_multiline_blob(display_bytes: &[u8]) -> (&[u8], Vec<String>) {
+    match memchr::memchr(b'\n', display_bytes) {
+        None => (display_bytes, Vec::new()),
+        Some(pos) => {
+            let first = strip_line_terminators(&display_bytes[..pos + 1]);
+            let extra = display_bytes[pos + 1..]
+                .split(|&b| b == b'\n')
+                .map(|l| String::from_utf8_lossy(strip_line_terminators(l)).into_owned())
+                .collect();
+            (first, extra)
+        }
+    }
+}
+
+/// Convert character-position indices from neo_frizbee into byte-offset
+/// pairs (start, end) suitable for `match_byte_offsets`.
+///
+/// frizbee returns character positions (0-based index into the char
+/// iterator). We need byte ranges because the UI renderer and Lua layer
+/// use byte offsets for extmark highlights.
+///
+/// Each matched character becomes its own (byte_start, byte_end) pair.
+/// Adjacent characters are merged into a single contiguous range.
+pub(super) fn char_indices_to_byte_offsets(
+    line: &str,
+    char_indices: &[usize],
+) -> SmallVec<[(u32, u32); 4]> {
+    if char_indices.is_empty() {
+        return SmallVec::new();
+    }
+
+    // Build a map: char_index -> (byte_start, byte_end) for all chars.
+    // Iterating all chars is O(n) in the line length which is bounded by MAX_LINE_DISPLAY_LEN (512).
+    let char_byte_ranges: Vec<(usize, usize)> = line
+        .char_indices()
+        .map(|(byte_pos, ch)| (byte_pos, byte_pos + ch.len_utf8()))
+        .collect();
+
+    // Convert char indices to byte ranges, merging adjacent ranges
+    let mut result: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(char_indices.len());
+
+    for &ci in char_indices {
+        if ci >= char_byte_ranges.len() {
+            continue; // out of bounds (shouldn't happen with valid data)
+        }
+        let (start, end) = char_byte_ranges[ci];
+        // Merge with previous range if adjacent
+        if let Some(last) = result.last_mut()
+            && last.1 == start as u32
+        {
+            last.1 = end as u32;
+            continue;
+        }
+        result.push((start as u32, end as u32));
+    }
+
+    result
+}
+
+// copied from the rust u8 private method
+#[inline]
+const fn is_utf8_char_boundary(b: u8) -> bool {
+    (b as i8) >= -0x40
+}

crates/fff-core/src/grep/types.rs

@@ -0,0 +1,243 @@
+use crate::types::FileItem;
+use smallvec::SmallVec;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+
+pub use crate::constants::MAX_FFFILE_SIZE;
+
+/// Controls how the grep pattern is interpreted.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum GrepMode {
+    /// Literal plain text match: default path that doesn't require any regex machinery
+    #[default]
+    PlainText,
+    /// Regex mode: uses the same exact matching engine as ripgrep
+    Regex,
+    /// Smart fuzzy mode, allows user to make either a couple of single char typos or long gaps
+    /// e.g. shcema -> shcema, or UserController -> UserAuthController
+    ///
+    /// Significatnly slower than plain text, especially on unindexed FilePicker
+    Fuzzy,
+}
+
+/// A single content match within a file
+#[derive(Debug, Clone)]
+pub struct GrepMatch {
+    /// Index into the deduplicated `files` vec of the GrepResult.
+    pub file_index: usize,
+    /// 1-based line number.
+    pub line_number: u64,
+    /// 0-based byte column of first match start within the line.
+    pub col: usize,
+    /// Absolute byte offset of the matched line from the start of the file.
+    /// Can be used by the preview to seek directly without scanning from the top.
+    pub byte_offset: u64,
+    /// The matched line text, truncated to `MAX_LINE_DISPLAY_LEN`.
+    pub line_content: String,
+    /// Byte offsets `(start, end)` within `line_content` for each match.
+    /// Stack-allocated for the common case of ≤4 spans per line.
+    pub match_byte_offsets: SmallVec<[(u32, u32); 4]>,
+    /// Fuzzy match score from neo_frizbee (only set in Fuzzy grep mode).
+    pub fuzzy_score: Option<u16>,
+    /// Whether the matched line looks like a definition (struct, fn, class, etc.).
+    /// Computed at match time so output formatters don't need to re-scan.
+    pub is_definition: bool,
+    /// Lines before the match (for context display). Empty when context is 0.
+    pub context_before: Vec<String>,
+    /// Lines after the match (for context display). Empty when context is 0.
+    pub context_after: Vec<String>,
+}
+
+impl GrepMatch {
+    /// Strip leading whitespace from `line_content` and all context lines,
+    /// adjusting `col` and `match_byte_offsets` so highlights remain correct.
+    pub fn trim_leading_whitespace(&mut self) {
+        let strip_len = self.line_content.len() - self.line_content.trim_start().len();
+        if strip_len > 0 {
+            self.line_content.drain(..strip_len);
+            let off = strip_len as u32;
+            self.col = self.col.saturating_sub(strip_len);
+            for range in &mut self.match_byte_offsets {
+                range.0 = range.0.saturating_sub(off);
+                range.1 = range.1.saturating_sub(off);
+            }
+        }
+        for line in &mut self.context_before {
+            let n = line.len() - line.trim_start().len();
+            if n > 0 {
+                line.drain(..n);
+            }
+        }
+        for line in &mut self.context_after {
+            let n = line.len() - line.trim_start().len();
+            if n > 0 {
+                line.drain(..n);
+            }
+        }
+    }
+}
+
+/// Options for grep search.
+#[derive(Debug, Clone)]
+pub struct GrepSearchOptions {
+    pub max_file_size: u64,
+    pub max_matches_per_file: usize,
+    pub smart_case: bool,
+    /// File-based pagination offset: index into the sorted/filtered file list
+    /// to start searching from. Pass 0 for the first page, then use
+    /// `GrepResult::next_file_offset` for subsequent pages.
+    pub file_offset: usize,
+    /// Maximum number of matches to collect before stopping.
+    pub page_limit: usize,
+    /// How to interpret the search pattern. Defaults to `PlainText`.
+    pub mode: GrepMode,
+    /// Maximum time in milliseconds to spend searching before returning partial
+    /// results. Prevents UI freezes on pathological queries. 0 = no limit.
+    pub time_budget_ms: u64,
+    /// Number of context lines to include before each match. 0 = disabled.
+    pub before_context: usize,
+    /// Number of context lines to include after each match. 0 = disabled.
+    pub after_context: usize,
+    /// Whether to classify each match as a definition line. Adds ~2% overhead
+    /// on large repos; disable for interactive grep where it is not needed.
+    pub classify_definitions: bool,
+    /// Strip leading whitespace from matched lines and context lines, adjusting
+    /// highlight byte offsets accordingly. Useful for AI/MCP consumers and UIs
+    /// that don't need indentation. Default: false.
+    pub trim_whitespace: bool,
+    /// External abort signal. When provided, overrides the picker's internal
+    /// cancellation flag. Set to `true` to stop the search early and return
+    /// partial results. Omit (or use `..Default::default()`) to let the
+    /// picker manage cancellation.
+    pub abort_signal: Option<Arc<AtomicBool>>,
+}
+
+impl Default for GrepSearchOptions {
+    fn default() -> Self {
+        Self {
+            max_file_size: MAX_FFFILE_SIZE,
+            max_matches_per_file: 200,
+            smart_case: true,
+            file_offset: 0,
+            page_limit: 50,
+            mode: GrepMode::default(),
+            time_budget_ms: 0,
+            before_context: 0,
+            after_context: 0,
+            classify_definitions: false,
+            trim_whitespace: false,
+            abort_signal: None,
+        }
+    }
+}
+
+/// Result of a grep search with a list of matches, list of matched files, and metadata.
+#[derive(Debug, Clone, Default)]
+pub struct GrepResult<'a> {
+    pub matches: Vec<GrepMatch>,
+    /// Deduplicated file references for the returned matches.
+    pub files: Vec<&'a FileItem>,
+    /// Number of files actually searched in this call.
+    pub total_files_searched: usize,
+    /// Total number of indexed files (before filtering).
+    pub total_files: usize,
+    /// Total number of searchable files (after filtering out binary, too-large, etc.).
+    pub filtered_file_count: usize,
+    /// Number of files that contained at least one match.
+    pub files_with_matches: usize,
+    /// The file offset to pass for the next page. `0` if there are no more files.
+    /// Callers should store this and pass it as `file_offset` in the next call.
+    pub next_file_offset: usize,
+    /// When regex mode fails to compile the pattern, the search falls back to
+    /// literal matching and this field contains the compilation error message.
+    /// The UI can display this to inform the user their regex was invalid.
+    pub regex_fallback_error: Option<String>,
+    /// Set to `true` if the constrained query found nothing and the results come from
+    /// retrying the whole raw query as literal text (ignored all the inferred constraints)
+    pub literal_fallback: bool,
+}
+
+impl<'a> GrepResult<'a> {
+    /// Empty result carrying only the file counts (empty query / prefilter miss)
+    pub(crate) fn empty(total_files: usize, filtered_file_count: usize) -> Self {
+        Self {
+            total_files,
+            filtered_file_count,
+            ..Default::default()
+        }
+    }
+
+    pub(crate) fn collect(
+        per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)>,
+        files_to_search_len: usize,
+        options: &GrepSearchOptions,
+        total_files: usize,
+        filtered_file_count: usize,
+        budget_exceeded: bool,
+    ) -> Self {
+        let page_limit = options.page_limit;
+
+        // Each match stores a `file_index` pointing into `result_files` so that
+        // consumers (FFI JSON, Lua) can look up file metadata without duplicating
+        // it across every match from the same file
+        let mut result_files: Vec<&'a FileItem> = Vec::new();
+        let mut all_matches: Vec<GrepMatch> = Vec::new();
+        // files_consumed tracks how far into files_to_search we have advanced,
+        // counting every file whose results were emitted (with or without matches).
+        // We use the batch_idx of the last consumed file + 1, which is correct
+        // because per_file_results only contains files that had matches, and
+        // files between them that had no matches were still searched and can be
+        // safely skipped on the next page
+        let mut files_consumed: usize = 0;
+
+        for (batch_idx, file, file_matches) in per_file_results {
+            // batch_idx is the 0-based position in files_to_search.
+            // Advance files_consumed to include this file and all no-match files before it.
+            files_consumed = batch_idx + 1;
+
+            let file_result_idx = result_files.len();
+            result_files.push(file);
+
+            for mut m in file_matches {
+                m.file_index = file_result_idx;
+                if options.trim_whitespace {
+                    m.trim_leading_whitespace();
+                }
+                all_matches.push(m);
+            }
+
+            // page_limit is a soft cap: we always finish the current file before
+            // stopping, so no matches are dropped. A page may return up to
+            // page_limit + max_matches_per_file - 1 matches in the worst case
+            if all_matches.len() >= page_limit {
+                break;
+            }
+        }
+
+        // If no file had any match, we searched the entire slice.
+        if result_files.is_empty() {
+            files_consumed = files_to_search_len;
+        }
+
+        let has_more = budget_exceeded
+            || (all_matches.len() >= page_limit && files_consumed < files_to_search_len);
+
+        let next_file_offset = if has_more {
+            options.file_offset + files_consumed
+        } else {
+            0
+        };
+
+        Self {
+            matches: all_matches,
+            files_with_matches: result_files.len(),
+            files: result_files,
+            total_files_searched: files_consumed,
+            total_files,
+            filtered_file_count,
+            next_file_offset,
+            regex_fallback_error: None,
+            literal_fallback: false,
+        }
+    }
+}

crates/fff-core/src/ignore.rs

@@ -1,40 +1,75 @@
 use std::path::Path;
 
-pub(crate) const NON_GIT_IGNORED_DIRS: &[&str] = &[
+/// Directories excluded when walking a non-git root. Entries are `cfg`-gated
+/// so a single iteration covers standard + platform-specific overrides.
+pub(crate) const IGNORED_DIRS: &[&str] = &[
+    // various dev tools that can be meet in the developer app
     "node_modules",
     "__pycache__",
     "venv",
     ".venv",
-    // Rust (these are glob-only patterns for non_git_repo_overrides,
-    // is_non_code_directory matches the "target" component separately)
     "target/debug",
     "target/release",
     "target/rust-analyzer",
     "target/criterion",
-];
-
-#[cfg(target_os = "macos")]
-pub(crate) const PLATFORM_IGNORED_DIRS: &[&str] =
-    &["Library/Application Support", "Library/Caches"];
-
-#[cfg(target_os = "windows")]
-pub(crate) const PLATFORM_IGNORED_DIRS: &[&str] = &[
+    // Language package caches in non-git roots.
+    "go/pkg/mod",
+    ".cargo/registry",
+    ".rustup/toolchains",
+    ".gradle/caches",
+    ".m2/repository",
+    ".npm/_cacache",
+    ".pub-cache",
+    #[cfg(not(target_os = "windows"))]
+    ".local/state", // this contains tons of logs which generate too much watcher noise
+    #[cfg(target_os = "macos")]
+    "Library/Application Support",
+    #[cfg(target_os = "macos")]
+    "Library/Caches",
+    #[cfg(target_os = "macos")]
+    "Library/Containers", // sandboxed apps data
+    #[cfg(target_os = "macos")]
+    "Library/Group Containers", // random application data and networking
+    #[cfg(target_os = "macos")]
+    "Library/pnpm",
+    #[cfg(target_os = "macos")]
+    "Library/Metadata",
+    #[cfg(target_os = "macos")]
+    "Library/Developer/CoreSimulator",
+    #[cfg(target_os = "macos")]
+    "Library/Android",
+    #[cfg(target_os = "macos")]
+    "Library/Logs",
+    #[cfg(target_os = "macos")]
+    "Library/Daemon Containers",
+    #[cfg(target_os = "macos")]
+    "Library/Trial",
+    #[cfg(target_os = "macos")]
+    "Library/Preferences",
+    #[cfg(target_os = "macos")]
+    "Library/Messages",
+    #[cfg(target_os = "macos")]
+    "Library/IdentityServices",
+    #[cfg(target_os = "windows")]
     "bin/Debug",
+    #[cfg(target_os = "windows")]
     "bin/Release",
+    #[cfg(target_os = "windows")]
     "Program Files",
+    #[cfg(target_os = "windows")]
     "Program Files (x86)",
+    #[cfg(target_os = "windows")]
     "AppData/Local",
+    #[cfg(target_os = "windows")]
     "AppData/Roaming",
 ];
 
-#[cfg(not(any(target_os = "macos", target_os = "windows")))]
-pub(crate) const PLATFORM_IGNORED_DIRS: &[&str] = &[];
-
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
 pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrides::Override> {
     use ignore::overrides::OverrideBuilder;
 
     let mut builder = OverrideBuilder::new(base_path);
-    for dir in NON_GIT_IGNORED_DIRS.iter().chain(PLATFORM_IGNORED_DIRS) {
+    for dir in IGNORED_DIRS {
         let pattern = format!("!**/{dir}/");
         if let Err(e) = builder.add(&pattern) {
             tracing::warn!("failed to add ignore pattern {pattern}: {e}");
@@ -46,16 +81,56 @@ pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrid
 
 pub(crate) fn is_non_code_directory(path: &Path) -> bool {
     let path_str = path.as_os_str().to_str().unwrap_or("");
-    NON_GIT_IGNORED_DIRS
-        .iter()
-        .chain(PLATFORM_IGNORED_DIRS)
-        .any(|&dir| {
-            #[cfg(target_os = "windows")]
-            let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
-            #[cfg(target_os = "windows")]
-            return path_str.contains(dir.as_str());
+    IGNORED_DIRS.iter().any(|&dir| {
+        // Entries are gitignore patterns for the walkers; here they are matched
+        // as substrings, so a leading `*` wildcard has to come off first.
+        let dir = dir.strip_prefix('*').unwrap_or(dir);
 
-            #[cfg(not(target_os = "windows"))]
-            path_str.contains(dir)
-        })
+        #[cfg(target_os = "windows")]
+        let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
+        #[cfg(target_os = "windows")]
+        return path_str.contains(dir.as_str());
+
+        #[cfg(not(target_os = "windows"))]
+        path_str.contains(dir)
+    })
+}
+
+#[cfg(all(test, target_os = "macos"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn home_machine_state_is_excluded_but_source_trees_are_not() {
+        // Representative machine state from a home index.
+        for rel in [
+            "Library/pnpm/store/v3/files/00/abcdef",
+            "Library/Preferences/com.apple.finder.plist",
+            "Library/Messages/prewarm.db-shm",
+            "Library/IdentityServices/TetraDB-identityservicesd.db-wal",
+            "Library/Developer/CoreSimulator/Devices/X/data/f",
+            "go/pkg/mod/github.com/x/y@v1/main.go",
+            ".cargo/registry/src/index.crates.io-1/serde-1.0/src/lib.rs",
+            "Library/Android/sdk/platforms/android-34/data/x",
+            ".local/state/nvim/fff+123+456.log",
+        ] {
+            assert!(
+                is_non_code_directory(Path::new(rel)),
+                "{rel} must not reach the index"
+            );
+        }
+
+        // Source trees under $HOME stay searchable.
+        for rel in [
+            "dev/chromium/third_party/blink/renderer/core/dom/node.cc",
+            "dev/fff/crates/fff-core/src/lib.rs",
+            "Documents/notes/todo.md",
+            "dev/myproj/pkg/mod/thing.go",
+        ] {
+            assert!(
+                !is_non_code_directory(Path::new(rel)),
+                "{rel} must stay searchable"
+            );
+        }
+    }
 }

crates/fff-core/src/index/bigram_query.rs

@@ -1,26 +1,48 @@
-//! Regex → bigram decomposition for the inverted bigram index.
-//!
-//! Parses a regex pattern with `regex-syntax`, walks the HIR to extract
-//! guaranteed bigram keys (u16), and evaluates them as an AND/OR query tree
-//! against [`BigramFilter`]'s inverted posting lists.
-//!
-//! Two bigram types are extracted:
-//! - **Consecutive** (gap=0): adjacent byte pairs `(pattern[i], pattern[i+1])`
-//! - **Sparse-1** (gap=1): pairs across a single-byte wildcard, e.g. `a.b → (a,b)`
-//!
-//! The sparse-1 extraction is the key feature: regex patterns like `foo.bar`
-//! yield the cross-boundary sparse-1 bigram `(o,b)` that provides strong
-//! filtering even when the `.` prevents any consecutive cross-boundary bigram.
-
-use crate::bigram_filter::BigramFilter;
+use crate::index::bigram_filter::BigramFilter;
 use regex_syntax::hir::{Class, Hir, HirKind};
 use smallvec::SmallVec;
 use std::borrow::Cow;
 
-/// Maximum byte values to enumerate from a character class.
-/// Larger classes are treated as unknown (no bigram extractable).
 const MAX_CLASS_EXPAND: usize = 16;
 
+// stack inlined array padded with 0 and tracked length
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct InlineArray {
+    bytes: [u8; MAX_CLASS_EXPAND],
+    len: usize,
+}
+
+impl InlineArray {
+    const fn new() -> Self {
+        Self {
+            bytes: [0; MAX_CLASS_EXPAND],
+            len: 0,
+        }
+    }
+
+    fn from_byte(b: u8) -> Self {
+        let mut set = Self::new();
+        set.push(b);
+        set
+    }
+
+    /// Append a byte; no-op if already full (callers guard against this).
+    fn push(&mut self, b: u8) {
+        if self.len < MAX_CLASS_EXPAND {
+            self.bytes[self.len] = b;
+            self.len += 1;
+        }
+    }
+}
+
+impl std::ops::Deref for InlineArray {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &self.bytes[..self.len]
+    }
+}
+
 #[inline]
 fn consec_key(a: u8, b: u8) -> Option<u16> {
     let al = a.to_ascii_lowercase();
@@ -121,19 +143,15 @@ impl BigramQuery {
                 }
                 let mut result: Option<Vec<u64>> = None;
                 for child in children {
-                    match child.evaluate_cow(index) {
-                        // Any branch can't be filtered → whole OR can't be filtered
-                        None => return None,
-                        Some(child_bits) => {
-                            result = Some(match result {
-                                None => child_bits.into_owned(),
-                                Some(mut r) => {
-                                    bitset_or(&mut r, &child_bits);
-                                    r
-                                }
-                            });
+                    // Any branch can't be filtered -> whole OR can't be filtered
+                    let child_bits = child.evaluate_cow(index)?;
+                    result = Some(match result {
+                        None => child_bits.into_owned(),
+                        Some(mut r) => {
+                            bitset_or(&mut r, &child_bits);
+                            r
                         }
-                    }
+                    });
                 }
                 result.map(Cow::Owned)
             }
@@ -141,14 +159,10 @@ impl BigramQuery {
     }
 }
 
-/// Intermediate state tracked during HIR traversal for bigram extraction.
 struct HirInfo {
     query: BigramQuery,
-    /// Possible first bytes (lowercased, printable ASCII) when this node matches.
-    first: Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>,
-    /// Possible last bytes.
-    last: Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>,
-    /// Whether this node can match the empty string.
+    first: Option<InlineArray>,
+    last: Option<InlineArray>,
     can_be_empty: bool,
 }
 
@@ -185,7 +199,7 @@ pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQue
         return BigramQuery::Any;
     }
 
-    // For very short queries (0 typos), AND all bigrams — exact subsequence.
+    // the simplest case, just check that every bigram is present either consec or not
     if max_typos == 0 {
         return simplify_and(
             bigram_keys
@@ -225,7 +239,7 @@ pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQue
         return simplify_and(probes.iter().map(|&k| BigramQuery::Consec(k)).collect());
     }
 
-    // Generate all C(n, required) subsets → OR(AND(subset), ...)
+    // Generate all C(n, required) subsets as OR(AND(subset), ...)
     let mut branches = Vec::new();
     let mut combo = vec![0u16; required];
     combine(&probes, required, 0, 0, &mut combo, &mut branches);
@@ -282,7 +296,7 @@ fn decompose(hir: &Hir) -> HirInfo {
             match bytes {
                 Some(b) if !b.is_empty() => HirInfo {
                     query: BigramQuery::Any,
-                    first: Some(b.clone()),
+                    first: Some(b),
                     last: Some(b),
                     can_be_empty,
                 },
@@ -344,13 +358,13 @@ fn decompose_literal(bytes: &[u8]) -> HirInfo {
     if lower.len() == 1 {
         let b = lower[0];
         let first = if (32..=126).contains(&b) {
-            Some(SmallVec::from_slice(&[b]))
+            Some(InlineArray::from_byte(b))
         } else {
             None
         };
         return HirInfo {
             query: BigramQuery::Any,
-            first: first.clone(),
+            first,
             last: first,
             can_be_empty: false,
         };
@@ -380,12 +394,12 @@ fn decompose_literal(bytes: &[u8]) -> HirInfo {
     HirInfo {
         query: simplify_and(qs),
         first: if (32..=126).contains(&first_byte) {
-            Some(SmallVec::from_slice(&[first_byte]))
+            Some(InlineArray::from_byte(first_byte))
         } else {
             None
         },
         last: if (32..=126).contains(&last_byte) {
-            Some(SmallVec::from_slice(&[last_byte]))
+            Some(InlineArray::from_byte(last_byte))
         } else {
             None
         },
@@ -401,22 +415,20 @@ fn decompose_concat(parts: &[Hir]) -> HirInfo {
     let infos: Vec<HirInfo> = parts.iter().map(decompose).collect();
     let mut qs: Vec<BigramQuery> = Vec::new();
 
-    // 1. Collect child bigrams
     for info in &infos {
         if !info.query.is_any() {
             qs.push(info.query.clone());
         }
     }
 
-    // 2. Dense cross-boundary between adjacent mandatory parts
+    //  Dense cross-boundary between adjacent mandatory parts
     for pair in infos.windows(2) {
         if !pair[0].can_be_empty && !pair[1].can_be_empty {
             push_cross_consec(&mut qs, pair[0].last.as_deref(), pair[1].first.as_deref());
         }
     }
 
-    // 3. Sparse-1 cross-boundary: across a single 1-byte-wide middle part.
-    //    Catches `foo.bar` → sparse-1 `(o,b)` across the dot.
+    //  Sparse-1 cross-boundary: across a single 1 byte wide middle part
     if parts.len() >= 3 {
         for i in 0..parts.len() - 2 {
             let left = &infos[i];
@@ -464,8 +476,8 @@ fn decompose_alternation(alts: &[Hir]) -> HirInfo {
     }
 }
 
-fn expand_class(class: &Class) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut bytes: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn expand_class(class: &Class) -> Option<InlineArray> {
+    let mut bytes = InlineArray::new();
     match class {
         Class::Bytes(bc) => {
             for range in bc.ranges() {
@@ -473,6 +485,7 @@ fn expand_class(class: &Class) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
                 if bytes.len() + count > MAX_CLASS_EXPAND {
                     return None;
                 }
+
                 for b in range.start()..=range.end() {
                     if (32..=126).contains(&b) {
                         let lower = b.to_ascii_lowercase();
@@ -554,11 +567,11 @@ fn cross_product(last: Option<&[u8]>, first: Option<&[u8]>, skip: bool) -> Optio
     }
 }
 
-fn collect_first(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn collect_first(infos: &[HirInfo]) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
     for info in infos {
         if let Some(ref bytes) = info.first {
-            for &b in bytes {
+            for &b in bytes.iter() {
                 if !result.contains(&b) {
                     if result.len() >= MAX_CLASS_EXPAND {
                         return None;
@@ -580,11 +593,11 @@ fn collect_first(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>
     }
 }
 
-fn collect_last(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn collect_last(infos: &[HirInfo]) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
     for info in infos.iter().rev() {
         if let Some(ref bytes) = info.last {
-            for &b in bytes {
+            for &b in bytes.iter() {
                 if !result.contains(&b) {
                     if result.len() >= MAX_CLASS_EXPAND {
                         return None;
@@ -606,22 +619,17 @@ fn collect_last(infos: &[HirInfo]) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
     }
 }
 
-fn merge_byte_sets<'a>(
-    iter: impl Iterator<Item = &'a Option<SmallVec<[u8; MAX_CLASS_EXPAND]>>>,
-) -> Option<SmallVec<[u8; MAX_CLASS_EXPAND]>> {
-    let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new();
+fn merge_byte_sets<'a>(iter: impl Iterator<Item = &'a Option<InlineArray>>) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
     for opt in iter {
-        match opt {
-            None => return None,
-            Some(bytes) => {
-                for &b in bytes {
-                    if !result.contains(&b) {
-                        if result.len() >= MAX_CLASS_EXPAND {
-                            return None;
-                        }
-                        result.push(b);
-                    }
+        let bytes = opt.as_ref()?;
+
+        for &b in bytes.iter() {
+            if !result.contains(&b) {
+                if result.len() >= MAX_CLASS_EXPAND {
+                    return None;
                 }
+                result.push(b);
             }
         }
     }
@@ -669,7 +677,7 @@ fn simplify_or(children: Vec<BigramQuery>) -> BigramQuery {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bigram_filter::BigramIndexBuilder;
+    use crate::index::bigram_filter::BigramIndexBuilder;
 
     /// Build a tiny index from the given file contents for testing.
     fn build_test_index(files: &[&[u8]]) -> BigramFilter {
@@ -757,7 +765,7 @@ mod tests {
 
     #[test]
     fn sparse1_across_digit() {
-        // "foo\dbar" → sparse-1 (o,b) across \d
+        // "foo\dbar" -> sparse-1 (o,b) across \d
         let idx = build_test_index(&[
             b"foo3bar baz", // 0: has all bigrams
             b"foobar baz",  // 1: has consecutive (o,b) but pattern needs sparse-1
@@ -795,7 +803,6 @@ mod tests {
 
     #[test]
     fn optional_group_excluded() {
-        // (bar)? is optional — its bigrams are not required
         let q = regex_to_bigram_query("foo(bar)?baz");
         assert!(!q.is_any());
 
@@ -813,7 +820,7 @@ mod tests {
 
     #[test]
     fn repetition_min2_cross_boundary() {
-        // (ab){2,} → bigram "ab" + cross-boundary "b","a"
+        // (ab){2,} -> bigram "ab" + cross-boundary "b","a"
         let q = regex_to_bigram_query("(ab){2,}");
         assert!(!q.is_any());
 
@@ -830,16 +837,14 @@ mod tests {
 
     #[test]
     fn two_dots_no_sparse1() {
-        // "a..b" — two 1-byte parts between a and b, not a single 1-byte part
-        // No sparse-1 (a,b) should be extracted
         let q = regex_to_bigram_query("a..b");
-        // Single-char literals with 2 unknown bytes between → Any
+        // Single-char literals with 2 unknown bytes between -> Any
         assert!(q.is_any());
     }
 
     #[test]
     fn character_class_cross_boundary() {
-        // [abc]de → cross-boundary OR(ad,bd,cd) + bigram de
+        // [abc]de -> cross-boundary OR(ad,bd,cd) + bigram de
         // All three class variants must appear in the corpus so the OR
         // branches are tracked in the index (untracked bigrams make the
         // OR conservatively return None, which is correct but untestable).
@@ -861,8 +866,6 @@ mod tests {
         assert!(!BigramFilter::is_candidate(&candidates, 3));
     }
 
-    // ── Helpers for inspecting query trees ──────────────────────────
-
     fn has_consec(q: &BigramQuery, a: u8, b: u8) -> bool {
         let Some(key) = consec_key(a, b) else {
             return false;
@@ -896,13 +899,12 @@ mod tests {
     /// plus typical grep patterns used by agentic tools.
     ///
     /// Each entry: `(regex, Option<&[Bg]>)`.
-    ///   - `None`        → pure classes / unsupported syntax, Any is acceptable.
-    ///   - `Some(&[..])` → must be non-Any, and every listed bigram must appear.
+    ///   - `None`        -> pure classes / unsupported syntax, Any is acceptable.
+    ///   - `Some(&[..])` -> must be non-Any, and every listed bigram must appear.
     #[test]
     fn common_regex_patterns() {
         #[rustfmt::skip]
         let cases: &[(&str, Option<&[Bg]>)] = &[
-            // ── Pure-class / anchor / unsupported → Any is fine ──────
             (r"^\d+$",                                                      None), // 1.  whole numbers
             (r"^\d*\.\d+$",                                                 None), // 2.  decimals
             (r"^\d*(\.\d+)?$",                                              None), // 3.  whole + decimal
@@ -931,11 +933,9 @@ mod tests {
             (r"^[\w,\s-]+\.[A-Za-z]{3}$",                                 None), // 27. filename
             (r"^[A-PR-WY][1-9]\d\s?\d{4}[1-9]$",                         None), // 28. HK ID
 
-            // ── Patterns with extractable literal bigrams ────────────
-
             // 13. URL with required protocol
             (r"https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", Some(&[
-                ("ht", C), ("tt", C), ("tp", C),   // from "http"
+                ("ht", C), ("tt", C), ("tp", C),    // from "http"
                 ("ht", S), ("tp", S),               // from "http" skip-1
                 (":/", C), ("//", C),               // from "://"
             ])),
@@ -943,31 +943,31 @@ mod tests {
             // 29. fn\s+\w+
             (r"fn\s+\w+", Some(&[
                 ("fn", C),                          // from "fn"
-                ("n ", C),                          // cross-boundary: 'n' → \s starts ' '
+                ("n ", C),                          // cross-boundary: 'n' -> \s starts ' '
             ])),
 
             // 30. use\s+crate::
             (r"use\s+crate::", Some(&[
-                ("us", C), ("se", C), ("ue", S),   // from "use"
-                ("cr", C), ("ra", C), ("at", C),   // from "crate"
+                ("us", C), ("se", C), ("ue", S),    // from "use"
+                ("cr", C), ("ra", C), ("at", C),    // from "crate"
                 ("te", C), ("::", C),
-                ("ca", S), ("rt", S), ("ae", S),   // "crate" skip-1
+                ("ca", S), ("rt", S), ("ae", S),    // "crate" skip-1
             ])),
 
             // 31. unwrap\(\)|expect\(
             (r"unwrap\(\)|expect\(", Some(&[
-                ("nw", C), ("wr", C), ("ra", C),   // "unwrap("
+                ("nw", C), ("wr", C), ("ra", C),    // "unwrap("
                 ("ap", C), ("p(", C),
-                ("xp", C), ("pe", C), ("ec", C),   // "expect("
+                ("xp", C), ("pe", C), ("ec", C),    // "expect("
                 ("ct", C), ("t(", C),
             ])),
 
             // 32. TODO|FIXME|HACK
             (r"TODO|FIXME|HACK", Some(&[
-                ("to", C), ("od", C), ("do", C),   // "TODO"
-                ("fi", C), ("ix", C), ("xm", C),   // "FIXME"
+                ("to", C), ("od", C), ("do", C),    // "TODO"
+                ("fi", C), ("ix", C), ("xm", C),    // "FIXME"
                 ("me", C),
-                ("ha", C), ("ac", C), ("ck", C),   // "HACK"
+                ("ha", C), ("ac", C), ("ck", C),    // "HACK"
                 ("hc", S), ("ak", S),               // "HACK" skip-1
             ])),
         ];

crates/fff-core/src/index/candidates.rs

@@ -0,0 +1,118 @@
+use super::{BigramFilter, BigramOverlay, extract_bigrams};
+use super::{fuzzy_to_bigram_query, regex_to_bigram_query};
+
+/// Number of evenly-spaced probe bigrams used by the fuzzy candidate query.
+const FUZZY_PROBE_COUNT: usize = 7;
+
+#[inline]
+fn set_bit(candidates: &mut [u64], file_idx: usize) {
+    let word = file_idx / 64;
+    if word < candidates.len() {
+        candidates[word] |= 1u64 << (file_idx % 64);
+    }
+}
+
+#[inline]
+fn clear_tombstones(candidates: &mut [u64], overlay: &BigramOverlay) {
+    for (r, t) in candidates.iter_mut().zip(overlay.tombstones().iter()) {
+        *r &= !t;
+    }
+}
+
+/// Number of base files covered by the bigram bitset; files past this
+/// boundary (overflow, max 1024) are always scanned.
+#[inline]
+pub(crate) fn bigram_boundary(overlay: Option<&BigramOverlay>, files_len: usize) -> usize {
+    overlay.map(|o| o.base_file_count()).unwrap_or(files_len)
+}
+
+/// Candidate bitset for literal patterns, OR-ed across all of them: a file is
+/// a candidate when it contains the bigrams of ANY pattern. Overlay-modified
+/// files are re-checked against each pattern's bigrams.
+pub(crate) fn literal_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    patterns: &[&str],
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let mut combined: Option<Vec<u64>> = None;
+    for pattern in patterns {
+        if let Some(candidates) = index.query(pattern.as_bytes()) {
+            combined = Some(match combined {
+                None => candidates,
+                Some(mut acc) => {
+                    acc.iter_mut()
+                        .zip(candidates.iter())
+                        .for_each(|(a, b)| *a |= *b);
+                    acc
+                }
+            });
+        }
+    }
+
+    let mut candidates = combined?;
+    if let Some(overlay) = overlay {
+        clear_tombstones(&mut candidates, overlay);
+        for pattern in patterns {
+            let pattern_bigrams = extract_bigrams(pattern.as_bytes());
+            for file_idx in overlay.query_modified(&pattern_bigrams) {
+                set_bit(&mut candidates, file_idx);
+            }
+        }
+    }
+    Some(candidates)
+}
+
+/// Candidate bitset for a regex pattern: the regex HIR is decomposed into an
+/// AND/OR bigram query tree (supports alternation, optional groups, character
+/// classes, and sparse-1 bigrams across single-byte wildcards). Since modified
+/// file contents can't be re-checked against a regex cheaply, all
+/// overlay-modified files are conservatively added.
+pub(crate) fn regex_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    pattern: &str,
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let bq = regex_to_bigram_query(pattern);
+    if bq.is_any() {
+        return None;
+    }
+    let candidates = bq.evaluate(index)?;
+    Some(add_all_modified(candidates, overlay))
+}
+
+/// Candidate bitset for a fuzzy pattern: evenly-spaced probe bigrams with a
+/// typo allowance (widely-spaced probes are far more selective than sliding
+/// windows of adjacent bigrams). All overlay-modified files are added.
+pub(crate) fn fuzzy_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    pattern: &str,
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let bq = fuzzy_to_bigram_query(pattern, FUZZY_PROBE_COUNT);
+    if bq.is_any() {
+        return None;
+    }
+    let candidates = bq.evaluate(index)?;
+    Some(add_all_modified(candidates, overlay))
+}
+
+#[inline]
+fn ready_index(index: Option<&BigramFilter>) -> Option<&BigramFilter> {
+    index.filter(|idx| idx.is_ready())
+}
+
+fn add_all_modified(mut candidates: Vec<u64>, overlay: Option<&BigramOverlay>) -> Vec<u64> {
+    if let Some(overlay) = overlay {
+        clear_tombstones(&mut candidates, overlay);
+        for file_idx in overlay.modified_indices() {
+            set_bit(&mut candidates, file_idx);
+        }
+    }
+    candidates
+}

crates/fff-core/src/index/constraints.rs

@@ -0,0 +1,950 @@
+//! Constraint-based prefiltering for search queries.
+
+use fff_query_parser::{Constraint, GitStatusFilter};
+use smallvec::SmallVec;
+
+use crate::git::is_modified_status;
+use crate::simd_path::ArenaPtr;
+use crate::simd_string_utils::memmem::find_case_insensitive_short;
+
+const PAR_THRESHOLD: usize = 10_000;
+
+pub(crate) trait Constrainable {
+    fn write_file_name(&self, arena: ArenaPtr, out: &mut String);
+    fn git_status(&self) -> Option<git2::Status>;
+    fn write_relative_path(&self, arena: ArenaPtr, out: &mut String);
+    fn is_overflow(&self) -> bool;
+}
+
+/// Stored/canonical paths use `/`; also accept `\` so a Windows user typing
+/// a native separator in a query still matches.
+#[inline]
+fn is_path_sep(b: u8) -> bool {
+    b == b'/' || b == b'\\'
+}
+
+#[inline]
+fn path_slice_eq(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    a.iter().zip(b).all(|(x, y)| {
+        if is_path_sep(*x) && is_path_sep(*y) {
+            true
+        } else {
+            x.eq_ignore_ascii_case(y)
+        }
+    })
+}
+
+/// Path ends with suffix at a path-separator boundary (case-insensitive).
+#[inline]
+pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool {
+    let path_bytes = path.as_bytes();
+    let suffix_bytes = suffix.as_bytes();
+    if path_bytes.len() < suffix_bytes.len() {
+        return false;
+    }
+
+    let start = path.len() - suffix.len();
+
+    // Multi-byte UTF-8 may put `start` inside a char.
+    if !path.is_char_boundary(start) {
+        return false;
+    }
+
+    if !path_slice_eq(&path_bytes[start..], suffix_bytes) {
+        return false;
+    }
+
+    // Exact or preceded by a separator. Scan backward past any multi-byte
+    // continuation bytes to find the preceding ASCII byte.
+    if start == 0 {
+        return true;
+    }
+    let mut i = start;
+    while i > 0 {
+        i -= 1;
+        if path_bytes[i] < 128 {
+            return is_path_sep(path_bytes[i]);
+        }
+    }
+    false
+}
+
+#[inline]
+pub fn file_has_extension(file_name: &str, ext: &str) -> bool {
+    let name_bytes = file_name.as_bytes();
+    let ext_bytes = ext.as_bytes();
+    if name_bytes.len() <= ext_bytes.len() + 1 {
+        return false;
+    }
+    let start = name_bytes.len() - ext_bytes.len() - 1;
+    if start > 0 && !file_name.is_char_boundary(start) {
+        return false;
+    }
+    name_bytes.get(start) == Some(&b'.') && name_bytes[start + 1..].eq_ignore_ascii_case(ext_bytes)
+}
+
+/// Matches multi-segment queries like `libswscale/aarch64`.
+#[inline]
+pub fn path_contains_segment(path: &str, segment: &str) -> bool {
+    let path_bytes = path.as_bytes();
+    let segment_bytes = segment.as_bytes();
+    let segment_len = segment_bytes.len();
+
+    if path_bytes.len() > segment_len
+        && is_path_sep(path_bytes[segment_len])
+        && path.is_char_boundary(segment_len)
+        && path_slice_eq(&path_bytes[..segment_len], segment_bytes)
+    {
+        return true;
+    }
+
+    if path_bytes.len() < segment_len + 2 {
+        return false;
+    }
+
+    for i in 0..path_bytes.len().saturating_sub(segment_len + 1) {
+        if is_path_sep(path_bytes[i]) {
+            let start = i + 1;
+            let end = start + segment_len;
+            if end < path_bytes.len()
+                && is_path_sep(path_bytes[end])
+                && path.is_char_boundary(start)
+                && path.is_char_boundary(end)
+                && path_slice_eq(&path_bytes[start..end], segment_bytes)
+            {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Returns `None` if no constraints are present, `Some(filtered)` otherwise.
+///
+/// Constraint semantics:
+/// - All `Extension` constraints OR together (file matches if ANY extension hits).
+///   They're split out up front so the per-item loop reads the OR predicate as a
+///   single short-circuit check, not as N AND-merged sub-constraints.
+/// - Every other constraint kind ANDs (file matches only if ALL hold). They're
+///   evaluated in order with short-circuit on first failure.
+pub(crate) fn apply_constraints<'a, T: Constrainable + Sync>(
+    items: &'a [T],
+    constraints: &[Constraint<'_>],
+    base_arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> Option<Vec<&'a T>> {
+    if constraints.is_empty() {
+        return None;
+    }
+    let plan = ConstraintPlan::build(constraints, items, base_arena, overflow_arena);
+    Some(plan.run(items, base_arena, overflow_arena))
+}
+
+#[cfg(feature = "zlob")]
+pub(crate) type GlobPattern = zlob::ZlobPattern;
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) type GlobPattern = globset::GlobMatcher;
+
+/// How `Constraint::Glob` is evaluated for each item.
+enum GlobStrategy {
+    /// No Glob constraint present.
+    None,
+    /// Pure-glob workload (no Extension filter to reject items first).
+    /// Batch all paths through zlob/globset once; per-item check is a Vec<bool> lookup.
+    Prepass(Vec<Vec<bool>>),
+    /// Mixed workload (Extension filter present). Compile patterns up front, then
+    /// only run them on items that survive the cheap Extension OR check.
+    /// `None` slot = compile failure -> never matches; preserves index alignment.
+    Inline(Vec<Option<GlobPattern>>),
+}
+
+/// Bundles preprocessed constraints for the per-item evaluator.
+pub(crate) struct ConstraintPlan<'q, 'c> {
+    /// OR semantics — file passes if ANY extension matches. Empty = no ext filter.
+    extensions: SmallVec<[&'q str; 8]>,
+    /// AND semantics — file passes only if ALL match.
+    rest: SmallVec<[&'c Constraint<'q>; 8]>,
+    glob: GlobStrategy,
+}
+
+pub(crate) struct ConstraintsBuffers {
+    fname: String,
+    path: String,
+}
+
+impl ConstraintsBuffers {
+    pub(crate) fn new() -> Self {
+        Self {
+            fname: String::with_capacity(64),
+            path: String::with_capacity(64),
+        }
+    }
+}
+
+impl<'q, 'c> ConstraintPlan<'q, 'c> {
+    pub(crate) fn build<T: Constrainable>(
+        constraints: &'c [Constraint<'q>],
+        items: &[T],
+        base_arena: ArenaPtr,
+        overflow_arena: ArenaPtr,
+    ) -> Self {
+        let mut extensions = SmallVec::new();
+        let mut rest: SmallVec<[&'c Constraint<'q>; 8]> = SmallVec::new();
+        for c in constraints {
+            match c {
+                Constraint::Extension(ext) => extensions.push(*ext),
+                _ => rest.push(c),
+            }
+        }
+        let has_pre_filter = !extensions.is_empty() || rest.iter().any(|&c| !is_glob_node(c));
+        let glob = build_glob_strategy(&rest, has_pre_filter, items, base_arena, overflow_arena);
+
+        Self {
+            extensions,
+            rest,
+            glob,
+        }
+    }
+
+    fn run<'a, T: Constrainable + Sync>(
+        &self,
+        items: &'a [T],
+        base_arean: ArenaPtr,
+        overflow_arena: ArenaPtr,
+    ) -> Vec<&'a T> {
+        if items.len() >= PAR_THRESHOLD {
+            use rayon::prelude::*;
+            items
+                .par_iter()
+                .enumerate()
+                .map_init(ConstraintsBuffers::new, |scratch, (i, item)| {
+                    self.matches(item, i, base_arean, overflow_arena, scratch)
+                        .then_some(item)
+                })
+                .flatten()
+                .collect()
+        } else {
+            let mut scratch = ConstraintsBuffers::new();
+            items
+                .iter()
+                .enumerate()
+                .filter_map(|(i, item)| {
+                    self.matches(item, i, base_arean, overflow_arena, &mut scratch)
+                        .then_some(item)
+                })
+                .collect()
+        }
+    }
+
+    #[inline]
+    pub(crate) fn matches<T: Constrainable>(
+        &self,
+        item: &T,
+        index: usize,
+        base_arena: ArenaPtr,
+        overflow_arena: ArenaPtr,
+        scratch: &mut ConstraintsBuffers,
+    ) -> bool {
+        let arena = if item.is_overflow() {
+            overflow_arena
+        } else {
+            base_arena
+        };
+
+        if !self.passes_extensions(item, arena, scratch) {
+            return false;
+        }
+
+        let mut glob_idx = 0;
+        self.rest.iter().all(|c| {
+            evaluate(
+                item,
+                index,
+                c,
+                &self.glob,
+                &mut glob_idx,
+                false,
+                arena,
+                scratch,
+            )
+        })
+    }
+
+    #[inline]
+    fn passes_extensions<T: Constrainable>(
+        &self,
+        item: &T,
+        arena: ArenaPtr,
+        scratch: &mut ConstraintsBuffers,
+    ) -> bool {
+        if self.extensions.is_empty() {
+            return true;
+        }
+        item.write_file_name(arena, &mut scratch.fname);
+        self.extensions
+            .iter()
+            .any(|ext| file_has_extension(&scratch.fname, ext))
+    }
+}
+
+#[inline]
+#[allow(clippy::too_many_arguments)]
+fn evaluate<T: Constrainable>(
+    item: &T,
+    index: usize,
+    constraint: &Constraint<'_>,
+    glob: &GlobStrategy,
+    glob_idx: &mut usize,
+    negate: bool,
+    arena: ArenaPtr,
+    scratch: &mut ConstraintsBuffers,
+) -> bool {
+    let raw = match constraint {
+        Constraint::Glob(_) => {
+            let m = match glob {
+                GlobStrategy::None => true,
+                GlobStrategy::Prepass(masks) => masks
+                    .get(*glob_idx)
+                    .and_then(|mask| mask.get(index).copied())
+                    .unwrap_or(false),
+                GlobStrategy::Inline(patterns) => {
+                    item.write_relative_path(arena, &mut scratch.path);
+                    patterns
+                        .get(*glob_idx)
+                        .and_then(|p| p.as_ref())
+                        .map(|p| compiled_matches(p, &scratch.path))
+                        .unwrap_or(false)
+                }
+            };
+            *glob_idx += 1;
+            m
+        }
+        // Reachable only via `Not(Extension(_))` — bare extensions are split out
+        // up front and handled in `passes_extensions`.
+        Constraint::Extension(ext) => {
+            item.write_file_name(arena, &mut scratch.fname);
+            file_has_extension(&scratch.fname, ext)
+        }
+        Constraint::PathSegment(segment) => {
+            item.write_relative_path(arena, &mut scratch.path);
+            path_contains_segment(&scratch.path, segment)
+        }
+        Constraint::FilePath(suffix) => {
+            item.write_relative_path(arena, &mut scratch.path);
+            path_ends_with_suffix(&scratch.path, suffix)
+        }
+        Constraint::Text(text) => {
+            // Only meaningful under negation (used as exclude filter).
+            item.write_relative_path(arena, &mut scratch.path);
+            find_case_insensitive_short(scratch.path.as_bytes(), text.as_bytes()).is_some()
+        }
+        Constraint::GitStatus(filter) => matches_git_status(item.git_status(), filter),
+        Constraint::Not(inner) => {
+            return evaluate(item, index, inner, glob, glob_idx, !negate, arena, scratch);
+        }
+        // Pass-throughs — handled at higher levels.
+        Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true,
+    };
+    if negate { !raw } else { raw }
+}
+
+#[inline]
+fn matches_git_status(status: Option<git2::Status>, filter: &GitStatusFilter) -> bool {
+    match (status, filter) {
+        (Some(s), GitStatusFilter::Modified) => is_modified_status(s),
+        (Some(s), GitStatusFilter::Untracked) => s.contains(git2::Status::WT_NEW),
+        (Some(s), GitStatusFilter::Staged) => s.intersects(
+            git2::Status::INDEX_NEW
+                | git2::Status::INDEX_MODIFIED
+                | git2::Status::INDEX_DELETED
+                | git2::Status::INDEX_RENAMED
+                | git2::Status::INDEX_TYPECHANGE,
+        ),
+        (Some(s), GitStatusFilter::Unmodified) => s.is_empty(),
+        (None, GitStatusFilter::Unmodified) => true,
+        (None, _) => false,
+    }
+}
+
+#[inline]
+#[cfg(feature = "zlob")]
+pub(crate) fn compiled_matches(p: &GlobPattern, path: &str) -> bool {
+    p.matches_default(path)
+}
+
+#[inline]
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn compiled_matches(p: &GlobPattern, path: &str) -> bool {
+    p.is_match(path)
+}
+
+/// Append indices (into `rels`) of paths matching `p`, in input order.
+/// zlob backend: ONE FFI call for the whole batch.
+#[cfg(feature = "zlob")]
+pub(crate) fn glob_matches_into(p: &GlobPattern, rels: &[&str], out: &mut Vec<usize>) {
+    match p.match_indices(rels, p.flags()) {
+        Ok(ix) => out.extend_from_slice(ix.as_slice()),
+        Err(e) => {
+            tracing::warn!(?e, "zlob batch match failed, falling back to per-path");
+            out.extend((0..rels.len()).filter(|&i| p.matches_default(rels[i])));
+        }
+    }
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn glob_matches_into(p: &GlobPattern, rels: &[&str], out: &mut Vec<usize>) {
+    out.extend((0..rels.len()).filter(|&i| p.is_match(rels[i])));
+}
+
+/// Decide between batch prepass and inline compiled patterns.
+///
+/// `has_pre_filter` = true when something cheaper than glob can reject items first
+/// (extensions OR non-glob constraints in `rest`). In that case inline pays glob
+/// cost only on survivors and beats prepass on every workload we benched. Pure-glob
+/// (no pre-filter) takes prepass — single batched zlob call beats N inline matches.
+fn build_glob_strategy<T: Constrainable>(
+    rest: &[&Constraint<'_>],
+    has_pre_filter: bool,
+    items: &[T],
+    arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> GlobStrategy {
+    if !contains_glob(rest) {
+        return GlobStrategy::None;
+    }
+    if has_pre_filter {
+        return GlobStrategy::Inline(compile_globs(rest));
+    }
+    let buf = PathBuffer::collect(items, arena, overflow_arena);
+    let path_refs = buf.as_strs();
+    GlobStrategy::Prepass(precompute_masks(rest, &path_refs))
+}
+
+/// `Glob` or `Not(Glob)` — the constraint kinds whose evaluation goes through
+/// the GlobStrategy. Everything else can pre-reject items before glob runs.
+fn is_glob_node(c: &Constraint<'_>) -> bool {
+    match c {
+        Constraint::Glob(_) => true,
+        Constraint::Not(inner) => is_glob_node(inner),
+        _ => false,
+    }
+}
+
+fn contains_glob(rest: &[&Constraint<'_>]) -> bool {
+    rest.iter().any(|c| is_glob_node(c))
+}
+
+/// Contiguous byte buffer holding every item's `relative_path`. Single allocation
+/// instead of N `String`s. On Windows the in-place pass folds `\\` -> `/` so the
+/// glob library sees a canonical separator.
+struct PathBuffer {
+    bytes: Vec<u8>,
+    offsets: Vec<(usize, usize)>,
+}
+
+impl PathBuffer {
+    fn collect<T: Constrainable>(items: &[T], arena: ArenaPtr, overflow_arena: ArenaPtr) -> Self {
+        let mut bytes = Vec::<u8>::new();
+        let mut offsets = Vec::with_capacity(items.len());
+        let mut tmp = String::with_capacity(64);
+        for item in items {
+            let item_arena = if item.is_overflow() {
+                overflow_arena
+            } else {
+                arena
+            };
+            let start = bytes.len();
+            item.write_relative_path(item_arena, &mut tmp);
+            bytes.extend_from_slice(tmp.as_bytes());
+            offsets.push((start, bytes.len() - start));
+        }
+        Self { bytes, offsets }
+    }
+
+    fn as_strs(&self) -> Vec<&str> {
+        self.offsets
+            .iter()
+            .map(|&(off, len)| unsafe {
+                std::str::from_utf8_unchecked(&self.bytes[off..off + len])
+            })
+            .collect()
+    }
+}
+
+fn precompute_masks(rest: &[&Constraint<'_>], paths: &[&str]) -> Vec<Vec<bool>> {
+    let mut out = Vec::new();
+    for c in rest {
+        walk_globs(c, &mut |pattern| {
+            out.push(match_glob_pattern(pattern, paths))
+        });
+    }
+    out
+}
+
+fn compile_globs(rest: &[&Constraint<'_>]) -> Vec<Option<GlobPattern>> {
+    let mut out = Vec::new();
+    for c in rest {
+        walk_globs(c, &mut |pattern| out.push(compile_one(pattern)));
+    }
+    out
+}
+
+/// Visit every Glob (including ones nested under Not) in constraint walk order.
+/// Order matters: `glob_idx` in the per-item evaluator increments by one per Glob node.
+fn walk_globs<F: FnMut(&str)>(c: &Constraint<'_>, f: &mut F) {
+    match c {
+        Constraint::Glob(p) => f(p),
+        Constraint::Not(inner) => walk_globs(inner, f),
+        _ => {}
+    }
+}
+
+#[cfg(feature = "zlob")]
+pub(crate) fn compile_one(pattern: &str) -> Option<GlobPattern> {
+    zlob::ZlobPattern::compile(pattern, zlob::ZlobFlags::RECOMMENDED).ok()
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn compile_one(pattern: &str) -> Option<GlobPattern> {
+    globset::Glob::new(pattern)
+        .ok()
+        .map(|g| g.compile_matcher())
+}
+
+/// Build a `paths.len()`-sized bitmap. Vec<bool> beats AHashSet ~2× in the per-item
+/// filter loop — no hashing, plain array indexing, sequential prefetcher-friendly.
+#[cfg(feature = "zlob")]
+fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec<bool> {
+    let mut mask = vec![false; paths.len()];
+    let Ok(hits) = zlob::zlob_match_paths_indices(pattern, paths, zlob::ZlobFlags::RECOMMENDED)
+    else {
+        return mask;
+    };
+    for i in hits.to_iter() {
+        if i < mask.len() {
+            mask[i] = true;
+        }
+    }
+    mask
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec<bool> {
+    let mut mask = vec![false; paths.len()];
+    let Ok(glob) = globset::Glob::new(pattern) else {
+        return mask;
+    };
+    let matcher = glob.compile_matcher();
+    if paths.len() >= PAR_THRESHOLD {
+        use rayon::prelude::*;
+        mask.par_iter_mut()
+            .zip(paths.par_iter())
+            .for_each(|(slot, p)| *slot = matcher.is_match(p));
+    } else {
+        for (slot, p) in mask.iter_mut().zip(paths.iter()) {
+            *slot = matcher.is_match(p);
+        }
+    }
+    mask
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Clone)]
+    struct TestItem {
+        relative_path: &'static str,
+        file_name: &'static str,
+    }
+
+    impl Constrainable for TestItem {
+        fn write_file_name(&self, _arena: ArenaPtr, out: &mut String) {
+            out.clear();
+            out.push_str(self.file_name);
+        }
+
+        fn write_relative_path(&self, _arena: ArenaPtr, out: &mut String) {
+            out.clear();
+            out.push_str(self.relative_path);
+        }
+
+        fn git_status(&self) -> Option<git2::Status> {
+            None
+        }
+
+        fn is_overflow(&self) -> bool {
+            false
+        }
+    }
+
+    #[test]
+    fn test_file_has_extension() {
+        assert!(file_has_extension("file.rs", "rs"));
+        assert!(file_has_extension("file.RS", "rs")); // case-insensitive
+        assert!(file_has_extension("file.test.rs", "rs"));
+        assert!(file_has_extension("a.rs", "rs"));
+
+        assert!(!file_has_extension("file.tsx", "rs"));
+        assert!(!file_has_extension("rs", "rs")); // too short
+        assert!(!file_has_extension(".rs", "rs")); // just extension
+        assert!(!file_has_extension("file.rsx", "rs")); // different extension
+        assert!(!file_has_extension("filers", "rs")); // no dot
+    }
+
+    #[test]
+    fn test_path_contains_segment() {
+        // Segment at start
+        assert!(path_contains_segment("src/lib.rs", "src"));
+        assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive
+
+        // Segment in middle
+        assert!(path_contains_segment("app/src/lib.rs", "src"));
+        assert!(path_contains_segment("app/SRC/lib.rs", "src"));
+
+        // Multiple levels
+        assert!(path_contains_segment("core/workflow/src/main.rs", "src"));
+        assert!(path_contains_segment(
+            "core/workflow/src/main.rs",
+            "workflow"
+        ));
+        assert!(path_contains_segment("core/workflow/src/main.rs", "core"));
+
+        // Should not match partial segments
+        assert!(!path_contains_segment("source/lib.rs", "src"));
+        assert!(!path_contains_segment("mysrc/lib.rs", "src"));
+
+        // Should not match filename
+        assert!(!path_contains_segment("lib/src", "src"));
+
+        // Multi-segment constraints
+        assert!(path_contains_segment(
+            "libswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "foo/libswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "foo/LibSwscale/AArch64/input.S",
+            "libswscale/aarch64"
+        )); // case-insensitive
+        assert!(!path_contains_segment(
+            "xlibswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        )); // partial match at start
+        assert!(!path_contains_segment(
+            "foo/libswscale/aarch64x/input.S",
+            "libswscale/aarch64"
+        )); // partial match at end
+        assert!(path_contains_segment(
+            "crates/fff-core/src/grep.rs",
+            "fff-core/src"
+        ));
+
+        // Edge cases
+        assert!(!path_contains_segment("", "src"));
+        assert!(!path_contains_segment("src", "src")); // no trailing slash
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn test_path_contains_segment_accepts_backslash() {
+        assert!(path_contains_segment("src\\lib.rs", "src"));
+        assert!(path_contains_segment(
+            "app\\modules\\src\\services\\x.lua",
+            "src"
+        ));
+        assert!(path_contains_segment("app\\SRC\\x.lua", "src"));
+
+        assert!(path_contains_segment(
+            "foo\\libswscale\\aarch64\\input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "crates\\fff-core\\src\\grep.rs",
+            "fff-core/src"
+        ));
+
+        assert!(!path_contains_segment("mysrc\\lib.rs", "src"));
+        assert!(!path_contains_segment(
+            "xlibswscale\\aarch64\\in.S",
+            "libswscale/aarch64"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix() {
+        // Exact match
+        assert!(path_ends_with_suffix(
+            "libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Suffix match at / boundary
+        assert!(path_ends_with_suffix(
+            "foo/libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Deep nesting
+        assert!(path_ends_with_suffix(
+            "a/b/c/libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // No boundary — partial directory name
+        assert!(!path_ends_with_suffix(
+            "xlibswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Case insensitive
+        assert!(path_ends_with_suffix(
+            "foo/LibSwscale/Input.C",
+            "libswscale/input.c"
+        ));
+
+        // Single file name
+        assert!(path_ends_with_suffix("input.c", "input.c"));
+        assert!(!path_ends_with_suffix("xinput.c", "input.c"));
+
+        // Suffix longer than path
+        assert!(!path_ends_with_suffix("input.c", "foo/input.c"));
+
+        // Simple path
+        assert!(path_ends_with_suffix("src/main.rs", "src/main.rs"));
+        assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs"));
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn test_path_ends_with_suffix_accepts_backslash() {
+        assert!(path_ends_with_suffix(
+            "app\\modules\\src\\services\\handler.lua",
+            "services/handler.lua"
+        ));
+        assert!(path_ends_with_suffix(
+            "foo\\libswscale\\input.c",
+            "libswscale/input.c"
+        ));
+        assert!(!path_ends_with_suffix(
+            "xlibswscale\\input.c",
+            "libswscale/input.c"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_does_not_panic_on_unicode_suffix() {
+        assert!(!path_ends_with_suffix("유니코드_파일_테스트.csv", "트.c"));
+        assert!(path_ends_with_suffix(
+            "data/유니코드_파일_테스트.csv",
+            "유니코드_파일_테스트.csv"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_unicode_apostrophe_mismatch() {
+        assert!(!path_ends_with_suffix(
+            "dir/\u{2019}bar/file.txt",
+            "'bar/file.txt"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_unicode_space_mismatch() {
+        assert!(!path_ends_with_suffix(
+            "dir/\u{202f}am/file.txt",
+            " am/file.txt"
+        ));
+    }
+
+    #[test]
+    fn test_path_contains_segment_does_not_panic_on_unicode_segment() {
+        assert!(!path_contains_segment("문서/notes.txt", "문x"));
+        assert!(path_contains_segment("프로젝트/문서/notes.txt", "문서"));
+    }
+
+    #[test]
+    fn test_path_contains_segment_unicode_no_panic() {
+        assert!(!path_contains_segment(
+            "Library/Cloud/Project\u{2019}s Folder/books.ttl",
+            "Project's Folder"
+        ));
+    }
+
+    #[test]
+    fn test_file_has_extension_unicode_no_panic() {
+        assert!(!file_has_extension("cat\u{00e9}.rs", "s"));
+    }
+
+    #[test]
+    fn test_file_has_extension_unicode_filename() {
+        assert!(file_has_extension("운영-가이드.md", "md"));
+        assert!(file_has_extension("테스트.csv", "csv"));
+        assert!(!file_has_extension("테스트.csv", "md"));
+    }
+
+    #[test]
+    fn test_apply_constraints_file_path_with_unicode_suffix() {
+        let arena_ptr = ArenaPtr(std::ptr::null());
+
+        let item = TestItem {
+            relative_path: "data/유니코드_파일_테스트.csv",
+            file_name: "유니코드_파일_테스트.csv",
+        };
+
+        let exact = [Constraint::FilePath("유니코드_파일_테스트.csv")];
+        let mismatch = [Constraint::FilePath("트.c")];
+
+        let exact_items = [item.clone()];
+        let exact_matches = apply_constraints(&exact_items, &exact, arena_ptr, arena_ptr)
+            .expect("constraints applied");
+        assert_eq!(exact_matches.len(), 1);
+
+        let mismatch_items = [item];
+        let mismatch_matches = apply_constraints(&mismatch_items, &mismatch, arena_ptr, arena_ptr)
+            .expect("constraints applied");
+        assert!(mismatch_matches.is_empty());
+    }
+
+    #[test]
+    fn test_unicode_path_no_panic_real_korean_cases() {
+        // Real Korean paths that caused panics
+        let path1 = "Downloads/(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv";
+        let path2 = "hermes-agent-lecture-materials/세부_커리큘럼_최종.csv";
+        let path3 = "projects/fastcampus-hermes-agent-curriculum/chapters/part-02-Hermes-설치-및-기본-사용/section-02-doctor로-설치-상태-검증/research/03-fix가-자동-수정하는-것과-못하는-것.md";
+
+        // These must not panic regardless of segment/suffix used
+        assert!(!path_contains_segment(path1, "작성"));
+        assert!(!path_ends_with_suffix(path1, "작성.csv"));
+        assert!(!path_contains_segment(path2, "최종"));
+        assert!(!path_ends_with_suffix(path2, "최종.csv"));
+        assert!(!path_contains_segment(path3, "수정"));
+        assert!(!path_ends_with_suffix(path3, "것.md"));
+
+        // Positive cases should still work
+        assert!(path_contains_segment(
+            path2,
+            "hermes-agent-lecture-materials"
+        ));
+        assert!(path_ends_with_suffix(
+            path1,
+            "(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv"
+        ));
+        assert!(path_ends_with_suffix(path2, "세부_커리큘럼_최종.csv"));
+    }
+
+    #[test]
+    fn test_negated_glob_excludes_matching_files() {
+        let arena_ptr = ArenaPtr(std::ptr::null());
+
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "src/lib.ts",
+                file_name: "lib.ts",
+            },
+            TestItem {
+                relative_path: "include/fff.h",
+                file_name: "fff.h",
+            },
+        ];
+
+        // Not(Glob("**/*.rs")) should exclude .rs files
+        let constraints = vec![Constraint::Not(Box::new(Constraint::Glob("**/*.rs")))];
+        let result = apply_constraints(&items, &constraints, arena_ptr, arena_ptr).unwrap();
+        let paths: Vec<&str> = result.iter().map(|i| i.relative_path).collect();
+        assert!(
+            !paths.contains(&"src/main.rs"),
+            "rs file should be excluded"
+        );
+        assert!(paths.contains(&"src/lib.ts"), "ts file should be included");
+        assert!(
+            paths.contains(&"include/fff.h"),
+            "h file should be included"
+        );
+    }
+
+    #[test]
+    fn test_inline_glob_path_matches_prepass() {
+        // Mixed (extensions + glob) takes the inline-compiled path.
+        // Pure glob takes the prepass bitmap path. Both must give identical results.
+        let arena_ptr = ArenaPtr(std::ptr::null());
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "src/lib.ts",
+                file_name: "lib.ts",
+            },
+            TestItem {
+                relative_path: "tests/foo.rs",
+                file_name: "foo.rs",
+            },
+            TestItem {
+                relative_path: "docs/readme.md",
+                file_name: "readme.md",
+            },
+        ];
+
+        let mixed = vec![Constraint::Extension("rs"), Constraint::Glob("src/**")];
+        let mixed_paths: Vec<&str> = apply_constraints(&items, &mixed, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert_eq!(mixed_paths, vec!["src/main.rs"]);
+
+        let pure_glob = vec![Constraint::Glob("src/**")];
+        let glob_paths: Vec<&str> = apply_constraints(&items, &pure_glob, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert!(glob_paths.contains(&"src/main.rs"));
+        assert!(glob_paths.contains(&"src/lib.ts"));
+        assert_eq!(glob_paths.len(), 2);
+    }
+
+    #[test]
+    fn test_inline_negated_glob_with_extension() {
+        // Mixed Not(Glob) on inline path — exercise the negate=true branch in
+        // glob_matches_inline through the Not->Glob recursion.
+        let arena_ptr = ArenaPtr(std::ptr::null());
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "vendor/foo.rs",
+                file_name: "foo.rs",
+            },
+            TestItem {
+                relative_path: "vendor/foo.ts",
+                file_name: "foo.ts",
+            },
+        ];
+
+        let constraints = vec![
+            Constraint::Extension("rs"),
+            Constraint::Not(Box::new(Constraint::Glob("vendor/**"))),
+        ];
+        let paths: Vec<&str> = apply_constraints(&items, &constraints, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert_eq!(paths, vec!["src/main.rs"]);
+    }
+}

crates/fff-core/src/index/mod.rs

@@ -0,0 +1,11 @@
+#[doc(hidden)] // for bench
+pub mod bigram_filter;
+pub(crate) use bigram_filter::*;
+
+mod bigram_query;
+pub use bigram_query::*;
+
+mod candidates;
+pub(crate) use candidates::*;
+
+pub mod constraints;

crates/fff-core/src/lib.rs

@@ -1,10 +1,13 @@
 //! # FFF Search — High-performance file finder core
 //!
-//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff.nvim).
+//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff).
 //! It includes filesystem indexing with real-time watching, fuzzy matching powered
 //! by [frizbee](https://docs.rs/neo_frizbee), frecency scoring backed by LMDB,
 //! and multi-mode grep search.
 //!
+//! > [!Important performance information]  
+//! > For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine.
+//!
 //! ## Architecture
 //!
 //! - [`file_picker::FilePicker`] — Main entry point. Indexes a directory tree in a
@@ -17,10 +20,13 @@
 //! - [`grep`] — Live grep search supporting regex, plain-text, and fuzzy modes
 //!   with optional constraint filtering.
 //! - [`git`] — Git status caching and repository detection.
+//! - [`watch`] — Client-facing filesystem watch subscriptions: glob, exact
+//!   path, or directory subtree with normalized batch delivery
+//!   (see [`SharedFilePicker::watch`]).
 //!
 //! ## Shared State
 //!
-//! [`SharedPicker`], [`SharedFrecency`], and [`SharedQueryTracker`] are
+//! [`SharedFilePicker`], [`SharedFrecency`], and [`SharedQueryTracker`] are
 //! newtype wrappers around `Arc<RwLock<Option<T>>>` for thread-safe shared
 //! access. They provide `read()` / `write()` methods with built-in error
 //! conversion and convenience helpers like `wait_for_scan()`.
@@ -33,10 +39,10 @@
 //! use fff_search::query_tracker::QueryTracker;
 //! use fff_search::{
 //!     FFFMode, FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser,
-//!     SharedFrecency, SharedPicker, SharedQueryTracker,
+//!     SharedFrecency, SharedFilePicker, SharedQueryTracker,
 //! };
 //!
-//! let shared_picker = SharedPicker::default();
+//! let shared_picker = SharedFilePicker::default();
 //! let shared_frecency = SharedFrecency::default();
 //! let shared_query_tracker = SharedQueryTracker::default();
 //!
@@ -44,10 +50,10 @@
 //! std::fs::create_dir_all(&tmp).unwrap();
 //!
 //! // 1. Optionally initialize frecency and query tracker databases
-//! let frecency = FrecencyTracker::new(tmp.join("frecency"), false)?;
+//! let frecency = FrecencyTracker::open(tmp.join("frecency"))?;
 //! shared_frecency.init(frecency)?;
 //!
-//! let query_tracker = QueryTracker::new(tmp.join("queries"), false)?;
+//! let query_tracker = QueryTracker::open(tmp.join("queries"))?;
 //! shared_query_tracker.init(query_tracker)?;
 //!
 //! // 2. Init the file picker (spawns background scan + watcher)
@@ -91,66 +97,78 @@
 //! # Ok::<(), Box<dyn std::error::Error>>(())
 //! ```
 
-mod background_watcher;
-mod bigram_filter;
-pub mod bigram_query;
-mod constraints;
-mod db_healthcheck;
-mod error;
-mod score;
-mod sort_buffer;
-// this is pub only for benchmarks
-pub mod case_insensitive_memmem;
+#[cfg(not(any(feature = "ripgrep", feature = "zlob")))]
+compile_error!(
+    "fff-search requires either the `ripgrep` (default) or `zlob` feature. \
+     Enable one, e.g. `--features ripgrep` or `--features zlob`."
+);
 
-pub(crate) mod simd_path;
+/// Primary entry points with thread-safe [`SharedFilePicker`](shared::FilePicker) instance
+pub mod shared;
+pub use shared::*;
 
-/// Core file picker: filesystem indexing, background watching, and fuzzy search.
-///
+/// Core file picker single thread: filesystem indexing, background watching, and fuzzy search.
 /// See [`FilePicker`](file_picker::FilePicker) for the main entry point.
 pub mod file_picker;
+pub use file_picker::*;
 
-/// Frecency (frequency + recency) database for file access scoring.
-///
-/// Backed by LMDB for persistent, crash-safe storage.
-pub mod frecency;
+/// Database-backed persistence: frecency, query history, LMDB plumbing.
+pub mod dbs;
+pub use dbs::*;
 
 /// Git status caching and repository detection utilities.
 pub mod git;
 
 /// Live grep search with regex, plain-text, and fuzzy matching modes.
-///
-/// Supports constraint filtering (file extensions, path segments, globs)
-/// and parallel execution via rayon.
 pub mod grep;
+pub use grep::*;
 
-/// Tracing/logging initialization and panic hook setup.
+/// Tracing/logging initialization
 pub mod log;
 
-/// Path manipulation utilities: cross platform canonicalization, tilde expansion, and
-/// directory distance penalties for search scoring.
+/// Various path utils might be handy for you to work with fff paths
 pub mod path_utils;
 
-/// Search query history tracker for combo-boost scoring.
-///
-/// Records which files a user selects for each query, enabling the scorer
-/// to boost files that were previously chosen for similar searches.
-pub mod query_tracker;
-
 /// Core data types shared across the crate.
 pub mod types;
-
-mod ignore;
-/// Thread-safe shared handles for [`FilePicker`], [`FrecencyTracker`],
-/// and [`QueryTracker`].
-pub mod shared;
-
-pub use bigram_filter::*;
-pub use db_healthcheck::{DbHealth, DbHealthChecker};
-pub use error::{Error, Result};
-pub use fff_query_parser::*;
-pub use file_picker::*;
-pub use frecency::*;
-pub use grep::*;
-pub use query_tracker::*;
-pub use shared::*;
 pub use types::*;
+
+pub mod constants;
+
+/// Watcher rescan request accounting.
+pub mod rescan_stats;
+pub use rescan_stats::{RESCAN_STATS_ENABLED, RescanReason, RescanStats};
+
+mod rescan_throttle;
+
+// ==================================
+// these are public only for benchmarks, no backward compatibility guaranteed
+#[doc(hidden)]
+pub use index::bigram_filter;
+#[doc(hidden)]
+pub mod simd_string_utils;
+// ==================================
+
+mod error;
+mod git_status_worker;
+mod ignore;
+mod scan;
+mod score;
+mod sort_buffer;
+
+pub(crate) mod index;
+pub(crate) mod parallelism;
+pub(crate) mod simd_path;
+pub(crate) mod stable_vec;
+pub(crate) mod walk;
+
+/// Filesystem watch subscriptions with glob filtering and batched delivery,
+/// plus the background OS watcher.
+#[path = "watcher/mod.rs"]
+pub mod watch;
+pub use watch::{WatchEvent, WatchEventKind, WatchId, WatchOptions};
+
+// fff error
+pub use error::{Error, Result};
+
+pub use fff_query_parser::*;

crates/fff-core/src/log.rs

@@ -1,31 +1,20 @@
-//! Shared logging utilities for FFF crates.
-//!
-//! Provides file-based tracing initialization and crash handlers (panic hook
-//! + SIGSEGV signal handler) that write diagnostics to both stderr and the
-//! configured log file.
-
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
 use tracing_appender::non_blocking;
 use tracing_subscriber::fmt::format::FmtSpan;
 use tracing_subscriber::{EnvFilter, fmt, prelude::*};
 
-static TRACING_INITIALIZED: std::sync::OnceLock<tracing_appender::non_blocking::WorkerGuard> =
-    std::sync::OnceLock::new();
-
-static CRASH_HANDLERS_INSTALLED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
-
-/// The log file path set by `init_tracing`. Crash handlers append to this file.
-static LOG_FILE_PATH: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
+// Set once on first init_tracing; doubles as the init-once gate.
+static LOG_FILE_PATH: OnceLock<PathBuf> = OnceLock::new();
+static CRASH_HOOKS: OnceLock<()> = OnceLock::new();
 
 fn write_crash_report(header: &str, body: &str) {
     let msg = format!(
-        "\n=== CRASH {} ===\n{}\n=== CRASH END {} ===\n",
+        "\n=== CRASH (this might NOT BE fff related) {} ===\n{}\n=== CRASH END {} ===\n",
         header, body, header
     );
-
     let _ = std::io::Write::write_all(&mut std::io::stderr(), msg.as_bytes());
-
     if let Some(path) = LOG_FILE_PATH.get() {
         let _ = std::fs::OpenOptions::new()
             .create(true)
@@ -35,54 +24,99 @@ fn write_crash_report(header: &str, body: &str) {
     }
 }
 
-extern "C" fn sigsegv_handler(sig: libc::c_int) {
-    let bt = std::backtrace::Backtrace::force_capture();
-    write_crash_report("SIGSEGV", &format!("signal {}\n{}", sig, bt));
+// SIGSEGV handler writes a banner to a pre-opened fd (open(2) inside a signal
+// handler is unsafe due to path-resolution allocs). Unix only.
+#[cfg(unix)]
+mod sigsegv {
+    use std::os::fd::IntoRawFd;
+    use std::path::Path;
+    use std::sync::atomic::{AtomicI32, Ordering};
 
-    unsafe {
-        libc::signal(sig, libc::SIG_DFL);
-        libc::raise(sig);
+    static LOG_FD: AtomicI32 = AtomicI32::new(-1);
+
+    // Must `create(true)` — this runs before init_tracing opens/creates the
+    // writer file, so an append-only open on a non-existent path silently
+    // fails, LOG_FD stays -1, and the SIGSEGV banner never reaches the log.
+    pub fn set_log_fd(path: &Path) {
+        if let Ok(file) = std::fs::OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(path)
+        {
+            let prev = LOG_FD.swap(file.into_raw_fd(), Ordering::Relaxed);
+            if prev >= 0 {
+                unsafe { libc::close(prev) };
+            }
+        }
+    }
+
+    // Body must be async-signal-safe: write(2), atomic load, signal(2). Nothing else.
+    fn handler(_info: &libc::siginfo_t) {
+        const BANNER: &[u8] = b"\n=== CRASH SIGSEGV (fff) ===\n\
+            fff.nvim's rust extension hit a segfault and is about to die.\n\
+            Please file the bug at https://github.com/dmtrKovalenko/fff/issues with this banner attached.\n\
+            === CRASH END SIGSEGV ===\n";
+        unsafe {
+            libc::write(2, BANNER.as_ptr().cast(), BANNER.len());
+            let log_fd = LOG_FD.load(Ordering::Relaxed);
+            if log_fd >= 0 {
+                libc::write(log_fd, BANNER.as_ptr().cast(), BANNER.len());
+            }
+            // Reset to default so handler return → kernel kills us instead of
+            // re-running the faulting instruction in an infinite loop.
+            libc::signal(libc::SIGSEGV, libc::SIG_DFL);
+        }
+    }
+
+    pub fn install() {
+        // signal-hook-registry chains to LuaJIT's prior handler automatically.
+        unsafe {
+            let _ = signal_hook_registry::register_unchecked(libc::SIGSEGV, handler);
+        }
     }
 }
 
-/// Install both the panic hook and the SIGSEGV signal handler.
+#[cfg(not(unix))]
+mod sigsegv {
+    use std::path::Path;
+    pub fn set_log_fd(_path: &Path) {}
+    pub fn install() {}
+}
+
 pub fn install_panic_hook() {
-    CRASH_HANDLERS_INSTALLED.get_or_init(|| {
-        let default_panic = std::panic::take_hook();
-        std::panic::set_hook(Box::new(move |panic_info| {
-            let message = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
-                s.to_string()
-            } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
-                s.clone()
-            } else {
-                "Unknown panic payload".to_string()
-            };
+    CRASH_HOOKS.get_or_init(install_crash_hooks);
+}
 
-            let location = panic_info
-                .location()
-                .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
-                .unwrap_or_else(|| "unknown location".to_string());
+fn install_crash_hooks() {
+    let default_panic = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |panic_info| {
+        let message = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            s.to_string()
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            s.clone()
+        } else {
+            "Unknown panic payload".to_string()
+        };
 
-            tracing::error!(
-                panic.message = %message,
-                panic.location = %location,
-                "PANIC occurred in FFF"
-            );
+        let location = panic_info
+            .location()
+            .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
+            .unwrap_or_else(|| "unknown location".to_string());
 
-            write_crash_report(
-                "RUST PANIC",
-                &format!("Message: {}\nLocation: {}", message, location),
-            );
-            default_panic(panic_info);
-        }));
+        tracing::error!(
+            panic.message = %message,
+            panic.location = %location,
+            "PANIC occurred in FFF"
+        );
 
-        unsafe {
-            libc::signal(
-                libc::SIGSEGV,
-                sigsegv_handler as *const () as libc::sighandler_t,
-            );
-        }
-    });
+        write_crash_report(
+            "RUST PANIC",
+            &format!("Message: {}\nLocation: {}", message, location),
+        );
+        default_panic(panic_info);
+    }));
+
+    sigsegv::install();
 }
 
 /// Parse a log level string into a `tracing::Level`.
@@ -97,54 +131,151 @@ pub fn parse_log_level(level: Option<&str>) -> tracing::Level {
     }
 }
 
-/// Initialize tracing with a single log file.
-pub fn init_tracing(log_file_path: &str, log_level: Option<&str>) -> Result<String, io::Error> {
-    let log_path = Path::new(log_file_path);
-    if let Some(parent) = log_path.parent() {
-        std::fs::create_dir_all(parent)?;
+/// Default retention: how many prior nvim sessions' log files to keep.
+const DEFAULT_RETAIN_RUNS: usize = 20;
+
+pub fn generate_trace_id() -> String {
+    use std::sync::atomic::{AtomicU64, Ordering};
+    static TRACE_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_nanos() as u64)
+        .unwrap_or(0);
+
+    let pid = std::process::id() as u64;
+    let counter = TRACE_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+    // very simple hash functions helps to distinguish trace ids visually
+    let id = nanos ^ (pid.wrapping_mul(0x9E37_79B9_7F4A_7C15)) ^ (counter << 32);
+    format!("{:016x}", id)
+}
+
+pub fn trace_span(trace_id: &str, label: &'static str) -> tracing::Span {
+    tracing::info_span!("fff.trace", trace_id = trace_id, label = label)
+}
+
+fn unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+fn session_path_from_hint(hint: &Path) -> PathBuf {
+    let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff");
+    let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log");
+    let parent = hint.parent().unwrap_or_else(|| Path::new("."));
+    parent.join(format!(
+        "{stem}+{ts}+{pid}.{ext}",
+        ts = unix_secs(),
+        pid = std::process::id(),
+    ))
+}
+
+fn rotate_logs(dir: &Path, stem: &str, ext: &str, retain_runs: usize) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    let prefix = format!("{stem}+");
+    let suffix = format!(".{ext}");
+
+    let mut files: Vec<(std::time::SystemTime, PathBuf)> = entries
+        .filter_map(|res| {
+            let entry = res.ok()?;
+            let name = entry.file_name();
+            let name = name.to_str()?;
+            if !name.starts_with(&prefix) || !name.ends_with(&suffix) {
+                return None;
+            }
+
+            let mtime = entry.metadata().ok()?.modified().ok()?;
+            Some((mtime, entry.path()))
+        })
+        .collect();
+
+    if files.len() <= retain_runs {
+        return;
+    }
+    // Newest first, then drop everything past retain_runs.
+    files.sort_by_key(|(mtime, _)| std::cmp::Reverse(*mtime));
+    for (_, path) in files.into_iter().skip(retain_runs) {
+        let _ = std::fs::remove_file(path);
+    }
+}
+
+/// `log_file_path` is a path-shape hint. Each call writes a unique sibling
+/// `<stem>+<unix-secs>+<pid>.<ext>` so concurrent processes never collide.
+/// Returns the absolute path of the session file.
+pub fn init_tracing(
+    log_file_path: &str,
+    log_level: Option<&str>,
+    retain_runs: Option<usize>,
+) -> Result<String, io::Error> {
+    let hint = Path::new(log_file_path);
+    let session_dir = hint
+        .parent()
+        .unwrap_or_else(|| Path::new("."))
+        .to_path_buf();
+    std::fs::create_dir_all(&session_dir)?;
+
+    let session_path = session_path_from_hint(hint);
+
+    // First init wins; repeat callers no-op and return the original path.
+    if LOG_FILE_PATH.set(session_path.clone()).is_err() {
+        return Ok(LOG_FILE_PATH
+            .get()
+            .map(|p| p.to_string_lossy().into_owned())
+            .unwrap_or_default());
     }
 
-    let _ = LOG_FILE_PATH.set(log_path.to_path_buf());
+    sigsegv::set_log_fd(&session_path);
     install_panic_hook();
 
-    let file_appender = std::fs::OpenOptions::new()
+    let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff");
+    let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log");
+    rotate_logs(
+        &session_dir,
+        stem,
+        ext,
+        retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS),
+    );
+
+    let writer_file = std::fs::OpenOptions::new()
         .create(true)
-        .write(true)
-        .truncate(true) // truncates a file on restart (instead of appending)
-        .open(log_path)?;
+        .append(true)
+        .open(&session_path)?;
 
-    let level = parse_log_level(log_level);
+    // we intinionally leark the guard we don't ever want to stop logging
+    let (non_blocking_appender, guard) = non_blocking(writer_file);
+    Box::leak(Box::new(guard));
 
-    TRACING_INITIALIZED.get_or_init(|| {
-        let (non_blocking_appender, guard) = non_blocking(file_appender);
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            fmt::layer()
+                .with_writer(non_blocking_appender)
+                .with_target(true)
+                .with_thread_ids(false)
+                .with_thread_names(true)
+                .with_ansi(false)
+                .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
+        )
+        .with(
+            EnvFilter::builder()
+                .with_default_directive(parse_log_level(log_level).into())
+                .from_env_lossy(),
+        );
 
-        let subscriber = tracing_subscriber::registry()
-            .with(
-                fmt::layer()
-                    .with_writer(non_blocking_appender)
-                    .with_target(true)
-                    .with_thread_ids(false)
-                    .with_thread_names(false)
-                    .with_ansi(false)
-                    .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
-            )
-            .with(
-                EnvFilter::builder()
-                    .with_default_directive(level.into())
-                    .from_env_lossy(),
-            );
+    if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
+        eprintln!("Failed to set tracing subscriber: {}", e);
+    } else {
+        tracing::info!(
+            "FFF tracing initialized: {} (pid={}, retain_runs={})",
+            session_path.display(),
+            std::process::id(),
+            retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS),
+        );
+    }
 
-        if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
-            eprintln!("Failed to set tracing subscriber: {}", e);
-        } else {
-            tracing::info!(
-                "FFF tracing initialized with log file: {}",
-                log_path.display()
-            );
-        }
-
-        guard
-    });
-
-    Ok(log_file_path.to_string())
+    Ok(session_path.to_string_lossy().into_owned())
 }

crates/fff-core/src/parallelism.rs

@@ -0,0 +1,84 @@
+//! Dedicated rayon pools. The global pool spans every logical core, which
+//! oversubscribes asymmetric chips (Apple P+E): E-cores are ~2× slower and
+//! `open()` contends on a per-VFS lock past P-core count, so a larger pool is
+//! slower on file-heavy work.
+
+use std::sync::LazyLock;
+
+/// Dedicated thread pool for background work (scan, warmup, bigram build).
+pub static BACKGROUND_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    let total = std::thread::available_parallelism()
+        .map(|p| p.get())
+        .unwrap_or(4);
+
+    // Background work is mostly syscall-bound; halving parallelism leaves
+    // cores for search/UI at negligible throughput cost.
+    let bg_threads = (total / 2).max(2);
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(bg_threads)
+        .thread_name(|i| format!("fff-bg-{i}"))
+        .start_handler(|_| {
+            // QoS pin keeps workers on P-cores; the kernel otherwise drifts
+            // them to ~2× slower E-cores.
+            #[cfg(target_os = "macos")]
+            unsafe {
+                let _ = libc::pthread_set_qos_class_self_np(
+                    libc::qos_class_t::QOS_CLASS_USER_INITIATED,
+                    0,
+                );
+            }
+        })
+        .build()
+        .expect("failed to create background rayon pool")
+});
+
+/// Physical performance-core count via sysctl, falling back to logical cores.
+/// On a 12P+4E M4 Max, grep runs 16t=6.2s vs 13t=4.9s — fewer threads win.
+#[cfg(target_os = "macos")]
+fn performance_core_count() -> usize {
+    let mut count: libc::c_int = 0;
+    let mut size = std::mem::size_of::<libc::c_int>();
+    let name = c"hw.perflevel0.physicalcpu";
+    let ok = unsafe {
+        libc::sysctlbyname(
+            name.as_ptr(),
+            &mut count as *mut _ as *mut libc::c_void,
+            &mut size,
+            std::ptr::null_mut(),
+            0,
+        )
+    };
+    if ok == 0 && count > 0 {
+        count as usize
+    } else {
+        std::thread::available_parallelism()
+            .map(|p| p.get())
+            .unwrap_or(4)
+    }
+}
+
+/// Pool for grep content search: P-core sized and QoS-pinned on macOS, full
+/// parallelism elsewhere. Avoids E-core drag and VFS-lock contention.
+pub static SEARCH_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    #[cfg(target_os = "macos")]
+    let threads = performance_core_count();
+    #[cfg(not(target_os = "macos"))]
+    let threads = std::thread::available_parallelism()
+        .map(|p| p.get())
+        .unwrap_or(4);
+
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(threads)
+        .thread_name(|i| format!("fff-search-{i}"))
+        .start_handler(|_| {
+            #[cfg(target_os = "macos")]
+            unsafe {
+                let _ = libc::pthread_set_qos_class_self_np(
+                    libc::qos_class_t::QOS_CLASS_USER_INITIATED,
+                    0,
+                );
+            }
+        })
+        .build()
+        .expect("failed to create search rayon pool")
+});

crates/fff-core/src/path_utils.rs

@@ -10,6 +10,56 @@ pub fn canonicalize(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
     std::fs::canonicalize(path)
 }
 
+/// The index stores relative paths with `/` on every platform. These helpers
+/// convert between that canonical form and the OS-native separator, and are
+/// no-ops on non-Windows where `/` is already native.
+
+/// Fold a relative path to the canonical `/` form (no-op off Windows).
+#[cfg(windows)]
+pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> {
+    if rel.contains('\\') {
+        std::borrow::Cow::Owned(rel.replace('\\', "/"))
+    } else {
+        std::borrow::Cow::Borrowed(rel)
+    }
+}
+
+#[cfg(not(windows))]
+#[inline]
+pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> {
+    std::borrow::Cow::Borrowed(rel)
+}
+
+/// Rewrite canonical `/` bytes to the OS-native separator in place (no-op off
+/// Windows). Used at OS/state boundaries (absolute-path reconstruction).
+#[cfg(windows)]
+#[inline]
+pub fn nativize_slashes_in_place(bytes: &mut [u8]) {
+    for b in bytes {
+        if *b == b'/' {
+            *b = b'\\';
+        }
+    }
+}
+
+#[cfg(not(windows))]
+#[inline]
+pub fn nativize_slashes_in_place(_bytes: &mut [u8]) {}
+
+/// Git requires a normalized forward-slashed paths on windows
+#[cfg(windows)]
+pub fn normalize(path: PathBuf) -> PathBuf {
+    let as_str = path.to_string_lossy();
+    let with_backslashes: String = as_str.replace('/', "\\");
+    let buf = PathBuf::from(with_backslashes);
+    dunce::canonicalize(&buf).unwrap_or(buf)
+}
+
+#[cfg(not(windows))]
+pub fn normalize(path: PathBuf) -> PathBuf {
+    path
+}
+
 #[cfg(windows)]
 pub fn expand_tilde(path: &str) -> PathBuf {
     return PathBuf::from(path);

crates/fff-core/src/rescan_stats.rs

@@ -0,0 +1,215 @@
+#[cfg(rescan_stats)]
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// Whether rescan accounting is compiled in.
+pub const RESCAN_STATS_ENABLED: bool = cfg!(rescan_stats);
+
+/// Cause recorded for a filesystem rescan request.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum RescanReason {
+    /// Requested through the public API (refresh, directory change).
+    Explicit,
+    /// The kernel dropped events and asked us to re-read the subtree.
+    KernelEventLoss,
+    /// A `.gitignore`/`.ignore` changed, so the cached ignore rules are stale.
+    IgnoreFileChanged,
+    /// A single debounce batch touched more paths than we apply incrementally.
+    EventBatchOverflow,
+    /// The picker refused an incremental insert/update.
+    IndexUpdateRejected,
+    /// The post-scan overflow region ran out of slots.
+    OverflowCapacity,
+}
+
+impl RescanReason {
+    pub const ALL: [RescanReason; 6] = [
+        RescanReason::Explicit,
+        RescanReason::KernelEventLoss,
+        RescanReason::IgnoreFileChanged,
+        RescanReason::EventBatchOverflow,
+        RescanReason::IndexUpdateRejected,
+        RescanReason::OverflowCapacity,
+    ];
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            RescanReason::Explicit => "explicit",
+            RescanReason::KernelEventLoss => "kernel_event_loss",
+            RescanReason::IgnoreFileChanged => "ignore_file_changed",
+            RescanReason::EventBatchOverflow => "event_batch_overflow",
+            RescanReason::IndexUpdateRejected => "index_update_rejected",
+            RescanReason::OverflowCapacity => "overflow_capacity",
+        }
+    }
+
+    const fn slot(self) -> usize {
+        match self {
+            RescanReason::Explicit => 0,
+            RescanReason::KernelEventLoss => 1,
+            RescanReason::IgnoreFileChanged => 2,
+            RescanReason::EventBatchOverflow => 3,
+            RescanReason::IndexUpdateRejected => 4,
+            RescanReason::OverflowCapacity => 5,
+        }
+    }
+}
+
+impl std::fmt::Display for RescanReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
+/// Snapshot of rescan requests grouped by reason.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct RescanStats {
+    pub total: usize,
+    /// Requests suppressed during the cooldown.
+    pub throttled: usize,
+    counts: [usize; RescanReason::ALL.len()],
+    throttled_counts: [usize; RescanReason::ALL.len()],
+}
+
+impl RescanStats {
+    pub fn count(&self, reason: RescanReason) -> usize {
+        self.counts[reason.slot()]
+    }
+
+    pub fn count_throttled(&self, reason: RescanReason) -> usize {
+        self.throttled_counts[reason.slot()]
+    }
+
+    /// Admitted requests originating from watcher fallbacks.
+    pub fn watcher_triggered(&self) -> usize {
+        self.total - self.count(RescanReason::Explicit)
+    }
+
+    /// Per-reason delta against an earlier snapshot.
+    pub fn since(&self, earlier: &RescanStats) -> RescanStats {
+        let mut counts = [0usize; RescanReason::ALL.len()];
+        let mut throttled_counts = [0usize; RescanReason::ALL.len()];
+        for slot in 0..RescanReason::ALL.len() {
+            counts[slot] = self.counts[slot].saturating_sub(earlier.counts[slot]);
+            throttled_counts[slot] =
+                self.throttled_counts[slot].saturating_sub(earlier.throttled_counts[slot]);
+        }
+
+        RescanStats {
+            total: self.total.saturating_sub(earlier.total),
+            throttled: self.throttled.saturating_sub(earlier.throttled),
+            counts,
+            throttled_counts,
+        }
+    }
+}
+
+impl std::fmt::Display for RescanStats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} rescan(s)", self.total)?;
+        let mut first = true;
+        for reason in RescanReason::ALL {
+            let count = self.count(reason);
+            if count == 0 {
+                continue;
+            }
+            f.write_str(if first { " [" } else { ", " })?;
+            write!(f, "{reason}={count}")?;
+            first = false;
+        }
+        if !first {
+            f.write_str("]")?;
+        }
+        if self.throttled > 0 {
+            write!(f, ", {} throttled", self.throttled)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(rescan_stats)]
+#[derive(Default)]
+pub(crate) struct RescanCounters {
+    counters: [AtomicUsize; RescanReason::ALL.len()],
+    throttled: [AtomicUsize; RescanReason::ALL.len()],
+}
+
+#[cfg(rescan_stats)]
+impl RescanCounters {
+    pub(crate) fn record(&self, reason: RescanReason) {
+        self.counters[reason.slot()].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn record_throttled(&self, reason: RescanReason) {
+        self.throttled[reason.slot()].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn snapshot(&self) -> RescanStats {
+        let mut stats = RescanStats::default();
+        for reason in RescanReason::ALL {
+            let count = self.counters[reason.slot()].load(Ordering::Relaxed);
+            stats.counts[reason.slot()] = count;
+            stats.total += count;
+
+            let throttled = self.throttled[reason.slot()].load(Ordering::Relaxed);
+            stats.throttled_counts[reason.slot()] = throttled;
+            stats.throttled += throttled;
+        }
+        stats
+    }
+
+    pub(crate) fn reset(&self) {
+        for counter in self.counters.iter().chain(self.throttled.iter()) {
+            counter.store(0, Ordering::Relaxed);
+        }
+    }
+}
+
+// Release builds retain the API without counter storage.
+#[cfg(not(rescan_stats))]
+#[derive(Default)]
+pub(crate) struct RescanCounters;
+
+#[cfg(not(rescan_stats))]
+impl RescanCounters {
+    pub(crate) fn record(&self, _reason: RescanReason) {}
+
+    pub(crate) fn record_throttled(&self, _reason: RescanReason) {}
+
+    pub(crate) fn snapshot(&self) -> RescanStats {
+        RescanStats::default()
+    }
+
+    pub(crate) fn reset(&self) {}
+}
+
+#[cfg(all(test, rescan_stats))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn counters_attribute_and_diff_per_reason() {
+        let counters = RescanCounters::default();
+        counters.record(RescanReason::Explicit);
+        let baseline = counters.snapshot();
+
+        counters.record(RescanReason::IgnoreFileChanged);
+        counters.record(RescanReason::IgnoreFileChanged);
+        counters.record(RescanReason::OverflowCapacity);
+
+        let stats = counters.snapshot();
+        assert_eq!(stats.total, 4);
+        assert_eq!(stats.watcher_triggered(), 3);
+
+        let delta = stats.since(&baseline);
+        assert_eq!(delta.total, 3);
+        assert_eq!(delta.count(RescanReason::Explicit), 0);
+        assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 2);
+        assert_eq!(
+            delta.to_string(),
+            "3 rescan(s) [ignore_file_changed=2, overflow_capacity=1]"
+        );
+
+        counters.reset();
+        assert_eq!(counters.snapshot(), RescanStats::default());
+    }
+}

crates/fff-core/src/rescan_throttle.rs

@@ -0,0 +1,124 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Instant;
+
+use crate::constants::{
+    LARGE_INDEX_FILE_COUNT, RESCAN_MIN_INTERVAL, RESCAN_MIN_INTERVAL_LARGE_INDEX,
+};
+
+const NEVER: u64 = u64::MAX;
+
+// Drops watcher rescan requests inside the cooldown after the last scan.
+// A slightly stale index is fine: the next admitted event rescans everything.
+pub(crate) struct RescanThrottle {
+    epoch: Instant,
+    last_admitted: AtomicU64,
+}
+
+impl Default for RescanThrottle {
+    fn default() -> Self {
+        Self {
+            epoch: Instant::now(),
+            last_admitted: AtomicU64::new(NEVER),
+        }
+    }
+}
+
+impl RescanThrottle {
+    /// Returns `true` if a rescan may start now and records it as the last scan
+    pub(crate) fn admit(&self, live_files: usize, has_git_repo: bool) -> bool {
+        let min_interval = if !has_git_repo && live_files >= LARGE_INDEX_FILE_COUNT {
+            RESCAN_MIN_INTERVAL_LARGE_INDEX
+        } else {
+            RESCAN_MIN_INTERVAL
+        };
+
+        let min_ms = min_interval.as_millis() as u64;
+        let now = self.elapsed_ms();
+
+        loop {
+            let last = self.last_admitted.load(Ordering::Acquire);
+            if last != NEVER && now.saturating_sub(last) < min_ms {
+                return false;
+            }
+            // CAS so two concurrent requests cannot both start a walk.
+            if self
+                .last_admitted
+                .compare_exchange(last, now, Ordering::AcqRel, Ordering::Acquire)
+                .is_ok()
+            {
+                return true;
+            }
+        }
+    }
+
+    /// Records an explicit (unthrottled) scan so watcher requests right after
+    /// it are dropped: the index is already fresh.
+    pub(crate) fn note_explicit_scan(&self) {
+        self.last_admitted
+            .store(self.elapsed_ms(), Ordering::Release);
+    }
+
+    fn elapsed_ms(&self) -> u64 {
+        self.epoch.elapsed().as_millis() as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+
+    fn throttle_at(ms_ago: u64) -> RescanThrottle {
+        let now = Instant::now();
+        RescanThrottle {
+            epoch: now
+                .checked_sub(Duration::from_millis(ms_ago))
+                .expect("monotonic clock older than the rewind"),
+            last_admitted: AtomicU64::new(0),
+        }
+    }
+
+    #[test]
+    fn first_request_is_always_admitted() {
+        let throttle = RescanThrottle::default();
+        assert!(throttle.admit(100, true));
+    }
+
+    #[test]
+    fn requests_inside_the_cooldown_are_dropped() {
+        let throttle = throttle_at(1_000);
+        assert!(!throttle.admit(100, false));
+        assert!(!throttle.admit(100, false));
+    }
+
+    #[test]
+    fn a_large_index_outside_a_git_repo_uses_the_slower_cadence() {
+        // A minute is past the normal cooldown but not the large-index one.
+        let throttle = throttle_at(60_000);
+        assert!(throttle.admit(100, false));
+
+        let throttle = throttle_at(60_000);
+        assert!(!throttle.admit(LARGE_INDEX_FILE_COUNT, false));
+    }
+
+    #[test]
+    fn a_git_repo_keeps_the_normal_cadence_at_any_size() {
+        let throttle = throttle_at(60_000);
+        assert!(throttle.admit(LARGE_INDEX_FILE_COUNT, true));
+    }
+
+    #[test]
+    fn cooldown_expiry_admits_again() {
+        let throttle = throttle_at(RESCAN_MIN_INTERVAL.as_millis() as u64 + 1);
+        assert!(throttle.admit(100, false));
+        // Admission rearms the cooldown.
+        assert!(!throttle.admit(100, false));
+    }
+
+    #[test]
+    fn explicit_scan_rearms_the_cooldown() {
+        let throttle = RescanThrottle::default();
+        throttle.note_explicit_scan();
+        assert!(!throttle.admit(100, false));
+    }
+}

crates/fff-core/src/scan.rs

@@ -0,0 +1,398 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use tracing::{error, info};
+
+use crate::FileSync;
+use crate::error::Error;
+use crate::file_picker::FFFMode;
+use crate::index::{build_bigram_index, sniff_binary_for_non_indexable};
+use crate::parallelism::BACKGROUND_THREAD_POOL;
+use crate::shared::{SharedFilePicker, SharedFrecency};
+use crate::types::ContentCacheBudget;
+use crate::watch::BackgroundWatcher;
+
+#[derive(Clone, Default)]
+pub(crate) struct ScanSignals {
+    /// Set to `true` while any scan phase is running
+    pub(crate) scanning: Arc<AtomicBool>,
+    /// Set to `true` once the filesystem watcher has been installed
+    pub(crate) watcher_ready: Arc<AtomicBool>,
+    /// Indicates that that owning picker was requested to shut down
+    pub(crate) cancelled: Arc<AtomicBool>,
+    /// Used to resolve conflicts if multiple rescans were triggered in a queue
+    pub(crate) rescan_pending: Arc<AtomicBool>,
+    /// Set by `post_scan_snapshot`, cleared by `PostScanSnapshot::drop`.
+    /// DO NOT set or clear this manually — it is managed exclusively by the
+    /// PostScanSnapshot lifecycle.
+    pub(crate) post_scan_indexing_active: Arc<AtomicBool>,
+}
+
+/// Which optional phases a scan should run.
+#[derive(Clone, Copy, Default, Debug)]
+pub(crate) struct ScanConfig {
+    pub(crate) warmup: bool,
+    pub(crate) content_indexing: bool,
+    pub(crate) watch: bool,
+    pub(crate) auto_cache_budget: bool,
+    pub(crate) install_watcher: bool,
+    pub(crate) follow_symlinks: bool,
+    pub(crate) enable_fs_root_scanning: bool,
+    pub(crate) enable_home_dir_scanning: bool,
+}
+
+/// A fully-configured scan job ready to run on a background thread.
+///
+/// Build with [`ScanJob::from_picker`] (reads all state from the
+/// current `FilePicker`) or [`ScanJob::initial`] (for the bootstrap
+/// scan, before the picker is published to `SharedPicker`).
+pub(crate) struct ScanJob {
+    shared_picker: SharedFilePicker,
+    shared_frecency: SharedFrecency,
+    base_path: PathBuf,
+    mode: FFFMode,
+    signals: ScanSignals,
+    config: ScanConfig,
+    /// Walker-maintained counter backing `get_scan_progress` on the UI
+    /// side. Reset to 0 at scan start, incremented per-file by the
+    /// walker. Shared `Arc` so the UI polls the same atomic.
+    scanned_files_counter: Arc<AtomicUsize>,
+    trace_span: tracing::Span,
+}
+
+impl ScanJob {
+    pub fn new_rescan(
+        shared_picker: &SharedFilePicker,
+        shared_frecency: &SharedFrecency,
+    ) -> Result<Option<Self>, Error> {
+        let guard = shared_picker.read()?;
+        let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?;
+
+        if picker.is_scan_active()
+            || picker
+                .signals
+                .post_scan_indexing_active
+                .load(Ordering::Acquire)
+        {
+            return Ok(None);
+        }
+
+        let mode = picker.mode();
+        let signals = picker.scan_signals();
+        let scanned_files_counter = picker.scanned_files_counter();
+        let base_path = picker.base_path().to_path_buf();
+        let trace_span = picker.trace_span();
+
+        let new_scan_config = ScanConfig {
+            warmup: picker.has_mmap_cache(),
+            content_indexing: picker.has_content_indexing(),
+            watch: picker.has_watcher(),
+            auto_cache_budget: !picker.has_explicit_cache_budget(),
+            install_watcher: false, // the watcher is independent of rescan, it is not restarting EVER
+            follow_symlinks: picker.follows_symlinks(),
+            enable_fs_root_scanning: picker.fs_root_scanning_enabled(),
+            enable_home_dir_scanning: picker.home_dir_scanning_enabled(),
+        };
+
+        drop(guard); // just a sanity check
+
+        Ok(Some(Self {
+            mode,
+            signals,
+            base_path,
+            scanned_files_counter,
+            config: new_scan_config,
+            shared_picker: shared_picker.clone(),
+            shared_frecency: shared_frecency.clone(),
+            trace_span,
+        }))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub fn new_initial(
+        shared_picker: SharedFilePicker,
+        shared_frecency: SharedFrecency,
+        base_path: PathBuf,
+        mode: FFFMode,
+        signals: ScanSignals,
+        scanned_files_counter: Arc<AtomicUsize>,
+        trace_span: tracing::Span,
+        config: ScanConfig,
+    ) -> Self {
+        Self {
+            shared_picker,
+            shared_frecency,
+            base_path,
+            mode,
+            signals,
+            scanned_files_counter,
+            config,
+            trace_span,
+        }
+    }
+
+    /// Run the job on `BACKGROUND_THREAD_POOL`. Returns immediately.
+    ///
+    /// Routed through the pool — and not a fresh `std::thread::spawn` — so the
+    /// orchestrator inherits rayon's QoS pin (USER_INITIATED). Without that
+    /// pin, an interactive nvim's USER_INTERACTIVE main thread spawns a child
+    /// at lower QoS, the walker's Zig worker pool inherits the demotion, and
+    /// the kernel drifts those workers onto E-cores. On chromium that turns a
+    /// ~800 ms walk into ~3 s.
+    pub fn spawn(self) {
+        self.signals.scanning.store(true, Ordering::Release);
+        let span = self.trace_span.clone();
+        BACKGROUND_THREAD_POOL.spawn(move || {
+            let _g = span.enter();
+            self.run();
+        });
+    }
+
+    fn run(self) {
+        let Self {
+            shared_picker,
+            shared_frecency,
+            base_path,
+            mode,
+            signals,
+            scanned_files_counter,
+            config,
+            trace_span: _,
+        } = self;
+
+        let _scanning = ScanningGuard::new(&signals);
+        scanned_files_counter.store(0, Ordering::Relaxed);
+
+        // 1. Walk the file system and collect the list of files
+        let git_workdir = FileSync::discover_git_workdir(&base_path);
+        let sync = match FileSync::walk_filesystem(
+            &base_path,
+            git_workdir.clone(),
+            &scanned_files_counter,
+            &shared_frecency,
+            mode,
+            config.follow_symlinks,
+        ) {
+            Ok(sync) => sync,
+            Err(e) => {
+                error!(?e, "scan walk failed");
+                return;
+            }
+        };
+
+        // 2. Populate the file list
+        let git_status_worker;
+        if let Ok(mut guard) = shared_picker.write()
+            && let Some(picker) = guard.as_mut()
+        {
+            if signals.cancelled.load(Ordering::Acquire) {
+                info!("scan cancelled between walk and commit, discarding");
+                return;
+            }
+
+            let live_count = sync.live_count;
+            picker.commit_new_sync(sync);
+            git_status_worker = Arc::clone(&picker.git_status_worker);
+
+            if config.auto_cache_budget && !picker.has_explicit_cache_budget() {
+                picker.set_cache_budget(ContentCacheBudget::new_for_repo(live_count));
+            }
+        } else {
+            error!("failed to install scan results into picker");
+            return;
+        }
+
+        // Spawn the git status worker once. BUG PINNNING. If the user initiated git in the folder
+        // which is a real use case we need to have a way to start the git worker background thread dynamically
+        if git_workdir.is_some() && !signals.cancelled.load(Ordering::Acquire) {
+            git_status_worker.spawn_once(shared_picker.weaken(), shared_frecency.clone());
+            git_status_worker.request_full_rescan(); // this runs anyway
+        }
+
+        // BUG pinning: take the snapshot *before* the storing the scan=true, otherwise there is a tiny
+        // race window when there scanned is set to true, but `post_scan_indexing_active` flag is `false`
+        let snapshot = if !signals.cancelled.load(Ordering::Acquire) {
+            shared_picker.read().ok().and_then(|guard| {
+                guard
+                    .as_ref()
+                    .and_then(|picker| unsafe { picker.post_scan_snapshot() })
+            })
+        } else {
+            None
+        };
+
+        signals.scanning.store(false, Ordering::Relaxed); // file are searchable
+
+        // in case we do a rescan, we have to resubscribe a watcher to the new set of directories
+        // all the already watched directories are not going to be resubscribed (this is internally deduped)
+        if !config.install_watcher && !signals.cancelled.load(Ordering::Acquire) {
+            rescubscribe_watcher_post_scan(&shared_picker);
+        }
+
+        // 3. Runs post scna in parallel with git status collection
+        if !signals.cancelled.load(Ordering::Acquire)
+            && let Some(snap) = snapshot.as_ref()
+        {
+            Self::run_post_scan(&shared_picker, &signals, &config, snap);
+        }
+
+        drop(snapshot); // SNAPSHOT SHOULD NOT BE USED AFTER THIS POINT
+
+        // 5. Install filesystem watcher (initial scan only).
+        if config.install_watcher && config.watch && !signals.cancelled.load(Ordering::Acquire) {
+            let shared_picker: &SharedFilePicker = &shared_picker;
+            let shared_frecency: &SharedFrecency = &shared_frecency;
+            let base_path: &std::path::Path = &base_path;
+
+            match BackgroundWatcher::new(
+                base_path.to_path_buf(),
+                git_workdir,
+                shared_picker.clone(),
+                shared_frecency.clone(),
+                mode,
+                config.enable_fs_root_scanning,
+                config.enable_home_dir_scanning,
+                git_status_worker,
+                tracing::Span::current(),
+            ) {
+                Ok(watcher) => {
+                    if let Ok(mut guard) = shared_picker.write()
+                        && let Some(picker) = guard.as_mut()
+                        && picker.base_path() == base_path
+                        && !signals.cancelled.load(Ordering::Acquire)
+                    {
+                        picker.background_watcher = Some(watcher);
+                        signals.watcher_ready.store(true, Ordering::Release);
+                    }
+                }
+                Err(e) => error!(?e, "failed to initialize background watcher"),
+            };
+        }
+
+        // 6. Drain any rescan that arrived while we were busy.
+        // if user initiated a new rescan we had no way to cancel current post scan, so do it again
+        if !signals.cancelled.load(Ordering::Acquire)
+            && signals.rescan_pending.swap(false, Ordering::AcqRel)
+        {
+            match Self::new_rescan(&shared_picker, &shared_frecency) {
+                Ok(Some(follow_up)) => {
+                    info!("Rescheduling deferred rescan after current scan finished");
+                    follow_up.spawn();
+                }
+                Ok(None) => {
+                    // this should be practically impossible because we do not have any
+                    // queue, but if somehow a new rescan was triggered JUST IN THIS MOMENT
+                    // just ignore it because the ongoing one is fresh enough
+                    tracing::warn!("Post scan was re-triggered, ignoring");
+                }
+                Err(e) => {
+                    error!(?e, "Failed to reschedule deferred rescan");
+                }
+            }
+        }
+    }
+
+    /// THIS IS VERY VERY IMPORTANT THAT ANYTHING INSIDE THIS FUNCTION TO NOT READ ANYTHING CLEARABLE OUTSIDE
+    /// this is a very silly off lock implementation that actually matters, and that's why it is crafted
+    /// to never read anything from the picker, it can only WRITE information using single instructions
+    ///
+    /// Things that are safe and immutable - file list, indexes of files, paths, and signals.
+    #[tracing::instrument(skip_all, fields(warmup = ?config.warmup, indexing = ?config.content_indexing))]
+    fn run_post_scan(
+        shared_picker: &SharedFilePicker,
+        signals: &ScanSignals,
+        config: &ScanConfig,
+        unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot,
+    ) {
+        let Some(arena) = unsafe_snapshot
+            .arena // we are never touching overlays so this arena is always correct
+            .as_ref()
+            .map(|s| s.as_arena_ptr())
+        else {
+            tracing::error!("Failed to run post scan: arena is invalid");
+            return;
+        };
+
+        let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
+        if signals.cancelled.load(Ordering::Acquire) {
+            return;
+        }
+
+        if config.content_indexing {
+            let indexable_count = unsafe_snapshot.indexable_count.min(files.len());
+            let (indexable_files, non_indexable_files) = files.split_at(indexable_count);
+            let index = build_bigram_index(indexable_files, &unsafe_snapshot.base_path, arena);
+
+            if let Ok(mut guard) = shared_picker.write()
+                && let Some(picker) = guard.as_mut()
+            {
+                picker.set_bigram_index(index);
+            }
+
+            // Bigram only sniffs files <= MAX_INDEXABLE_FILE_SIZE; large
+            // unknown-extension binaries slip past it and would otherwise be
+            // grep-able as text. Cheap header sniff catches those.
+            if !signals.cancelled.load(Ordering::Acquire) {
+                sniff_binary_for_non_indexable(
+                    non_indexable_files,
+                    &unsafe_snapshot.base_path,
+                    arena,
+                    &signals.cancelled,
+                );
+            }
+        } else {
+            // this potentially a long running as we are not parallelizing it but it's okay
+            sniff_binary_for_non_indexable(
+                files,
+                &unsafe_snapshot.base_path,
+                arena,
+                &signals.cancelled,
+            );
+        }
+
+        // TODO Skipped as potentially unsafe - figure this out later
+        // if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
+        //     warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
+        // }
+    }
+}
+
+// Ensures early returns clear the scanning signal.
+struct ScanningGuard<'a> {
+    signals: &'a ScanSignals,
+}
+
+impl<'a> ScanningGuard<'a> {
+    fn new(signals: &'a ScanSignals) -> Self {
+        signals.scanning.store(true, Ordering::Relaxed);
+        Self { signals }
+    }
+}
+
+impl Drop for ScanningGuard<'_> {
+    fn drop(&mut self) {
+        self.signals.scanning.store(false, Ordering::Relaxed);
+    }
+}
+
+/// If the scan encounters new directories created we have to add them to the watch list
+/// this is fine because the watcher does deduplicate the entries and doesn't add a lot of
+/// garbage notify watchers / fs events streams
+#[tracing::instrument(skip_all)]
+fn rescubscribe_watcher_post_scan(shared_picker: &SharedFilePicker) {
+    let Ok(guard) = shared_picker.read() else {
+        return;
+    };
+    let Some(picker) = guard.as_ref() else {
+        return;
+    };
+    let Some(watcher) = picker.background_watcher.as_ref() else {
+        return;
+    };
+
+    picker.for_each_dir(|dir: &std::path::Path| {
+        watcher.request_watch_dir(dir.to_path_buf());
+        std::ops::ControlFlow::Continue(())
+    });
+}

crates/fff-core/src/score.rs

@@ -1,14 +1,15 @@
 use crate::{
-    constraints::apply_constraints,
     git::is_modified_status,
+    index::constraints::apply_constraints,
     path_utils::calculate_distance_penalty,
-    simd_path::ArenaPtr,
+    simd_path::{ArenaPtr, MAX_PATH_CHUNKS},
     sort_buffer::{sort_by_key_with_buffer, sort_with_buffer},
     types::{DirItem, FileItem, Score, ScoringContext},
 };
-use fff_query_parser::FuzzyQuery;
+use fff_query_parser::{FFFQuery, FuzzyQuery};
 use neo_frizbee::Scoring;
 use rayon::prelude::*;
+use smallvec::SmallVec;
 use std::{borrow::Cow, path::MAIN_SEPARATOR};
 
 enum FileItems<'a> {
@@ -32,7 +33,7 @@ impl<'a> FileItems<'a> {
 fn resolve_file_chunks(
     file: &FileItem,
     arena: ArenaPtr,
-    buf: &mut [*const u8; 32],
+    buf: &mut [*const u8; MAX_PATH_CHUNKS],
 ) -> Option<(usize, u16)> {
     if file.is_deleted() {
         return None;
@@ -60,15 +61,15 @@ fn match_fuzzy_parts(
         return vec![];
     }
 
-    let resolve = |file: &FileItem, buf: &mut [*const u8; 32]| -> Option<(usize, u16)> {
-        resolve_file_chunks(file, arena, buf)
-    };
+    let resolve = |file: &FileItem,
+                   buf: &mut [*const u8; MAX_PATH_CHUNKS]|
+     -> Option<(usize, u16)> { resolve_file_chunks(file, arena, buf) };
 
     // because we reassemble the vec of reference we have to use a different type
     // to narrow down the [&FileItem] which would be resolved by frizbee as &&
-    let resolve_ref = |file: &&FileItem, buf: &mut [*const u8; 32]| -> Option<(usize, u16)> {
-        resolve_file_chunks(file, arena, buf)
-    };
+    let resolve_ref = |file: &&FileItem,
+                       buf: &mut [*const u8; MAX_PATH_CHUNKS]|
+     -> Option<(usize, u16)> { resolve_file_chunks(file, arena, buf) };
 
     let first_part_matches = match working_files {
         FileItems::All(files) => neo_frizbee::match_list_parallel_resolved(
@@ -168,13 +169,141 @@ pub(crate) fn fuzzy_match_and_score_files<'a>(
     sort_and_paginate(results, context)
 }
 
+pub(crate) fn fuzzy_match_byte_offsets_for_page<'q>(
+    query: &'q FFFQuery<'q>,
+    items: &[&FileItem],
+    max_typos: u16,
+    base_arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> Vec<SmallVec<[(u32, u32); 4]>> {
+    let parts: Vec<&str> = match &query.fuzzy_query {
+        FuzzyQuery::Text(text) if text.len() >= 2 => vec![*text],
+        FuzzyQuery::Parts(parts) => parts.iter().copied().filter(|p| p.len() >= 2).collect(),
+        _ => Vec::new(),
+    };
+
+    let mut ranges_by_item = vec![SmallVec::new(); items.len()];
+    if parts.is_empty() || items.is_empty() {
+        return ranges_by_item;
+    }
+
+    let paths: Vec<String> = items
+        .iter()
+        .map(|item| {
+            let arena = if item.is_overflow() {
+                overflow_arena
+            } else {
+                base_arena
+            };
+            let mut path = String::with_capacity(item.relative_path_len());
+            item.write_relative_path_from_arena(arena, &mut path);
+            path
+        })
+        .collect();
+
+    let has_uppercase = parts
+        .iter()
+        .any(|part| part.chars().any(|ch| ch.is_uppercase()));
+    let config = neo_frizbee::Config {
+        max_typos: Some(max_typos),
+        sort: false,
+        scoring: Scoring {
+            capitalization_bonus: if has_uppercase { 8 } else { 0 },
+            matching_case_bonus: if has_uppercase { 4 } else { 0 },
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+
+    for (idx, part) in parts.iter().copied().enumerate() {
+        let mut part_config = config;
+        if idx > 0 {
+            part_config.max_typos = config.max_typos.map(|t| t.min(part.len() as u16));
+        }
+
+        let mut matcher = neo_frizbee::Matcher::new(part, &part_config);
+        for mut matched in matcher.match_list_indices(&paths) {
+            let item_idx = matched.index as usize;
+            let Some(path) = paths.get(item_idx) else {
+                continue;
+            };
+
+            matched.indices.sort_unstable();
+            ranges_by_item[item_idx].extend(char_indices_to_byte_offsets(path, &matched.indices));
+        }
+    }
+
+    for ranges in &mut ranges_by_item {
+        *ranges = merge_byte_offsets(std::mem::take(ranges));
+    }
+
+    ranges_by_item
+}
+
+fn char_indices_to_byte_offsets(line: &str, char_indices: &[usize]) -> SmallVec<[(u32, u32); 4]> {
+    let char_byte_ranges: Vec<(usize, usize)> = line
+        .char_indices()
+        .map(|(byte_pos, ch)| (byte_pos, byte_pos + ch.len_utf8()))
+        .collect();
+    let mut result: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(char_indices.len());
+
+    for &char_idx in char_indices {
+        let Some(&(start, end)) = char_byte_ranges.get(char_idx) else {
+            continue;
+        };
+
+        if let Some(last) = result.last_mut()
+            && last.1 == start as u32
+        {
+            last.1 = end as u32;
+            continue;
+        }
+
+        result.push((start as u32, end as u32));
+    }
+
+    result
+}
+
+fn merge_byte_offsets(mut ranges: SmallVec<[(u32, u32); 4]>) -> SmallVec<[(u32, u32); 4]> {
+    if ranges.len() <= 1 {
+        return ranges;
+    }
+
+    ranges.sort_unstable_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
+    let mut merged: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(ranges.len());
+
+    for (start, end) in ranges {
+        if end <= start {
+            continue;
+        }
+
+        if let Some(last) = merged.last_mut()
+            && start <= last.1
+        {
+            last.1 = last.1.max(end);
+            continue;
+        }
+
+        merged.push((start, end));
+    }
+
+    merged
+}
+
 /// Resolve a DirItem's chunked path into frizbee's pointer buffer.
 #[inline]
 fn resolve_dir_chunks(
     dir: &DirItem,
     arena: ArenaPtr,
-    buf: &mut [*const u8; 32],
+    overflow_arena: ArenaPtr,
+    buf: &mut [*const u8; MAX_PATH_CHUNKS],
 ) -> Option<(usize, u16)> {
+    let arena = if dir.is_overflow() {
+        overflow_arena
+    } else {
+        arena
+    };
     let ptrs = dir.path.resolve_ptrs(arena, buf);
     Some((ptrs.len(), dir.path.byte_len))
 }
@@ -187,6 +316,7 @@ fn match_fuzzy_parts_dirs(
     options: &neo_frizbee::Config,
     max_threads: usize,
     arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
 ) -> Vec<neo_frizbee::Match> {
     let valid_parts: Vec<&str> = fuzzy_parts
         .iter()
@@ -199,8 +329,8 @@ fn match_fuzzy_parts_dirs(
     }
 
     let resolve_chunks_for_frizbee =
-        |dir: &&DirItem, buf: &mut [*const u8; 32]| -> Option<(usize, u16)> {
-            resolve_dir_chunks(dir, arena, buf)
+        |dir: &&DirItem, buf: &mut [*const u8; MAX_PATH_CHUNKS]| -> Option<(usize, u16)> {
+            resolve_dir_chunks(dir, arena, overflow_arena, buf)
         };
 
     let first_part_matches = neo_frizbee::match_list_parallel_resolved(
@@ -267,19 +397,23 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
     dirs: &'a [DirItem],
     context: &ScoringContext,
     arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
 ) -> (Vec<&'a DirItem>, Vec<Score>, usize) {
     if dirs.is_empty() {
         return (vec![], vec![], 0);
     }
 
     let parsed_query = context.query;
+    // Ghost dirs (all files tombstoned) never surface in search results.
     let working_dirs: Vec<&DirItem> = if parsed_query.constraints.is_empty() {
-        dirs.iter().collect()
+        dirs.iter().filter(|d| !d.is_deleted()).collect()
     } else {
-        match apply_constraints(dirs, &parsed_query.constraints, arena) {
-            Some(filtered) if !filtered.is_empty() => filtered,
+        match apply_constraints(dirs, &parsed_query.constraints, arena, overflow_arena) {
+            Some(filtered) if !filtered.is_empty() => {
+                filtered.into_iter().filter(|d| !d.is_deleted()).collect()
+            }
             Some(_) => return (vec![], vec![], 0),
-            None => dirs.iter().collect(),
+            None => dirs.iter().filter(|d| !d.is_deleted()).collect(),
         }
     };
 
@@ -313,6 +447,7 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
             matching_case_bonus: if has_uppercase { 4 } else { 0 },
             ..Default::default()
         },
+        ..Default::default()
     };
 
     let path_matches = match_fuzzy_parts_dirs(
@@ -321,6 +456,7 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
         &options,
         context.max_threads,
         arena,
+        overflow_arena,
     );
 
     let main_needle = valid_parts[0].as_bytes();
@@ -333,12 +469,17 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
         .into_iter()
         .map(|path_match| {
             let dir = working_dirs[path_match.index as usize];
+            let dir_arena = if dir.is_overflow() {
+                overflow_arena
+            } else {
+                arena
+            };
             let base_score = path_match.score as i32;
             let frecency_boost = base_score.saturating_mul(dir.max_access_frecency()) / 100;
 
             // Distance penalty from current file's directory.
             let distance_penalty = if context.current_file.is_some() {
-                dir.path.write_to_string(arena, &mut dir_buf);
+                dir.path.write_to_string(dir_arena, &mut dir_buf);
                 calculate_distance_penalty(context.current_file, &dir_buf)
             } else {
                 0
@@ -349,7 +490,7 @@ pub(crate) fn fuzzy_match_and_score_dirs<'a>(
             let match_start_approx = path_match.end_col.saturating_sub(main_needle_len - 1);
             let is_dirname_match = match_start_approx >= last_seg_offset;
 
-            dir.write_dir_name(arena, &mut dirname_buf);
+            dir.write_dir_name(dir_arena, &mut dirname_buf);
             let dirname_len = dirname_buf.len();
             let is_exact_dirname = is_dirname_match
                 && main_needle_len as usize == dirname_len
@@ -474,7 +615,7 @@ fn match_and_score_in_arena<'a>(
     let working_files: FileItems<'a> = if parsed.constraints.is_empty() {
         FileItems::All(files)
     } else {
-        match apply_constraints(files, &parsed.constraints, arena) {
+        match apply_constraints(files, &parsed.constraints, arena, arena) {
             Some(filtered) if !filtered.is_empty() => FileItems::Filtered(filtered),
             Some(_) => {
                 return vec![];
@@ -495,7 +636,11 @@ fn match_and_score_in_arena<'a>(
     let has_uppercase = fuzzy_parts
         .iter()
         .any(|p| p.chars().any(|c| c.is_uppercase()));
-    let query_contains_path_separator = fuzzy_parts.iter().any(|p| p.contains(MAIN_SEPARATOR));
+    // Users type `/` regardless of platform. Checking the OS separator alone
+    // would miss forward-slash queries on Windows.
+    let query_contains_path_separator = fuzzy_parts
+        .iter()
+        .any(|p| p.contains('/') || p.contains(MAIN_SEPARATOR));
 
     let options = neo_frizbee::Config {
         max_typos: Some(context.max_typos),
@@ -505,6 +650,7 @@ fn match_and_score_in_arena<'a>(
             matching_case_bonus: if has_uppercase { 4 } else { 0 },
             ..Default::default()
         },
+        ..Default::default()
     };
 
     let path_matches = match_fuzzy_parts(
@@ -807,13 +953,17 @@ fn score_filtered_by_frecency<'a>(
     match files {
         FileItems::All(s) => s
             .par_iter()
-            .filter(|f| !f.is_deleted())
-            .map(&score_file)
+            .filter_map(|f| {
+                let live = !f.is_deleted();
+                live.then_some(score_file(f))
+            })
             .collect(),
         FileItems::Filtered(v) => v
             .iter()
-            .filter(|f| !f.is_deleted())
-            .map(|&file| score_file(file))
+            .filter_map(|f| {
+                let live = !f.is_deleted();
+                live.then_some(score_file(f))
+            })
             .collect(),
     }
 }
@@ -1307,6 +1457,31 @@ mod filename_bonus_tests {
         );
     }
 
+    /// Regression: PR #652 / field panic in pi-fff v0.9.6.
+    /// A path >512 bytes (but within PATH_MAX) overflows the fixed
+    /// `[*const u8; 32]` chunk-pointer buffer during scoring and panics with
+    /// "index out of bounds: the len is 32 but the index is 32".
+    #[test]
+    fn test_path_longer_than_512_bytes_does_not_panic_and_matches() {
+        let mut long_path = String::new();
+        while long_path.len() < 600 {
+            long_path.push_str("deeply_nested_directory_segment/");
+        }
+        long_path.push_str("needle_file.rs");
+        assert!(long_path.len() > 512 && long_path.len() < crate::simd_path::PATH_BUF_SIZE);
+
+        let (files, arena) = make_files(&[long_path.as_str(), "src/other.rs"]);
+
+        // Panics here on unfixed code: frizbee resolves chunk ptrs per file.
+        let results = search(&files, "needle", arena);
+
+        assert!(
+            results.iter().any(|(p, _)| p == &long_path),
+            "filename at the tail of a >512-byte path must still match, got: {:?}",
+            results.iter().map(|(p, _)| p).collect::<Vec<_>>()
+        );
+    }
+
     #[test]
     fn test_single_path_matching() {
         let path = "core_workflow_service/kafka_event_consumer/src/ai_part_extraction_request/ai_part_extraction_request_handler.rs";
@@ -1462,3 +1637,108 @@ mod typo_resistance_tests {
         );
     }
 }
+
+#[cfg(test)]
+mod constraint_only_query_tests {
+    use super::*;
+    use crate::types::PaginationArgs;
+    use fff_query_parser::QueryParser;
+
+    #[test]
+    fn constraint_only_git_status_query_returns_filtered_files() {
+        let paths = ["modified.rs", "clean.rs"];
+        let path_strings: Vec<String> = paths.iter().map(|p| p.to_string()).collect();
+        let items: Vec<FileItem> = paths
+            .iter()
+            .map(|p| {
+                let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16;
+                FileItem::new_raw(fname, 0, 0, None, false)
+            })
+            .collect();
+        let (store, strings) =
+            crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items);
+        let arena = store.as_arena_ptr();
+        let mut files: Vec<FileItem> = items;
+        for (i, file) in files.iter_mut().enumerate() {
+            file.set_path(strings[i].clone());
+        }
+        files[0].git_status = Some(git2::Status::WT_MODIFIED);
+        files[1].git_status = Some(git2::Status::CURRENT);
+        std::mem::forget(store);
+
+        let parser = QueryParser::default();
+        let parsed = parser.parse("git:modified");
+
+        let ctx = ScoringContext {
+            query: &parsed,
+            max_threads: 1,
+            max_typos: 2,
+            current_file: None,
+            last_same_query_match: None,
+            project_path: None,
+            combo_boost_score_multiplier: 100,
+            min_combo_count: 3,
+            pagination: PaginationArgs {
+                offset: 0,
+                limit: 50,
+            },
+        };
+
+        let (items, _scores, total_matched) =
+            fuzzy_match_and_score_files(&files, &ctx, files.len(), arena, arena);
+
+        assert_eq!(
+            total_matched, 1,
+            "git:modified constraint-only query should match exactly 1 modified file"
+        );
+        assert_eq!(items.len(), 1);
+        assert_eq!(items[0].relative_path(arena), "modified.rs");
+    }
+
+    #[test]
+    fn constraint_only_status_modified_alias_returns_filtered_files() {
+        let paths = ["a.rs", "b.rs"];
+        let path_strings: Vec<String> = paths.iter().map(|p| p.to_string()).collect();
+        let items: Vec<FileItem> = paths
+            .iter()
+            .map(|p| {
+                let fname = p.rfind(std::path::is_separator).map(|i| i + 1).unwrap_or(0) as u16;
+                FileItem::new_raw(fname, 0, 0, None, false)
+            })
+            .collect();
+        let (store, strings) =
+            crate::simd_path::build_chunked_path_store_from_strings(&path_strings, &items);
+        let arena = store.as_arena_ptr();
+        let mut files: Vec<FileItem> = items;
+        for (i, file) in files.iter_mut().enumerate() {
+            file.set_path(strings[i].clone());
+        }
+        files[0].git_status = Some(git2::Status::WT_MODIFIED);
+        files[1].git_status = Some(git2::Status::CURRENT);
+        std::mem::forget(store);
+
+        let parser = QueryParser::default();
+        let parsed = parser.parse("status:modified");
+
+        let ctx = ScoringContext {
+            query: &parsed,
+            max_threads: 1,
+            max_typos: 2,
+            current_file: None,
+            last_same_query_match: None,
+            project_path: None,
+            combo_boost_score_multiplier: 100,
+            min_combo_count: 3,
+            pagination: PaginationArgs {
+                offset: 0,
+                limit: 50,
+            },
+        };
+
+        let (items, _scores, total_matched) =
+            fuzzy_match_and_score_files(&files, &ctx, files.len(), arena, arena);
+
+        assert_eq!(total_matched, 1);
+        assert_eq!(items[0].relative_path(arena), "a.rs");
+    }
+}

crates/fff-core/src/shared.rs

@@ -1,113 +1,374 @@
-use std::path::PathBuf;
-use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
-use std::time::Duration;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak};
+use std::time::{Duration, Instant};
 
+use crate::dbs::lmdb::{LmdbStore, spawn_lmdb_gc};
 use crate::error::Error;
 use crate::file_picker::FilePicker;
 use crate::frecency::FrecencyTracker;
 use crate::git::GitStatusCache;
 use crate::query_tracker::QueryTracker;
+use crate::rescan_stats::{RescanCounters, RescanReason, RescanStats};
+use crate::rescan_throttle::RescanThrottle;
+use crate::scan::ScanJob;
+use crate::watch::{WatchEvent, WatchId, WatchOptions, WatchRegistry};
+use git2::Repository;
+
+/// Poll `.git/index.lock` until it disappears (git write completed), giving up
+/// after [`GIT_LOCK_MAX_WAIT`]. Used by [`SharedPicker::refresh_git_status`]
+/// to avoid reading a half-updated index when the watcher fires mid-`git add`.
+///
+/// The wait is bounded and cheap: the lock file is typically cleared within
+/// a few milliseconds of the git command exiting.
+fn wait_for_git_index_lock_release(git_root: &Path) {
+    const GIT_LOCK_POLL: Duration = Duration::from_millis(10);
+    const GIT_LOCK_MAX_WAIT: Duration = Duration::from_millis(500);
+
+    let lock = git_root.join(".git").join("index.lock");
+    // Fast path: no lock present.
+    if !lock.exists() {
+        return;
+    }
+    let deadline = Instant::now() + GIT_LOCK_MAX_WAIT;
+    while lock.exists() && Instant::now() < deadline {
+        std::thread::sleep(GIT_LOCK_POLL);
+    }
+    if lock.exists() {
+        tracing::warn!(
+            "Proceeding with git status refresh despite lingering \
+             .git/index.lock at {} — will retry once it clears",
+            lock.display()
+        );
+    }
+}
+
+/// Poll `done` every 10ms until it returns `true`, or until `timeout` elapses.
+/// Returns `true` if the condition was met, `false` on timeout.
+fn poll_until(timeout: Duration, mut done: impl FnMut() -> bool) -> bool {
+    let start = Instant::now();
+    while !done() {
+        if start.elapsed() >= timeout {
+            return false;
+        }
+        std::thread::sleep(Duration::from_millis(10));
+    }
+    true
+}
 
 /// Thread-safe shared handle to the [`FilePicker`] instance.
+/// This accumulates only asynchronous non-blocking operations against the
+/// file picker: creating, triggering various rescans and so on.
 ///
-/// Uses `parking_lot::RwLock` which is reader-fair — new readers are not
-/// blocked when a writer is waiting, preventing search query stalls during
-/// background bigram builds or watcher writes.
+/// For blocking access use internal picker via `.read()` or `.write()`
 ///
-/// `Clone` gives a new handle to the same picker (Arc clone).
-/// `Default` creates an empty handle suitable for `Lazy::new(SharedPicker::default)`.
+/// ```ignore
+/// let shared_picker = SharedFilePicker::default();
+///
+/// if let Some(picker) = shared_picker.read()?.as_ref() {
+///     let files = picker.fuzzy_search(&query, options);
+///     println!("Found {} files", files.len());
+/// } else {
+///     println!("Picker not initialized");
+/// }
+/// ```
 #[derive(Clone, Default)]
-pub struct SharedPicker(pub(crate) Arc<parking_lot::RwLock<Option<FilePicker>>>);
+pub struct SharedFilePicker(pub(crate) Arc<SharedPickerInner>);
 
-impl std::fmt::Debug for SharedPicker {
+pub struct SharedPickerInner {
+    picker: parking_lot::RwLock<Option<FilePicker>>,
+    /// Watch subscriptions live outside the picker lock so delivery and
+    /// (un)subscribing never contend with searches.
+    watchers: Arc<WatchRegistry>,
+    rescans: RescanCounters,
+    rescan_throttle: RescanThrottle,
+}
+
+impl Default for SharedPickerInner {
+    fn default() -> Self {
+        Self {
+            picker: parking_lot::RwLock::new(None),
+            watchers: Arc::new(WatchRegistry::default()),
+            rescans: RescanCounters::default(),
+            rescan_throttle: RescanThrottle::default(),
+        }
+    }
+}
+
+/// Non-owning handle to a [`SharedPicker`].
+#[derive(Clone)]
+pub(crate) struct WeakFilePicker(Weak<SharedPickerInner>);
+
+impl WeakFilePicker {
+    /// Try to promote the weak handle back to a strong [`SharedPicker`].
+    ///
+    /// Returns `None` once every strong `SharedPicker` clone has been
+    /// dropped. Callers should treat that as "the picker is being
+    /// torn down" and exit their current iteration cleanly.
+    pub(crate) fn upgrade(&self) -> Option<SharedFilePicker> {
+        self.0.upgrade().map(SharedFilePicker)
+    }
+}
+
+impl std::fmt::Debug for SharedFilePicker {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_tuple("SharedPicker").field(&"..").finish()
     }
 }
 
-impl SharedPicker {
+impl SharedFilePicker {
     pub fn read(&self) -> Result<parking_lot::RwLockReadGuard<'_, Option<FilePicker>>, Error> {
-        Ok(self.0.read())
+        Ok(self.0.picker.read())
     }
 
     pub fn write(&self) -> Result<parking_lot::RwLockWriteGuard<'_, Option<FilePicker>>, Error> {
-        Ok(self.0.write())
+        Ok(self.0.picker.write())
+    }
+
+    /// Signal the background scan to cancel. Non-blocking: post-scan
+    /// threads check this flag and bail out at their next cancellation point.
+    pub fn cancel(&self) {
+        if let Ok(guard) = self.read()
+            && let Some(picker) = guard.as_ref()
+        {
+            picker.cancel();
+        }
+    }
+
+    /// Produce a non-owning handle to the same inner picker.
+    /// Use it if you don't need to block internal threads from dropping while owning this ref
+    pub(crate) fn weaken(&self) -> WeakFilePicker {
+        WeakFilePicker(Arc::downgrade(&self.0))
     }
 
     /// Return `true` if this is an instance of the picker that requires a complicated post-scan
     /// indexing/cache warmup job. The indexing is not crazy but it takes time.
     pub fn need_complex_rebuild(&self) -> bool {
-        let guard = self.0.read();
+        let guard = self.0.picker.read();
         guard
             .as_ref()
-            .is_some_and(|p| p.need_enable_mmap_cache() || p.need_enable_content_indexing())
+            .is_some_and(|p| p.has_mmap_cache() || p.has_content_indexing())
     }
 
     /// Block until the background filesystem scan finishes.
     /// Returns `true` if scan completed, `false` on timeout.
     pub fn wait_for_scan(&self, timeout: Duration) -> bool {
         let signal = {
-            let guard = self.0.read();
+            let guard = self.0.picker.read();
             match &*guard {
-                Some(picker) => picker.scan_signal(),
+                Some(picker) => Arc::clone(&picker.signals.scanning),
                 None => return true,
             }
         };
 
-        let start = std::time::Instant::now();
-        while signal.load(std::sync::atomic::Ordering::Acquire) {
-            if start.elapsed() >= timeout {
-                return false;
-            }
-            std::thread::sleep(Duration::from_millis(10));
-        }
-        true
+        poll_until(timeout, || {
+            !signal.load(std::sync::atomic::Ordering::Acquire)
+        })
     }
 
     /// Block until the background file watcher is ready.
     /// Returns `true` if watcher ready, `false` on timeout.
     pub fn wait_for_watcher(&self, timeout: Duration) -> bool {
-        let signal = {
-            let guard = self.0.read();
+        let watch_ready_signal = {
+            let guard = self.0.picker.read();
             match &*guard {
-                Some(picker) => picker.watcher_signal(),
+                Some(picker) => Arc::clone(&picker.signals.watcher_ready),
                 None => return true,
             }
         };
 
-        let start = std::time::Instant::now();
-        while !signal.load(std::sync::atomic::Ordering::Acquire) {
-            if start.elapsed() >= timeout {
-                return false;
-            }
-            std::thread::sleep(Duration::from_millis(10));
-        }
-        true
+        poll_until(timeout, || {
+            watch_ready_signal.load(std::sync::atomic::Ordering::Acquire)
+        })
     }
 
-    /// Refresh git statuses for all indexed files.
+    /// Blocks until both the filesystem walk and post-scan indexing are done.
+    /// Returns true once scanning=false AND post_scan_indexing_active=false.
+    pub fn wait_for_indexing_complete(&self, timeout: Duration) -> bool {
+        let (scanning, post_scan_active) = {
+            let guard = self.0.picker.read();
+            match &*guard {
+                Some(picker) => (
+                    Arc::clone(&picker.signals.scanning),
+                    Arc::clone(&picker.signals.post_scan_indexing_active),
+                ),
+                None => return true,
+            }
+        };
+
+        poll_until(timeout, || {
+            !scanning.load(std::sync::atomic::Ordering::Acquire)
+                && !post_scan_active.load(std::sync::atomic::Ordering::Acquire)
+        })
+    }
+
+    /// Trigger a full filesystem rescan without blocking the caller.
+    /// Performs a safe async rescan. Guarantees only single active rescan per picker.
+    /// If many rescans requested the last one guaranteed to be finished.
+    pub fn trigger_full_rescan_async(&self, shared_frecency: &SharedFrecency) -> Result<(), Error> {
+        self.trigger_full_rescan_with_reason(shared_frecency, RescanReason::Explicit)
+            .map(|_| ())
+    }
+
+    /// Returns admitted and throttled rescan requests by reason.
+    /// Counters start at picker creation or the last reset.
+    pub fn rescan_stats(&self) -> RescanStats {
+        self.0.rescans.snapshot()
+    }
+
+    pub fn reset_rescan_stats(&self) {
+        self.0.rescans.reset();
+    }
+
+    /// Returns `Ok(true)` when a rescan was started (or queued behind an
+    /// active scan) and `Ok(false)` when the request was throttled — the
+    /// caller must then fall back to incremental event processing.
+    pub(crate) fn trigger_full_rescan_with_reason(
+        &self,
+        shared_frecency: &SharedFrecency,
+        reason: RescanReason,
+    ) -> Result<bool, Error> {
+        // for giant folders we have no other choice other than throttling rescans
+        // if user is running application in millions of files with a ton of rescan events
+        // we drop / throttle some of requests to avoid constant burst of IO
+        if reason == RescanReason::Explicit {
+            self.0.rescan_throttle.note_explicit_scan();
+        } else if !self.check_rescan_throttle(reason) {
+            return Ok(false);
+        }
+
+        self.0.rescans.record(reason);
+
+        match ScanJob::new_rescan(self, shared_frecency)? {
+            Some(job) => {
+                job.spawn();
+            }
+            None => {
+                // we can not abort the ongoing sync, but if the events
+                if let Ok(guard) = self.read()
+                    && let Some(picker) = guard.as_ref()
+                {
+                    picker
+                        .scan_signals()
+                        .rescan_pending
+                        .store(true, std::sync::atomic::Ordering::Release);
+                    tracing::info!(
+                        "Full rescan requested while another scan is active — \
+                         deferred via rescan_pending flag"
+                    );
+                }
+            }
+        }
+        Ok(true)
+    }
+
+    fn check_rescan_throttle(&self, reason: RescanReason) -> bool {
+        let (live_files, has_git) = self
+            .read()
+            .ok()
+            .and_then(|guard| {
+                guard
+                    .as_ref()
+                    .map(|picker| (picker.live_file_count(), picker.has_git_repo()))
+            })
+            .unwrap_or((0, false));
+
+        if self.0.rescan_throttle.admit(live_files, has_git) {
+            return true;
+        }
+
+        self.0.rescans.record_throttled(reason);
+        tracing::debug!(%reason, live_files, "Rescan throttled, skipping");
+        false
+    }
+
+    /// Subscribe to filesystem changes matching `pattern`.
+    ///
+    /// Patterns may be base-relative globs (./ works), exact paths inside the indexed
+    /// tree, or existing directories. An empty pattern watches the whole tree.
+    ///
+    /// Events are debounced and submitted in batches per 100-ms window at most 128 events.
+    /// Gitignored and other ignored files are never triggering watcher.
+    pub fn watch(
+        &self,
+        pattern: &str,
+        options: WatchOptions,
+        callback: impl Fn(WatchId, &[WatchEvent]) + Send + Sync + 'static,
+    ) -> Result<WatchId, Error> {
+        let (base_path, has_watcher, watcher_ready) = {
+            let guard = self.read()?;
+            let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?;
+
+            (
+                picker.base_path().to_path_buf(),
+                picker.has_watcher(),
+                picker.is_watcher_ready(),
+            )
+        };
+
+        if !has_watcher {
+            return Err(Error::WatcherDisabled);
+        }
+        if !watcher_ready {
+            return Err(Error::WatcherNotReady);
+        }
+
+        self.0
+            .watchers
+            .subscribe(&base_path, pattern, options, Box::new(callback))
+    }
+
+    /// Remove a watch subscription. Returns `true` if the id was active.
+    pub fn unwatch(&self, id: WatchId) -> bool {
+        self.0.watchers.unsubscribe(id)
+    }
+
+    /// Return whether a watch subscription is active.
+    pub fn is_watch_active(&self, id: WatchId) -> bool {
+        self.0.watchers.contains(id)
+    }
+
+    /// Remove every subscription without waiting for an executing callback.
+    pub fn shutdown_watches(&self) {
+        self.0.watchers.shutdown();
+    }
+
+    /// Remove every subscription and wait for an executing callback.
+    /// When called by that callback, it does not wait on itself.
+    pub fn shutdown_watches_and_wait(&self) {
+        self.0.watchers.shutdown_and_wait();
+    }
+
+    pub(crate) fn rebase_watches(&self, base_path: &Path) {
+        self.0.watchers.rebase(base_path);
+    }
+
+    pub(crate) fn watch_registry(&self) -> &Arc<WatchRegistry> {
+        &self.0.watchers
+    }
+
+    /// Refresh git statuses for all indexed files
+    #[tracing::instrument(level = "info", skip_all)]
     pub fn refresh_git_status(&self, shared_frecency: &SharedFrecency) -> Result<usize, Error> {
-        use git2::StatusOptions;
         use tracing::debug;
 
         let git_status = {
-            let guard = self.read()?;
-            let Some(ref picker) = *guard else {
-                return Err(Error::FilePickerMissing);
+            let git_root = {
+                let guard = self.read()?;
+                let Some(ref picker) = *guard else {
+                    return Err(Error::FilePickerMissing);
+                };
+                picker.git_root().map(|p| p.to_path_buf())
             };
 
-            debug!(
-                "Refreshing git statuses for picker: {:?}",
-                picker.git_root()
-            );
+            debug!(?git_root, "Refreshing git status for picker");
+
+            if let Some(ref root) = git_root {
+                wait_for_git_index_lock_release(root);
+            }
 
             GitStatusCache::read_git_status(
-                picker.git_root(),
-                StatusOptions::new()
-                    .include_untracked(true)
-                    .recurse_untracked_dirs(true)
-                    .include_unmodified(true)
-                    .exclude_submodules(true),
+                git_root.as_deref(),
+                &mut crate::git::default_status_options(),
             )
         };
 
@@ -124,16 +385,62 @@ impl SharedPicker {
 
         Ok(statuses_count)
     }
+
+    /// Recompute and apply git status for a specific set of paths.
+    pub fn update_git_status_for_paths(
+        &self,
+        paths: &[PathBuf],
+        shared_frecency: &SharedFrecency,
+    ) -> Result<(), Error> {
+        if paths.is_empty() {
+            return Ok(());
+        }
+
+        let git_root = {
+            let guard = self.read()?;
+            let Some(ref picker) = *guard else {
+                return Err(Error::FilePickerMissing);
+            };
+            picker.git_root().map(|p| p.to_path_buf())
+        };
+        let Some(git_root) = git_root else {
+            return Ok(());
+        };
+
+        wait_for_git_index_lock_release(&git_root);
+
+        let repo = Repository::open(&git_root)?;
+        let status = GitStatusCache::git_status_for_paths(&repo, paths)?;
+
+        let mut guard = self.write()?;
+        let picker = guard.as_mut().ok_or(Error::FilePickerMissing)?;
+        picker.update_git_statuses(status, shared_frecency)
+    }
 }
 
-/// Thread-safe shared handle to the [`FrecencyTracker`] instance.
-#[derive(Clone)]
-pub struct SharedFrecency {
-    inner: Arc<RwLock<Option<FrecencyTracker>>>,
+/// Thread-safe shared handle to an LMDB-backed store. A disabled (`noop`)
+/// instance silently ignores writes. See the [`SharedFrecency`] and
+/// [`SharedQueryTracker`] aliases.
+///
+/// `LmdbStore` is intentionally crate-private, so the store type is sealed:
+/// only `FrecencyTracker` / `QueryTracker` can ever instantiate this.
+#[allow(private_bounds)]
+pub struct SharedDb<T: LmdbStore> {
+    inner: Arc<RwLock<Option<T>>>,
     enabled: bool,
 }
 
-impl Default for SharedFrecency {
+// Hand-written to avoid a spurious `T: Clone` bound — `Arc` is always `Clone`.
+impl<T: LmdbStore> Clone for SharedDb<T> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            enabled: self.enabled,
+        }
+    }
+}
+
+impl<T: LmdbStore> Default for SharedDb<T> {
     fn default() -> Self {
         Self {
             inner: Arc::new(RwLock::new(None)),
@@ -142,13 +449,14 @@ impl Default for SharedFrecency {
     }
 }
 
-impl std::fmt::Debug for SharedFrecency {
+impl<T: LmdbStore> std::fmt::Debug for SharedDb<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("SharedFrecency").field(&"..").finish()
+        f.debug_tuple("SharedDb").field(&T::LABEL).finish()
     }
 }
 
-impl SharedFrecency {
+#[allow(private_bounds)]
+impl<T: LmdbStore> SharedDb<T> {
     /// Creates a disabled instance that silently ignores all writes.
     pub fn noop() -> Self {
         Self {
@@ -157,50 +465,57 @@ impl SharedFrecency {
         }
     }
 
-    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<FrecencyTracker>>, Error> {
+    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<T>>, Error> {
         self.inner.read().map_err(|_| Error::AcquireFrecencyLock)
     }
 
-    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<FrecencyTracker>>, Error> {
+    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<T>>, Error> {
         self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
     }
 
-    /// Initialize the frecency tracker. No-op if this is a disabled instance.
-    pub fn init(&self, tracker: FrecencyTracker) -> Result<(), Error> {
+    /// Initialize the store + spawn GC in the background. No-op when disabled.
+    pub fn init(&self, tracker: T) -> Result<(), Error> {
         if !self.enabled {
             return Ok(());
         }
-        let mut guard = self.write()?;
-        *guard = Some(tracker);
-        Ok(())
-    }
 
-    /// Spawn a background GC thread for this frecency tracker.
-    pub fn spawn_gc(
-        &self,
-        db_path: String,
-        use_unsafe_no_lock: bool,
-    ) -> crate::Result<std::thread::JoinHandle<()>> {
-        FrecencyTracker::spawn_gc(self.clone(), db_path, use_unsafe_no_lock)
+        {
+            let mut guard = self.write()?;
+            *guard = Some(tracker);
+        }
+
+        // GC holds a read guard on this lock, so destroy / re-init wait won't race
+        spawn_lmdb_gc(self.inner.clone());
+        Ok(())
     }
 
     /// Drop the in-memory tracker and delete the on-disk database directory.
     ///
-    /// Acquires the write lock, ensuring all readers (including any active mmap
-    /// access) are finished before the LMDB environment is closed and the files
-    /// are removed.
-    ///
-    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no
-    /// tracker was initialized.
+    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no tracker was initialized.
     pub fn destroy(&self) -> Result<Option<PathBuf>, Error> {
         let mut guard = self.write()?;
         let Some(tracker) = guard.take() else {
             return Ok(None);
         };
-        let db_path = tracker.db_path().to_path_buf();
+
+        let closing_event = match tracker.shared_env().destroy() {
+            Ok(closing) => closing,
+            Err(e) => {
+                *guard = Some(tracker);
+                return Err(e);
+            }
+        };
+
+        let db_path = tracker.env().path().to_path_buf();
         // Drop closes the LMDB env and unmaps the files
         drop(tracker);
         drop(guard);
+
+        // Deleting before mdb_env_close finishes would race the unmap.
+        if let Some(event) = closing_event {
+            event.wait_timeout(Duration::from_secs(5));
+        }
+
         std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir {
             path: db_path.clone(),
             source,
@@ -209,75 +524,8 @@ impl SharedFrecency {
     }
 }
 
+/// Thread-safe shared handle to the [`FrecencyTracker`] instance.
+pub type SharedFrecency = SharedDb<FrecencyTracker>;
+
 /// Thread-safe shared handle to the [`QueryTracker`] instance.
-#[derive(Clone)]
-pub struct SharedQueryTracker {
-    inner: Arc<RwLock<Option<QueryTracker>>>,
-    enabled: bool,
-}
-
-impl Default for SharedQueryTracker {
-    fn default() -> Self {
-        Self {
-            inner: Arc::new(RwLock::new(None)),
-            enabled: true,
-        }
-    }
-}
-
-impl std::fmt::Debug for SharedQueryTracker {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_tuple("SharedQueryTracker").field(&"..").finish()
-    }
-}
-
-impl SharedQueryTracker {
-    /// Creates a disabled instance that silently ignores all writes.
-    pub fn noop() -> Self {
-        Self {
-            inner: Arc::new(RwLock::new(None)),
-            enabled: false,
-        }
-    }
-
-    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<QueryTracker>>, Error> {
-        self.inner.read().map_err(|_| Error::AcquireFrecencyLock)
-    }
-
-    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<QueryTracker>>, Error> {
-        self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
-    }
-
-    /// Initialize the query tracker. No-op if this is a disabled instance.
-    pub fn init(&self, tracker: QueryTracker) -> Result<(), Error> {
-        if !self.enabled {
-            return Ok(());
-        }
-        let mut guard = self.write()?;
-        *guard = Some(tracker);
-        Ok(())
-    }
-
-    /// Drop the in-memory tracker and delete the on-disk database directory.
-    ///
-    /// Acquires the write lock, ensuring all readers (including any active mmap
-    /// access) are finished before the LMDB environment is closed and the files
-    /// are removed.
-    ///
-    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no
-    /// tracker was initialized.
-    pub fn destroy(&self) -> Result<Option<PathBuf>, Error> {
-        let mut guard = self.write()?;
-        let Some(tracker) = guard.take() else {
-            return Ok(None);
-        };
-        let db_path = tracker.db_path().to_path_buf();
-        drop(tracker);
-        drop(guard);
-        std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir {
-            path: db_path.clone(),
-            source,
-        })?;
-        Ok(Some(db_path))
-    }
-}
+pub type SharedQueryTracker = SharedDb<QueryTracker>;

crates/fff-core/src/simd_path.rs

@@ -60,7 +60,10 @@ impl std::fmt::Debug for SimdChunk {
     }
 }
 
-pub const PATH_BUF_SIZE: usize = 4096;
+pub use crate::constants::PATH_BUF_SIZE;
+
+/// Chunk pointer capacity needed for the longest path the platform allows.
+pub(crate) const MAX_PATH_CHUNKS: usize = PATH_BUF_SIZE.div_ceil(SIMD_CHUNK_BYTES);
 
 /// Indices into a shared `SimdChunk` arena representing a file path.
 ///
@@ -98,14 +101,10 @@ impl ChunkedString {
     }
 
     #[inline]
-    pub fn resolve_ptrs<'a>(
-        &self,
-        arena: ArenaPtr,
-        buf: &'a mut [*const u8; 32],
-    ) -> &'a [*const u8] {
-        let count = self.indices.len();
+    pub fn resolve_ptrs<'a>(&self, arena: ArenaPtr, buf: &'a mut [*const u8]) -> &'a [*const u8] {
+        let count = self.indices.len().min(buf.len());
         let base = arena.as_ptr();
-        for (i, &idx) in self.indices.iter().enumerate() {
+        for (i, &idx) in self.indices[..count].iter().enumerate() {
             buf[i] = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
         }
         &buf[..count]
@@ -172,10 +171,12 @@ impl ChunkedString {
         let usable_chunks = total.div_ceil(SIMD_CHUNK_BYTES);
         let chunks_to_copy = usable_chunks.min(self.indices.len());
         let base = arena.as_ptr();
+
         for (i, &idx) in self.indices[..chunks_to_copy].iter().enumerate() {
             let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
             let dst_offset = i * SIMD_CHUNK_BYTES;
             let take = SIMD_CHUNK_BYTES.min(total - dst_offset);
+
             unsafe {
                 core::ptr::copy_nonoverlapping(src, buf.as_mut_ptr().add(dst_offset), take);
             }
@@ -293,10 +294,11 @@ pub(crate) struct ChunkedPathStoreBuilder {
 
 impl ChunkedPathStoreBuilder {
     pub fn new(estimated_files: usize) -> Self {
-        let est_chunks = estimated_files * 3;
+        let est_chunks = estimated_files * INLINE_CHUNKS; // we know that most of repos will fit
+        // most paths into 64 = 16 * INLINE_CHUNKS
         Self {
-            arena: Vec::with_capacity(est_chunks / 2),
-            chunk_dedup: AHashMap::with_capacity(est_chunks / 2),
+            arena: Vec::with_capacity(est_chunks),
+            chunk_dedup: AHashMap::with_capacity(est_chunks),
         }
     }
 
@@ -317,15 +319,11 @@ impl ChunkedPathStoreBuilder {
     pub fn add_file_immediate(&mut self, rel_path: &str, filename_offset: u16) -> ChunkedString {
         let path_bytes = rel_path.as_bytes();
         let byte_len = rel_path.len();
-        let n_chunks = chunks_needed(byte_len);
-        let mut indices = ChunkIndices::with_capacity(n_chunks);
+        let mut indices = ChunkIndices::with_capacity(chunks_needed(byte_len));
 
-        for i in 0..n_chunks {
-            let chunk_start = i * SIMD_CHUNK_BYTES;
-            let chunk_end = (chunk_start + SIMD_CHUNK_BYTES).min(byte_len);
+        for chunk in path_bytes.chunks(SIMD_CHUNK_BYTES) {
             let mut chunk_bytes = [0u8; SIMD_CHUNK_BYTES];
-            chunk_bytes[..chunk_end - chunk_start]
-                .copy_from_slice(&path_bytes[chunk_start..chunk_end]);
+            chunk_bytes[..chunk.len()].copy_from_slice(chunk);
 
             let arena_idx = match self.chunk_dedup.get(&chunk_bytes) {
                 Some(&idx) => idx,
@@ -336,6 +334,7 @@ impl ChunkedPathStoreBuilder {
                     idx
                 }
             };
+
             indices.push(arena_idx);
         }
 
@@ -416,7 +415,10 @@ mod tests {
 
     #[test]
     fn test_chunked_string_full_path() {
-        let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);
+        let (store, strings, _files) = build_test_store(&[
+            "src/components/Button.tsx",
+            "src/components/Button.test.tsx",
+        ]);
         let arena = store.as_arena_ptr();
         let cs = &strings[0];
 
@@ -424,6 +426,15 @@ mod tests {
         assert_eq!(cs.read_to_buf(arena, &mut buf), "src/components/Button.tsx");
         assert_eq!(cs.byte_len, 25);
         assert_eq!(cs.filename_offset, 15);
+
+        let cs = &strings[1];
+        let mut buf = [0u8; 512];
+        assert_eq!(
+            cs.read_to_buf(arena, &mut buf),
+            "src/components/Button.test.tsx"
+        );
+        assert_eq!(cs.byte_len, 30);
+        assert_eq!(cs.filename_offset, 15);
     }
 
     #[test]
@@ -460,7 +471,7 @@ mod tests {
         let arena = store.as_arena_ptr();
         let cs = &strings[0];
 
-        let mut ptrs = [std::ptr::null::<u8>(); 32];
+        let mut ptrs = [std::ptr::null::<u8>(); MAX_PATH_CHUNKS];
         let resolved = cs.resolve_ptrs(arena, &mut ptrs);
         assert_eq!(resolved.len(), 2); // 25 bytes = 2 chunks
 
@@ -478,6 +489,43 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_resolve_ptrs_path_exceeding_512_bytes() {
+        // Regression: a fixed 32-ptr buffer covered only 512 bytes while
+        // PATH_BUF_SIZE (libc::PATH_MAX) allows longer paths, panicking with
+        // "index out of bounds: the len is 32 but the index is 32"
+        let mut path = String::new();
+        while path.len() < 600 {
+            path.push_str("deeply_nested_directory_segment/");
+        }
+        path.push_str("needle_file.rs");
+        assert!(path.len() > 512 && path.len() < PATH_BUF_SIZE);
+
+        let (store, strings, _files) = build_test_store(&[path.as_str()]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+        assert!(cs.chunk_count() > 32, "path must span more than 32 chunks");
+
+        let mut ptrs = [std::ptr::null::<u8>(); MAX_PATH_CHUNKS];
+        let resolved = cs.resolve_ptrs(arena, &mut ptrs);
+
+        // Truncation is not acceptable either: it silently drops the tail of
+        // the path (including the filename here) from fuzzy matching.
+        assert_eq!(
+            resolved.len(),
+            cs.chunk_count(),
+            "resolve_ptrs must resolve every chunk of a PATH_MAX-legal path"
+        );
+
+        let total = cs.byte_len as usize;
+        let mut reconstructed = Vec::with_capacity(total);
+        for (i, &ptr) in resolved.iter().enumerate() {
+            let take = SIMD_CHUNK_BYTES.min(total - i * SIMD_CHUNK_BYTES);
+            reconstructed.extend_from_slice(unsafe { std::slice::from_raw_parts(ptr, take) });
+        }
+        assert_eq!(std::str::from_utf8(&reconstructed).unwrap(), path);
+    }
+
     #[test]
     fn test_filename_cow_mid_chunk() {
         let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);

crates/fff-core/src/simd_string_utils/case.rs

@@ -0,0 +1,168 @@
+#[inline]
+pub fn ascii_swap_case(b: u8) -> u8 {
+    b ^ 0x20
+}
+
+#[inline]
+fn eq_lowered_scalar(h: *const u8, needle_lower: &[u8]) -> bool {
+    for (i, &n) in needle_lower.iter().enumerate() {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != n {
+            return false;
+        }
+    }
+    true
+}
+
+/// AVX2 only has a **signed** byte compare (`cmpgt`), but we need an
+/// **unsigned** range check (`'A' <= byte <= 'Z'`). XOR-ing every byte with
+/// `0x80` maps the unsigned range `[0, 255]` into the signed range
+/// `[-128, 127]` preserving order, so signed `cmpgt` becomes correct
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn eq_lowered_avx2(h: *const u8, needle_lower: &[u8]) -> bool {
+    use core::arch::x86_64::*;
+
+    let len = needle_lower.len();
+    let mut i = 0usize;
+
+    let flip = _mm256_set1_epi8(0x80u8 as i8);
+    let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8);
+    let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8);
+    let bit20 = _mm256_set1_epi8(0x20u8 as i8);
+
+    while i + 32 <= len {
+        let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) };
+        let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) };
+
+        // Signed-domain range check selects uppercase lanes, OR bit 5 folds them.
+        let x = _mm256_xor_si256(hv, flip);
+        let ge_a = _mm256_cmpgt_epi8(x, a_minus_1);
+        let le_z = _mm256_cmpgt_epi8(z_plus_1, x);
+        let upper = _mm256_and_si256(ge_a, le_z);
+        let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20));
+
+        let eq = _mm256_cmpeq_epi8(folded, nv);
+        if _mm256_movemask_epi8(eq) != -1i32 {
+            return false;
+        }
+
+        i += 32;
+    }
+
+    while i < len {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+/// Unsigned range checks (`vcge`/`vcle`) detect uppercase ASCII, bit 5 folds
+/// to lowercase, then equality is checked via udot: xors the folded haystack
+/// with the pre-lowered needle and dot-product the difference with itself
+/// any non-zero byte produces a non-zero u32 lane. udot is emitted via inline
+/// asm because `vdotq_u32` is still behind an unstable feature gate.
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon,dotprod")]
+unsafe fn eq_lowered_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool {
+    use core::arch::aarch64::*;
+
+    let len = needle_lower.len();
+    let mut i = 0usize;
+
+    let a_val = vdupq_n_u8(b'A');
+    let z_val = vdupq_n_u8(b'Z');
+    let bit20 = vdupq_n_u8(0x20);
+
+    while i + 16 <= len {
+        let hv = unsafe { vld1q_u8(h.add(i)) };
+        let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) };
+
+        let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val));
+        let folded = vorrq_u8(hv, vandq_u8(upper, bit20));
+        let xored = veorq_u8(folded, nv);
+
+        let dots: uint32x4_t;
+        let zero = vdupq_n_u32(0);
+        unsafe {
+            core::arch::asm!(
+                "udot {d:v}.4s, {a:v}.16b, {b:v}.16b",
+                d = inlateout(vreg) zero => dots,
+                a = in(vreg) xored,
+                b = in(vreg) xored,
+            );
+        }
+
+        if vmaxvq_u32(dots) != 0 {
+            return false;
+        }
+
+        i += 16;
+    }
+
+    while i < len {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+/// Case-insensitive equality of `needle_lower` against the haystack bytes
+/// starting at `h`. `needle_lower` must be pre-lowercased (ASCII).
+///
+/// # Safety
+/// `h` must be valid for reads of `needle_lower.len()` bytes.
+#[inline]
+pub(crate) unsafe fn eq_lowered_case(haystack: *const u8, needle_lower: &[u8]) -> bool {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") {
+            return unsafe { eq_lowered_avx2(haystack, needle_lower) };
+        }
+    }
+    #[cfg(target_arch = "aarch64")]
+    {
+        if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") {
+            return unsafe { eq_lowered_neon_dotprod(haystack, needle_lower) };
+        }
+    }
+
+    eq_lowered_scalar(haystack, needle_lower)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn eq_lowered(haystack: &[u8], needle_lower: &[u8]) -> bool {
+        assert!(haystack.len() >= needle_lower.len());
+        unsafe { eq_lowered_case(haystack.as_ptr(), needle_lower) }
+    }
+
+    #[test]
+    fn swap_case_toggles_letters() {
+        assert_eq!(ascii_swap_case(b'n'), b'N');
+        assert_eq!(ascii_swap_case(b'N'), b'n');
+        assert_eq!(ascii_swap_case(b'z'), b'Z');
+    }
+
+    #[test]
+    fn eq_matches_std_semantics() {
+        assert!(eq_lowered(b"Hello", b"hello"));
+        assert!(eq_lowered(b"HELLO WORLD", b"hello"));
+        assert!(!eq_lowered(b"Hellp", b"hello"));
+        // Non-letters must not fold: '[' (0x5B) vs '{' (0x7B) differ only in bit 5.
+        assert!(!eq_lowered(b"A[", b"a{"));
+        assert!(eq_lowered(b"A{", b"a{"));
+        // Long inputs exercise the SIMD kernels.
+        let hay = b"INT STRUCT MUTEX *LOCK(STRUCT MUTEX *LOCK) { RETURN 0; }";
+        let needle: Vec<u8> = hay.iter().map(|b| b.to_ascii_lowercase()).collect();
+        assert!(eq_lowered(hay, &needle));
+        let mut bad = needle.clone();
+        *bad.last_mut().unwrap() = b'!';
+        assert!(!eq_lowered(hay, &bad));
+    }
+}

crates/fff-core/src/simd_string_utils/memmem.rs

@@ -0,0 +1,494 @@
+use super::case::{ascii_swap_case, eq_lowered_case};
+use smallvec::SmallVec;
+
+// Byte frequency table stolen from memchr
+const BYTE_FREQUENCIES: [u8; 256] = [
+    55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00
+    42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10
+    255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20
+    208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30
+    120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185,
+    167, // 0x40 A-O
+    186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114,
+    223, // 0x50 P-_
+    151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246,
+    244, // 0x60 a-o
+    231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127,
+    27, // 0x70 p-DEL
+    212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80
+    207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90
+    118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0
+    166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0
+];
+
+#[inline]
+fn rank(lower: u8) -> u8 {
+    if lower.is_ascii_lowercase() {
+        BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[ascii_swap_case(lower) as usize])
+    } else {
+        BYTE_FREQUENCIES[lower as usize]
+    }
+}
+
+/// Pick two needle positions with the rarest bytes (case-insensitive)
+fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) {
+    debug_assert!(needle_lower.len() >= 2);
+
+    let mut best1 = (u8::MAX, 0usize); // (rank, position)
+    let mut best2 = (u8::MAX, 1usize);
+
+    for (i, &b) in needle_lower.iter().enumerate() {
+        let r = rank(b);
+        if r < best1.0 {
+            best2 = best1;
+            best1 = (r, i);
+        } else if r < best2.0 && i != best1.1 {
+            best2 = (r, i);
+        }
+    }
+
+    let i1 = best1.1.min(best2.1);
+    let i2 = best1.1.max(best2.1);
+    (i1, i2)
+}
+
+/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF)
+/// Bit *i* of the result corresponds to byte *i* of the input vector
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+#[inline]
+unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 {
+    use core::arch::aarch64::*;
+
+    // AND each byte with its bit-position mask, then horizontally sum each half.
+    static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
+    let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) };
+    let masked = vandq_u8(v, bit_mask);
+    let lo = vaddv_u8(vget_low_u8(masked));
+    let hi = vaddv_u8(vget_high_u8(masked));
+    (lo as u16) | ((hi as u16) << 8)
+}
+
+/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration,
+/// checking two rare bytes (case-insensitive) simultaneously
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn find_packed_pair_avx2(
+    haystack: &[u8],
+    needle_lower: &[u8],
+    i1: usize,
+    i2: usize,
+) -> Option<usize> {
+    use core::arch::x86_64::*;
+
+    let n = needle_lower.len();
+    let hlen = haystack.len();
+    let ptr = haystack.as_ptr();
+    let last_start = hlen - n; // last valid match-start position
+
+    let b1 = needle_lower[i1];
+    let b1_alt = if b1.is_ascii_lowercase() {
+        ascii_swap_case(b1)
+    } else {
+        b1
+    };
+    let b2 = needle_lower[i2];
+    let b2_alt = if b2.is_ascii_lowercase() {
+        ascii_swap_case(b2)
+    } else {
+        b2
+    };
+
+    let v1_lo = _mm256_set1_epi8(b1 as i8);
+    let v1_hi = _mm256_set1_epi8(b1_alt as i8);
+    let v2_lo = _mm256_set1_epi8(b2 as i8);
+    let v2_hi = _mm256_set1_epi8(b2_alt as i8);
+
+    // Loads come from ptr+offset+i1 and ptr+offset+i2, so we need offset + max(i1,i2) + 31 < hlen.
+    let max_idx = i1.max(i2);
+    let max_offset = hlen.saturating_sub(max_idx + 32);
+    let mut offset = 0usize;
+
+    while offset <= max_offset {
+        let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) };
+        let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) };
+
+        // Case-insensitive match: OR both case variants, then AND the two positions.
+        let eq1 = _mm256_or_si256(
+            _mm256_cmpeq_epi8(chunk1, v1_lo),
+            _mm256_cmpeq_epi8(chunk1, v1_hi),
+        );
+        let eq2 = _mm256_or_si256(
+            _mm256_cmpeq_epi8(chunk2, v2_lo),
+            _mm256_cmpeq_epi8(chunk2, v2_hi),
+        );
+
+        let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32;
+
+        // Candidates are visited in increasing position order, so the first
+        // verified candidate is the leftmost match
+        while mask != 0 {
+            let bit = mask.trailing_zeros() as usize;
+            let candidate = offset + bit;
+            if candidate > last_start {
+                return None;
+            }
+            if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                return Some(candidate);
+            }
+            mask &= mask - 1;
+        }
+
+        offset += 32;
+    }
+
+    // handle remaining characters
+    if offset <= last_start {
+        let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) {
+            i1
+        } else {
+            i2
+        };
+        let rare_byte = needle_lower[rare_pos];
+        let tail_start = offset + rare_pos;
+        let tail_end = last_start + rare_pos + 1;
+        if tail_start < tail_end {
+            let tail_space = &haystack[tail_start..tail_end];
+            if rare_byte.is_ascii_lowercase() {
+                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            } else {
+                for pos in memchr::memchr_iter(rare_byte, tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+unsafe fn find_packed_pair_neon(
+    haystack: &[u8],
+    needle_lower: &[u8],
+    i1: usize,
+    i2: usize,
+) -> Option<usize> {
+    use core::arch::aarch64::*;
+
+    let n = needle_lower.len();
+    let hlen = haystack.len();
+    let ptr = haystack.as_ptr();
+    let last_start = hlen - n;
+
+    let b1 = needle_lower[i1];
+    let b1_alt = if b1.is_ascii_lowercase() {
+        ascii_swap_case(b1)
+    } else {
+        b1
+    };
+    let b2 = needle_lower[i2];
+    let b2_alt = if b2.is_ascii_lowercase() {
+        ascii_swap_case(b2)
+    } else {
+        b2
+    };
+
+    let v1_lo = vdupq_n_u8(b1);
+    let v1_hi = vdupq_n_u8(b1_alt);
+    let v2_lo = vdupq_n_u8(b2);
+    let v2_hi = vdupq_n_u8(b2_alt);
+
+    let max_idx = i1.max(i2);
+    let max_offset = hlen.saturating_sub(max_idx + 16);
+    let mut offset = 0usize;
+
+    while offset <= max_offset {
+        let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) };
+        let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) };
+
+        // Case-insensitive match: OR both case variants, then AND the two positions.
+        let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi));
+        let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi));
+
+        let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) };
+
+        while mask != 0 {
+            let bit = mask.trailing_zeros() as usize;
+            let candidate = offset + bit;
+            if candidate > last_start {
+                return None;
+            }
+            if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                return Some(candidate);
+            }
+            mask &= mask - 1;
+        }
+
+        offset += 16;
+    }
+
+    // Tail: remaining positions that couldn't fill a full vector.
+    if offset <= last_start {
+        let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) {
+            i1
+        } else {
+            i2
+        };
+        let rare_byte = needle_lower[rare_pos];
+        let tail_start = offset + rare_pos;
+        let tail_end = last_start + rare_pos + 1;
+        if tail_start < tail_end {
+            let tail_space = &haystack[tail_start..tail_end];
+            if rare_byte.is_ascii_lowercase() {
+                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            } else {
+                for pos in memchr::memchr_iter(rare_byte, tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+fn find_first_byte_with_memchr(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+    let n = needle_lower.len();
+    debug_assert!(n >= 1 && n <= haystack.len());
+
+    let search_space = &haystack[..=haystack.len() - n];
+    let first = needle_lower[0];
+
+    if first.is_ascii_lowercase() {
+        let alt = ascii_swap_case(first);
+        for pos in memchr::memchr2_iter(first, alt, search_space) {
+            if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } {
+                return Some(pos);
+            }
+        }
+    } else {
+        for pos in memchr::memchr_iter(first, search_space) {
+            if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } {
+                return Some(pos);
+            }
+        }
+    }
+    None
+}
+
+/// ASCII case-insensitive substring search returning the leftmost match
+/// position. `needle_lower` must be pre-lowercased (ASCII).
+// pub because it is used in out of the crate benchmarks
+#[doc(hidden)] // it's pub only for benches
+pub fn find(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+    let n = needle_lower.len();
+    if n == 0 {
+        return Some(0);
+    }
+    if n > haystack.len() {
+        return None;
+    }
+
+    if n == 1 {
+        let first = needle_lower[0];
+        return if first.is_ascii_lowercase() {
+            memchr::memchr2(first, ascii_swap_case(first), haystack)
+        } else {
+            memchr::memchr(first, haystack)
+        };
+    }
+
+    #[cfg_attr(
+        not(any(target_arch = "x86_64", target_arch = "aarch64")),
+        allow(unused_variables)
+    )]
+    let (i1, i2) = select_rare_pair(needle_lower);
+
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::is_x86_feature_detected!("avx2") {
+            // Need enough haystack for at least one vector load.
+            let max_idx = i1.max(i2);
+            if haystack.len() >= max_idx + 32 {
+                return unsafe { find_packed_pair_avx2(haystack, needle_lower, i1, i2) };
+            }
+        }
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    {
+        // Packed-pair wins when the first byte is common (memchr2 drowns in
+        // false positives), but a rare first byte (z, q, x, ...) makes
+        // memchr2's raw throughput dominate. Threshold 200 on the frequency
+        // table splits common letters (s=243, e=253) from rare ones (z=152).
+        let first_byte_rank = rank(needle_lower[0]);
+        let max_idx = i1.max(i2);
+        if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 {
+            return unsafe { find_packed_pair_neon(haystack, needle_lower, i1, i2) };
+        }
+    }
+
+    // fallbacks to memchr based implementation cause we still have it and it supports more SIMD backends
+    // TODO convert all the supported backend by memchr and get rid of the fallback
+    find_first_byte_with_memchr(haystack, needle_lower)
+}
+
+/// A case insensitive find that works better with smaller strings, doesn't unwrap a complicated
+/// AVX backend we use for grep because only cpu flags check takes usually more time than find itself
+pub fn find_case_insensitive_short(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+    debug_assert!(haystack.len() < 1024);
+    let mut needle_lower: SmallVec<[u8; 64]> = SmallVec::from_slice(needle);
+    needle_lower.make_ascii_lowercase();
+
+    find(haystack, &needle_lower)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn reference_find(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+        if needle_lower.is_empty() {
+            return Some(0);
+        }
+        if needle_lower.len() > haystack.len() {
+            return None;
+        }
+        haystack
+            .windows(needle_lower.len())
+            .position(|w| w.eq_ignore_ascii_case(needle_lower))
+    }
+
+    #[test]
+    fn basic_case_insensitive() {
+        assert_eq!(find(b"Hello World", b"hello"), Some(0));
+        assert_eq!(find(b"Hello World", b"world"), Some(6));
+        assert_eq!(find(b"NOMORE bugs", b"nomore"), Some(0));
+        assert_eq!(find(b"Hello World", b"xyz"), None);
+        assert!(find(b"Hello World", b"o w").is_some());
+    }
+
+    #[test]
+    fn edge_cases() {
+        assert_eq!(find(b"ab", b"ab"), Some(0));
+        assert_eq!(find(b"AB", b"ab"), Some(0));
+        assert_eq!(find(b"a", b"ab"), None);
+        assert_eq!(find(b"anything", b""), Some(0));
+        assert_eq!(find(b"", b"x"), None);
+        assert_eq!(find(b"xxA", b"a"), Some(2));
+        assert_eq!(find(b"xx:", b":"), Some(2));
+    }
+
+    #[test]
+    fn returns_leftmost_match() {
+        assert_eq!(find(b"foo FOO foo", b"foo"), Some(0));
+        let mut big = vec![b'.'; 300];
+        big[100..103].copy_from_slice(b"FoO");
+        big[200..203].copy_from_slice(b"foo");
+        assert_eq!(find(&big, b"foo"), Some(100));
+    }
+
+    #[test]
+    fn non_letter_bytes_do_not_case_fold() {
+        // '[' (0x5B) and '{' (0x7B) differ only in bit 5 but are not letters.
+        // A fold implemented as a bare `| 0x20` would falsely match these.
+        assert_eq!(find(b"A[", b"a{"), None);
+        assert_eq!(find(b"x@y", b"x`y"), None);
+        assert_eq!(find(b"a]b", b"a}b"), None);
+        assert_eq!(find(b"A{", b"a{"), Some(0));
+    }
+
+    #[test]
+    fn matches_reference_on_random_inputs() {
+        // Deterministic xorshift PRNG — no external deps.
+        let mut state = 0x9E3779B97F4A7C15u64;
+        let mut next = move || {
+            state ^= state << 13;
+            state ^= state >> 7;
+            state ^= state << 17;
+            state
+        };
+
+        // Alphabet with letters, both-case pairs, and 0x20-differing symbols.
+        let alphabet = b"aAbBzZ [{@`]}^~_0.\n";
+        for _ in 0..2000 {
+            let hlen = (next() % 200) as usize;
+            let nlen = (next() % 8) as usize;
+            let haystack: Vec<u8> = (0..hlen)
+                .map(|_| alphabet[(next() % alphabet.len() as u64) as usize])
+                .collect();
+            let needle: Vec<u8> = (0..nlen)
+                .map(|_| alphabet[(next() % alphabet.len() as u64) as usize].to_ascii_lowercase())
+                .collect();
+
+            assert_eq!(
+                find(&haystack, &needle),
+                reference_find(&haystack, &needle),
+                "mismatch for haystack={:?} needle={:?}",
+                haystack,
+                needle,
+            );
+        }
+    }
+
+    #[test]
+    fn long_haystack_simd_paths() {
+        let haystack =
+            b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb";
+        assert_eq!(find(haystack, b"this_is_a_long_needle_test"), Some(32));
+        assert_eq!(find(haystack, b"this_is_a_long_needle_testz"), None);
+
+        // Needle >= 16 bytes exercises SIMD verify.
+        let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }";
+        assert_eq!(find(haystack2, b"struct mutex *lock"), Some(4));
+
+        let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        assert_eq!(find(upper_hay, b"qrstuvwxyz0123456789a"), Some(16));
+        assert_eq!(find(upper_hay, b"qrstuvwxyz01234567899"), None);
+
+        // Needle at very end / very start.
+        let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me";
+        assert_eq!(find(end_hay, b"find_me"), Some(end_hay.len() - 7));
+        assert_eq!(find(end_hay, b"xx"), Some(0));
+
+        // 1KB haystack with needle near the end.
+        let mut big = vec![b'z'; 1024];
+        big[1000..1010].copy_from_slice(b"hElLo_WoRl");
+        assert_eq!(find(&big, b"hello_wo"), Some(1000));
+        assert_eq!(find(&big, b"hello_world"), None);
+    }
+
+    #[test]
+    fn rare_pair_selection() {
+        let (i1, i2) = select_rare_pair(b"nomore");
+        let ranks: Vec<u8> = b"nomore".iter().map(|&b| rank(b)).collect();
+        let (r1, r2) = (ranks[i1], ranks[i2]);
+        for (i, &r) in ranks.iter().enumerate() {
+            if i != i1 && i != i2 {
+                assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal");
+            }
+        }
+    }
+}

crates/fff-core/src/simd_string_utils/mod.rs

@@ -0,0 +1,5 @@
+//! SIMD-accelerated string utilities: case flipping/folding and
+//! case-insensitive substring search.
+
+pub mod case;
+pub mod memmem;

crates/fff-core/src/stable_vec.rs

@@ -0,0 +1,177 @@
+use std::alloc::{self, Layout};
+use std::ptr::NonNull;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// Vector that guarantees no re-alloc happening at runtime
+pub(crate) struct StableVec<T> {
+    inner: Arc<StableBuf<T>>,
+}
+
+struct StableBuf<T> {
+    ptr: NonNull<T>,
+    cap: usize,
+    /// Atomic because:
+    ///   1. `push(&self)` must mutate this through a shared `&StableBuf`,
+    ///      which requires interior mutability.
+    ///   2. Arc clones (e.g. post-scan snapshots) read `len` outside the
+    ///      picker lock, concurrent with an appending writer. Acquire/Release
+    ///      on len is what makes "observed len ⇒ element bytes initialized"
+    ///      actually hold.
+    ///
+    /// Arc wrapping only shares ownership of the buffer; it does NOT
+    /// synchronize access to fields inside the shared buffer.
+    len: AtomicUsize,
+}
+
+// SAFETY: StableBuf is a thread-safe container when T is send + sync
+// There is another application level constraint: mutations are safe
+// when they are atomic updates, not read + update.
+unsafe impl<T: Send> Send for StableBuf<T> {}
+unsafe impl<T: Sync> Sync for StableBuf<T> {}
+
+impl<T> Drop for StableBuf<T> {
+    fn drop(&mut self) {
+        let len = *self.len.get_mut();
+        unsafe {
+            std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), len));
+            if self.cap > 0 {
+                let layout = Layout::array::<T>(self.cap).expect("layout");
+                alloc::dealloc(self.ptr.as_ptr().cast(), layout);
+            }
+        }
+    }
+}
+
+impl<T> StableVec<T> {
+    pub fn from_vec_with_reserve(mut vec: Vec<T>, extra: usize) -> Self {
+        vec.reserve(extra);
+        let cap = vec.capacity();
+        let len = vec.len();
+
+        let inner = if cap == 0 {
+            StableBuf {
+                ptr: NonNull::dangling(),
+                cap: 0,
+                len: AtomicUsize::new(0),
+            }
+        } else {
+            // Take ownership of the Vec's buffer without running element
+            // drops; we hand them off to the StableBuf.
+            let mut vec = std::mem::ManuallyDrop::new(vec);
+            let ptr = NonNull::new(vec.as_mut_ptr()).expect("non-null");
+            StableBuf {
+                ptr,
+                cap,
+                len: AtomicUsize::new(len),
+            }
+        };
+
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+
+    /// Append. Returns `false` if capacity is exhausted (item dropped).
+    ///
+    /// Safe to call via `&self` as long as the caller holds the outer
+    /// picker write lock (single-writer invariant).
+    #[inline]
+    pub fn push(&self, item: T) -> bool {
+        let cap = self.inner.cap;
+        let len = self.inner.len.load(Ordering::Acquire);
+        if len >= cap {
+            debug_assert!(
+                false,
+                "StableVec: push would exceed capacity ({len} at capacity {cap})"
+            );
+            tracing::error!(
+                len,
+                capacity = cap,
+                "StableVec: capacity exhausted — dropping item to prevent reallocation"
+            );
+            return false;
+        }
+        unsafe {
+            std::ptr::write(self.inner.ptr.as_ptr().add(len), item);
+        }
+        self.inner.len.store(len + 1, Ordering::Release);
+        true
+    }
+
+    // this method is specifically private because you probably need to use
+    // live_count if you are trying to access this method
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.inner.len.load(Ordering::Acquire)
+    }
+
+    /// Mutable element access for in-place field updates. Never shifts.
+    ///
+    /// LATENT UB: produces `&mut T` aliasing Arc-shared memory; the
+    /// `&mut self` on StableVec does NOT imply unique access to the
+    /// `StableBuf` when sibling Arc clones exist. Safe in practice
+    /// because callers hold the picker write lock and writes target
+    /// disjoint fields, but strictly forbidden by the aliasing model.
+    #[inline]
+    pub fn get_mut(&mut self, index: usize) -> Option<&mut T> {
+        let len = self.inner.len.load(Ordering::Acquire);
+        if index >= len {
+            return None;
+        }
+        unsafe { Some(&mut *self.inner.ptr.as_ptr().add(index)) }
+    }
+
+    #[inline]
+    pub fn last(&self) -> Option<&T> {
+        let len = self.len();
+        if len == 0 {
+            None
+        } else {
+            unsafe { Some(&*self.inner.ptr.as_ptr().add(len - 1)) }
+        }
+    }
+
+    /// Iterate mutably for in-place field updates. Never shifts storage.
+    /// Same latent-UB caveat as [`get_mut`]: `&mut T` into Arc-shared memory.
+    #[inline]
+    pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> {
+        let len = self.inner.len.load(Ordering::Acquire);
+        unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len).iter_mut() }
+    }
+}
+
+impl<T> Clone for StableVec<T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        Self {
+            inner: Arc::clone(&self.inner),
+        }
+    }
+}
+
+impl<T: std::fmt::Debug> std::fmt::Debug for StableVec<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("StableVec").field(&self.len()).finish()
+    }
+}
+
+impl<T> std::ops::Deref for StableVec<T> {
+    type Target = [T];
+    #[inline]
+    fn deref(&self) -> &[T] {
+        let len = self.len();
+        unsafe { std::slice::from_raw_parts(self.inner.ptr.as_ptr(), len) }
+    }
+}
+
+impl<T> std::ops::DerefMut for StableVec<T> {
+    /// LATENT UB: `&mut [T]` aliases Arc-shared memory. Kept for
+    /// Index/IndexMut ergonomics at call sites that write disjoint
+    /// fields under the picker write lock. See module-level doc.
+    #[inline]
+    fn deref_mut(&mut self) -> &mut [T] {
+        let len = self.inner.len.load(Ordering::Acquire);
+        unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len) }
+    }
+}

crates/fff-core/src/types.rs

@@ -1,11 +1,15 @@
 use std::io::Read;
 use std::path::{Path, PathBuf};
+#[cfg(not(target_os = "windows"))]
 use std::sync::OnceLock;
-use std::sync::atomic::{AtomicI32, AtomicU64, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicI32, AtomicU8, AtomicU64, AtomicUsize, Ordering};
 
-use crate::constraints::Constrainable;
+#[cfg(not(target_os = "windows"))]
+use crate::constants::{FRESH_MMAP_THRESHOLD, MMAP_THRESHOLD};
+use crate::constants::{MAX_CACHED_CONTENT_BYTES, MAX_FFFILE_SIZE, PATH_BUF_SIZE};
+use crate::index::constraints::Constrainable;
 use crate::query_tracker::QueryMatchEntry;
-use crate::simd_path::{ArenaPtr, PATH_BUF_SIZE};
+use crate::simd_path::ArenaPtr;
 use fff_query_parser::{FFFQuery, FuzzyQuery, Location};
 
 /// Different sources of the string storage used by FFF
@@ -38,30 +42,14 @@ impl FFFStringStorage for ArenaPtr {
     }
 }
 
-/// Cached file contents — mmap on Unix, heap buffer on Windows.
-///
-/// On Windows, memory-mapped files hold the file handle open and prevent
-/// editors from saving (writing/replacing) those files. Reading into a
-/// `Vec<u8>` releases the handle immediately after the read completes.
-///
-/// The `Buffer` variant is also used on Unix for temporary (uncached) reads
-/// where the mmap/munmap syscall overhead exceeds the cost of a heap copy.
-#[derive(Debug)]
-#[allow(dead_code)] // variants are conditionally used per platform
-pub enum FileContent {
-    #[cfg(not(target_os = "windows"))]
-    Mmap(memmap2::Mmap),
-    Buffer(Vec<u8>),
+pub trait FileSliceExt {
+    fn live_count(&self) -> usize;
 }
 
-impl std::ops::Deref for FileContent {
-    type Target = [u8];
-    fn deref(&self) -> &[u8] {
-        match self {
-            #[cfg(not(target_os = "windows"))]
-            FileContent::Mmap(m) => m,
-            FileContent::Buffer(b) => b,
-        }
+impl FileSliceExt for [FileItem] {
+    #[inline]
+    fn live_count(&self) -> usize {
+        self.iter().filter(|f| !f.is_deleted()).count()
     }
 }
 
@@ -81,6 +69,7 @@ pub struct DirFlags;
 
 impl DirFlags {
     pub const OVERFLOW: u8 = 1 << 0;
+    pub const DELETED: u8 = 1 << 1;
 }
 
 /// A directory in the file index. Shares chunk arena with file paths.
@@ -110,7 +99,25 @@ impl Clone for DirItem {
 impl DirItem {
     #[inline(always)]
     pub fn is_overflow(&self) -> bool {
-        self.flags & DirFlags::OVERFLOW == 0
+        self.flags & DirFlags::OVERFLOW != 0
+    }
+
+    #[inline(always)]
+    pub fn is_deleted(&self) -> bool {
+        self.flags & DirFlags::DELETED != 0
+    }
+
+    /// Marks the dir deleted/restored. Returns `true` when the state changed.
+    pub(crate) fn set_deleted(&mut self, deleted: bool) -> bool {
+        if self.is_deleted() == deleted {
+            return false;
+        }
+        if deleted {
+            self.flags |= DirFlags::DELETED;
+        } else {
+            self.flags &= !DirFlags::DELETED;
+        }
+        true
     }
 
     pub(crate) fn new(path: crate::simd_path::ChunkedString, last_segment_offset: u16) -> Self {
@@ -122,6 +129,19 @@ impl DirItem {
         }
     }
 
+    /// A dir appended after the initial scan; its path lives in the overflow arena.
+    pub(crate) fn new_overflow(
+        path: crate::simd_path::ChunkedString,
+        last_segment_offset: u16,
+    ) -> Self {
+        Self {
+            path,
+            flags: DirFlags::OVERFLOW,
+            last_segment_offset,
+            max_access_frecency: AtomicI32::new(0),
+        }
+    }
+
     /// Byte offset of the last path segment within the directory path.
     #[inline]
     pub fn last_segment_offset(&self) -> u16 {
@@ -217,6 +237,11 @@ impl Constrainable for DirItem {
     fn git_status(&self) -> Option<git2::Status> {
         None
     }
+
+    #[inline]
+    fn is_overflow(&self) -> bool {
+        DirItem::is_overflow(self)
+    }
 }
 
 #[derive(Debug)]
@@ -227,28 +252,43 @@ pub struct FileItem {
     pub modification_frecency_score: i16,
     pub git_status: Option<git2::Status>,
     pub(crate) path: crate::simd_path::ChunkedString,
-    parent_dir: u32,
-    flags: u8,
-    content: OnceLock<FileContent>,
+    pub(crate) parent_dir_index: u32,
+    flags: AtomicU8,
+    /// Lazy mmap cache. Only populated by the actual file read, controlled by the budget.
+    #[cfg(not(target_os = "windows"))]
+    content: OnceLock<memmap2::Mmap>,
 }
 
 impl Clone for FileItem {
     fn clone(&self) -> Self {
         Self {
             path: self.path.clone(),
-            parent_dir: self.parent_dir,
+            parent_dir_index: self.parent_dir_index,
             size: self.size,
             modified: self.modified,
             access_frecency_score: self.access_frecency_score,
             modification_frecency_score: self.modification_frecency_score,
             git_status: self.git_status,
-            flags: self.flags,
+            flags: AtomicU8::new(self.flags.load(Ordering::Relaxed)),
             // on clone we have to reset the content lock
+            #[cfg(not(target_os = "windows"))]
             content: OnceLock::new(),
         }
     }
 }
 
+/// Single-block read used by the binary classifier. Most binaries reveal a
+/// NUL byte within the first filesystem block, so 16 KB lets one read settle
+/// the classification for typical files while keeping the scratch buffer
+/// small enough to live on the stack.
+pub const BINARY_CLASSIFICATION_CHUNK_SIZE: usize = 16 * 1024;
+
+/// A file is treated as binary if any NUL byte appears in the scanned prefix.
+#[inline]
+pub(crate) fn detect_binary_content(content: &[u8]) -> bool {
+    memchr::memchr(0, content).is_some()
+}
+
 impl FileItem {
     pub fn new_raw(
         filename_start: u16,
@@ -267,13 +307,14 @@ impl FileItem {
 
         Self {
             path,
-            parent_dir: u32::MAX,
+            parent_dir_index: u32::MAX,
             size,
             modified,
             access_frecency_score: 0,
             modification_frecency_score: 0,
             git_status,
-            flags,
+            flags: AtomicU8::new(flags),
+            #[cfg(not(target_os = "windows"))]
             content: OnceLock::new(),
         }
     }
@@ -289,14 +330,6 @@ impl FileItem {
         self.path = path;
     }
 
-    pub(crate) fn parent_dir_index(&self) -> u32 {
-        self.parent_dir
-    }
-
-    pub(crate) fn set_parent_dir(&mut self, idx: u32) {
-        self.parent_dir = idx;
-    }
-
     pub fn dir_str(&self, arena: impl FFFStringStorage) -> String {
         let mut s = String::with_capacity(64);
         self.path.write_dir_to(arena.arena_for(self), &mut s);
@@ -350,6 +383,9 @@ impl FileItem {
         path.starts_with(prefix)
     }
 
+    /// Write `base_path + '/' + relative_path` into `buf` and return it
+    /// as `&Path`. Takes a fixed-size array so the buffer can live on
+    /// the stack (no heap allocation, no bounds checks in the hot loop).
     pub(crate) fn write_absolute_path<'a>(
         &self,
         arena: ArenaPtr,
@@ -359,77 +395,242 @@ impl FileItem {
         let base = base_path.as_os_str().as_encoded_bytes();
         let base_len = base.len();
         buf[..base_len].copy_from_slice(base);
-        // Add separator if base doesn't end with one
-        let sep_len = if base_len > 0 && base[base_len - 1] != b'/' {
-            buf[base_len] = b'/';
+        let sep_len = if base_len > 0 && base[base_len - 1] != std::path::MAIN_SEPARATOR as u8 {
+            buf[base_len] = std::path::MAIN_SEPARATOR as u8;
             1
         } else {
             0
         };
-        let rel_start = base_len + sep_len;
-        let mut rel_buf = [0u8; PATH_BUF_SIZE];
-        let rel = self.path.read_to_buf(arena, &mut rel_buf);
-        let rel_bytes = rel.as_bytes();
-        buf[rel_start..rel_start + rel_bytes.len()].copy_from_slice(rel_bytes);
-        let total = rel_start + rel_bytes.len();
+
+        let base_end_idx = base_len + sep_len;
+        let relative_portion_str = self.path.read_to_buf(arena, &mut buf[base_end_idx..]);
+        let rel_len = relative_portion_str.len();
+        let total = base_end_idx + rel_len;
+        // Stored relative paths are '/'-canonical; rewrite to the OS-native
+        // separator so the result matches git-cache keys, the frecency DB, and
+        // Win32 file APIs. No-op off Windows.
+        crate::path_utils::nativize_slashes_in_place(&mut buf[base_end_idx..total]);
         Path::new(unsafe { std::str::from_utf8_unchecked(&buf[..total]) })
     }
 
+    /// Write the relative path into `buf` and NUL-terminate, returning
+    /// a `&CStr`. Fixed-size array so the buffer is stack-allocatable.
+    ///
+    /// Paired with a parent-directory fd this eliminates the per-file
+    /// absolute-path memcpy: `openat(dir_fd, cstr.as_ptr(), O_RDONLY)`
+    /// resolves the name relative to `dir_fd`. Unix-only.
+    #[cfg(unix)]
+    pub(crate) fn write_relative_cstr<'a>(
+        &self,
+        arena: ArenaPtr,
+        buf: &'a mut [u8; PATH_BUF_SIZE],
+    ) -> &'a std::ffi::CStr {
+        // Reserve the last byte for the NUL terminator.
+        let rel = self.path.read_to_buf(arena, &mut buf[..PATH_BUF_SIZE - 1]);
+        let n = rel.len();
+        buf[n] = 0;
+        // SAFETY: `buf[..=n]` ends with the NUL we just wrote and
+        // filesystem paths never contain interior NULs.
+        unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(&buf[..=n]) }
+    }
+
     #[inline]
     pub fn total_frecency_score(&self) -> i32 {
         self.access_frecency_score as i32 + self.modification_frecency_score as i32
     }
 
+    #[allow(dead_code)]
     #[inline]
-    pub fn is_binary(&self) -> bool {
-        self.flags & FileItemFlags::BINARY != 0
+    pub(crate) fn is_likely_hot(&self) -> bool {
+        self.access_frecency_score > 0 || self.git_status.is_some()
+    }
+
+    /// Reads a fixed bytes count from the file optimized for quick speed of opening
+    #[inline]
+    pub(crate) fn read_trimmed_into_buf(
+        &self,
+        base_fd: i32,
+        base_path: &Path,
+        arena: ArenaPtr,
+        path_buf: &mut [u8; PATH_BUF_SIZE],
+        buf: &mut [u8],
+    ) -> usize {
+        #[cfg(unix)]
+        {
+            self.read_into_buf_unix(base_fd, base_path, arena, path_buf, buf)
+        }
+        #[cfg(not(unix))]
+        {
+            let _ = base_fd;
+            self.read_into_buf_std(base_path, arena, path_buf, buf)
+        }
+    }
+
+    #[cfg(unix)]
+    fn read_into_buf_unix(
+        &self,
+        base_fd: libc::c_int,
+        base_path: &Path,
+        arena: ArenaPtr,
+        path_buf: &mut [u8; PATH_BUF_SIZE],
+        buf: &mut [u8],
+    ) -> usize {
+        let fd = if base_fd >= 0 {
+            let relative_path = self.write_relative_cstr(arena, path_buf);
+            // SAFETY: `relative_path` is NUL-terminated, `base_fd` is a
+            // valid directory descriptor owned by the caller.
+            unsafe { libc::openat(base_fd, relative_path.as_ptr(), libc::O_RDONLY) }
+        } else {
+            use std::os::unix::io::IntoRawFd;
+            let abs = self.write_absolute_path(arena, base_path, path_buf);
+            match std::fs::File::open(abs) {
+                Ok(f) => f.into_raw_fd(),
+                Err(e) => {
+                    tracing::error!(?e, "Failed to fopen file");
+                    return 0;
+                }
+            }
+        };
+        if fd < 0 {
+            return 0;
+        }
+
+        let mut filled = 0usize;
+        while filled < buf.len() {
+            // SAFETY: `fd` is an owned descriptor, `buf[filled..]` is a
+            // valid writable slice for `buf.len() - filled` bytes.
+            let n = unsafe {
+                libc::read(
+                    fd,
+                    buf[filled..].as_mut_ptr() as *mut libc::c_void,
+                    (buf.len() - filled) as libc::size_t,
+                )
+            };
+            if n <= 0 {
+                break;
+            }
+            filled += n as usize;
+        }
+
+        // SAFETY: matching close for the owned descriptor.
+        unsafe { libc::close(fd) };
+        filled
+    }
+
+    #[cfg(not(unix))]
+    fn read_into_buf_std(
+        &self,
+        base_path: &Path,
+        arena: ArenaPtr,
+        path_buf: &mut [u8; PATH_BUF_SIZE],
+        buf: &mut [u8],
+    ) -> usize {
+        let abs = self.write_absolute_path(arena, base_path, path_buf);
+        let Ok(mut f) = std::fs::File::open(abs) else {
+            return 0;
+        };
+        let mut filled = 0usize;
+        while filled < buf.len() {
+            match f.read(&mut buf[filled..]) {
+                Ok(0) => break,
+                Ok(n) => filled += n,
+                Err(_) => return 0,
+            }
+        }
+        filled
     }
 
     #[inline]
-    pub fn set_binary(&mut self, val: bool) {
+    pub fn is_binary(&self) -> bool {
+        self.flags.load(Ordering::Relaxed) & FileItemFlags::BINARY != 0
+    }
+
+    #[inline]
+    pub fn set_binary(&self, val: bool) {
         if val {
-            self.flags |= FileItemFlags::BINARY;
+            self.flags
+                .fetch_or(FileItemFlags::BINARY, Ordering::Relaxed);
         } else {
-            self.flags &= !FileItemFlags::BINARY;
+            self.flags
+                .fetch_and(!FileItemFlags::BINARY, Ordering::Relaxed);
+        }
+    }
+
+    /// Chunked classifier of the binary content of the file chunk by chunk
+    /// accepts path which to reuse the allocated buffer for absolute path read
+    pub(crate) fn detect_binary_per_byte(&self, path: &Path, chunk: &mut [u8]) {
+        if self.size == 0 {
+            return;
+        }
+
+        let Ok(mut file) = std::fs::OpenOptions::new()
+            .write(false)
+            .read(true)
+            .open(path)
+        else {
+            tracing::error!(path = ?path.display(), "Failed to open indexed file");
+            return;
+        };
+
+        loop {
+            match file.read(chunk) {
+                Ok(0) => break,
+                Err(e) => {
+                    tracing::error!(?e, "Failed to read file chunk");
+                    break;
+                }
+                Ok(n) => {
+                    if detect_binary_content(&chunk[..n]) {
+                        self.set_binary(true);
+                    }
+                }
+            }
         }
     }
 
     #[inline]
     pub fn is_deleted(&self) -> bool {
-        self.flags & FileItemFlags::DELETED != 0
+        self.flags.load(Ordering::Relaxed) & FileItemFlags::DELETED != 0
     }
 
     #[inline]
-    pub fn set_deleted(&mut self, val: bool) {
+    #[doc(hidden)]
+    /// Don't use it, use FilePicker::delete_file
+    pub fn set_deleted(&self, val: bool) {
         if val {
-            self.flags |= FileItemFlags::DELETED;
+            self.flags
+                .fetch_or(FileItemFlags::DELETED, Ordering::Relaxed);
         } else {
-            self.flags &= !FileItemFlags::DELETED;
+            self.flags
+                .fetch_and(!FileItemFlags::DELETED, Ordering::Relaxed);
         }
     }
 
     #[inline]
     pub fn is_overflow(&self) -> bool {
-        self.flags & FileItemFlags::OVERFLOW != 0
+        self.flags.load(Ordering::Relaxed) & FileItemFlags::OVERFLOW != 0
     }
 
     #[inline]
-    pub fn set_overflow(&mut self, val: bool) {
+    pub fn set_overflow(&self, val: bool) {
         if val {
-            self.flags |= FileItemFlags::OVERFLOW;
+            self.flags
+                .fetch_or(FileItemFlags::OVERFLOW, Ordering::Relaxed);
         } else {
-            self.flags &= !FileItemFlags::OVERFLOW;
+            self.flags
+                .fetch_and(!FileItemFlags::OVERFLOW, Ordering::Relaxed);
         }
     }
 }
 
 impl FileItem {
-    /// Invalidate the cached content so the next `get_content()` call creates a fresh one.
+    /// Invalidate the cached mmap content, has to be called every time the file is updated.
     ///
     /// Call this when the background watcher detects that the file has been modified.
     /// On Unix, a file that is truncated while mapped can cause SIGBUS. On Windows,
     /// the stale buffer simply won't reflect the new contents. In both cases,
     /// invalidating ensures a fresh read on the next access.
+    #[cfg(not(target_os = "windows"))]
     pub fn invalidate_mmap(&mut self, budget: &ContentCacheBudget) {
         if self.content.get().is_some() {
             budget.cached_count.fetch_sub(1, Ordering::Relaxed);
@@ -439,6 +640,28 @@ impl FileItem {
         self.content = OnceLock::new();
     }
 
+    #[cfg(target_os = "windows")]
+    pub fn invalidate_mmap(&mut self, _: &ContentCacheBudget) {}
+
+    pub fn update_metadata(
+        &mut self,
+        budget: &ContentCacheBudget,
+        modified_secs: Option<u64>,
+        new_size: Option<u64>,
+    ) {
+        if let Some(modified) = modified_secs
+            && self.modified < modified
+        {
+            self.modified = modified;
+        }
+
+        self.invalidate_mmap(budget);
+
+        if let Some(size) = new_size {
+            self.size = size;
+        }
+    }
+
     /// Get the cached file contents or lazily load and cache them.
     ///
     /// Returns `None` if the file is too large, empty, can't be opened, **or
@@ -446,7 +669,26 @@ impl FileItem {
     /// of the budget should use [`get_content_for_search`].
     ///
     /// After the first call, this is lock-free (just an atomic load + pointer deref).
-    pub(crate) fn get_content(
+    ///
+    /// On Windows we never back this cache — `memmap2` would require a full
+    /// `std::fs::read` heap copy and the OS page cache already absorbs repeat
+    /// reopens. Returning `None` keeps callers on the scratch-read slow path
+    /// and avoids duplicating every indexed file on the heap.
+    #[cfg(target_os = "windows")]
+    pub(crate) fn get_cached_content(
+        &self,
+        _arena: ArenaPtr,
+        _base_path: &Path,
+        _budget: &ContentCacheBudget,
+    ) -> Option<&[u8]> {
+        None
+    }
+
+    /// Returns a reference to a cached mmap of the file's contents.
+    ///
+    /// SAFETY-CRITICAL: callers must hold the picker read lock for as long as the returned slice is in use.
+    #[cfg(not(target_os = "windows"))]
+    pub(crate) fn get_cached_content(
         &self,
         arena: ArenaPtr,
         base_path: &Path,
@@ -456,8 +698,7 @@ impl FileItem {
             return Some(content);
         }
 
-        let max_file_size = budget.max_file_size;
-        if self.size == 0 || self.size > max_file_size {
+        if self.size < MMAP_THRESHOLD || self.size > budget.max_file_size {
             return None;
         }
 
@@ -470,11 +711,14 @@ impl FileItem {
             return None;
         }
 
-        let content = load_file_content(&self.absolute_path(arena, base_path), self.size)?;
-        let result = self.content.get_or_init(|| content);
+        let path = self.absolute_path(arena, base_path);
+        let file = std::fs::File::open(&path).ok()?;
+        // SAFETY: the mmap is backed by the kernel page cache and reflects
+        // file updates; the only risk is SIGBUS on a concurrent truncate,
+        // which the watcher mitigates by invalidating on modification.
+        let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
+        let result = self.content.get_or_init(|| mmap);
 
-        // Bump counters. Slight over-count under races is fine — the budget
-        // is a soft limit and the overshoot is bounded by rayon thread count.
         budget.cached_count.fetch_add(1, Ordering::Relaxed);
         budget.cached_bytes.fetch_add(self.size, Ordering::Relaxed);
 
@@ -489,14 +733,20 @@ impl FileItem {
     #[inline]
     pub(crate) fn get_content_for_search<'a>(
         &'a self,
-        buf: &'a mut Vec<u8>, // we allow it to grow
+        buf: &'a mut Vec<u8>,
+        #[cfg_attr(target_os = "windows", allow(unused_variables))] mmap_slot: &'a mut MmapSlot,
         arena: ArenaPtr,
         base_path: &Path,
         budget: &ContentCacheBudget,
     ) -> Option<&'a [u8]> {
-        // Fast path: persistent cache hit (zero-copy).
-        if let Some(cached) = self.get_content(arena, base_path, budget) {
-            return Some(cached);
+        #[cfg(not(target_os = "windows"))]
+        {
+            // Fast path: persistent cache hit (zero-copy). Safe here because
+            // grep callers hold the picker read lock for the lifetime of the
+            // returned slice — see [`Self::get_cached_content`] safety note.
+            if let Some(cached) = self.get_cached_content(arena, base_path, budget) {
+                return Some(cached);
+            }
         }
 
         let max_file_size = budget.max_file_size;
@@ -504,48 +754,34 @@ impl FileItem {
             return None;
         }
 
-        // Slow path: read into the reusable buffer — open() + read_exact() + close().
-        // No mmap()/munmap() syscalls, no page table setup/teardown.
-        // We know the exact size so we use read_exact (1 read syscall) instead of
-        // read_to_end (2 read syscalls — one for data, one for EOF confirmation).
         let abs = self.absolute_path(arena, base_path);
+
+        #[cfg(not(target_os = "windows"))]
+        if self.size >= FRESH_MMAP_THRESHOLD {
+            let file = std::fs::File::open(&abs).ok()?;
+            let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
+            let stored = mmap_slot.insert(mmap);
+            return Some(&stored[..]);
+        } else {
+            let _ = (mmap_slot, arena);
+        }
+
         let len = self.size as usize;
         buf.resize(len, 0);
+
         let mut file = std::fs::File::open(&abs).ok()?;
         file.read_exact(buf).ok()?;
         Some(buf.as_slice())
     }
 }
 
-/// Files smaller than one page waste the remainder when mmapped.
-#[cfg(target_arch = "aarch64")]
-const MMAP_THRESHOLD: u64 = 16 * 1024;
-#[cfg(not(target_arch = "aarch64"))]
-const MMAP_THRESHOLD: u64 = 4 * 1024;
-
-fn load_file_content(path: &Path, size: u64) -> Option<FileContent> {
-    #[cfg(not(target_os = "windows"))]
-    {
-        if size < MMAP_THRESHOLD {
-            let data = std::fs::read(path).ok()?;
-            Some(FileContent::Buffer(data))
-        } else {
-            let file = std::fs::File::open(path).ok()?;
-            // SAFETY: The mmap is backed by the kernel page cache and automatically
-            // reflects file modifications. The only risk is SIGBUS if the file is
-            // truncated while mapped.
-            let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
-            Some(FileContent::Mmap(mmap))
-        }
-    }
-
-    #[cfg(target_os = "windows")]
-    {
-        let _ = size;
-        let data = std::fs::read(path).ok()?;
-        Some(FileContent::Buffer(data))
-    }
-}
+/// Per-thread scratch slot owning a transient mmap returned from
+/// [`FileItem::get_content_for_search`]. `Option<Mmap>` on Unix,
+/// unit on Windows where mmap is unused.
+#[cfg(not(target_os = "windows"))]
+pub type MmapSlot = Option<memmap2::Mmap>;
+#[cfg(target_os = "windows")]
+pub type MmapSlot = ();
 
 impl Constrainable for FileItem {
     #[inline]
@@ -562,6 +798,11 @@ impl Constrainable for FileItem {
     fn git_status(&self) -> Option<git2::Status> {
         self.git_status
     }
+
+    #[inline]
+    fn is_overflow(&self) -> bool {
+        FileItem::is_overflow(self)
+    }
 }
 
 #[derive(Debug, Clone, Default)]
@@ -622,6 +863,7 @@ impl ScoringContext<'_> {
 pub struct SearchResult<'a> {
     pub items: Vec<&'a FileItem>,
     pub scores: Vec<Score>,
+    pub match_byte_offsets: Vec<smallvec::SmallVec<[(u32, u32); 4]>>,
     pub total_matched: usize,
     pub total_files: usize,
     pub location: Option<Location>,
@@ -662,10 +904,6 @@ impl Default for MixedItemRef<'_> {
     }
 }
 
-const MAX_MMAP_FILE_SIZE: u64 = 10 * 1024 * 1024;
-
-const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
-
 #[derive(Debug)]
 pub struct ContentCacheBudget {
     pub max_files: usize,
@@ -680,7 +918,7 @@ impl ContentCacheBudget {
         Self {
             max_files: usize::MAX,
             max_bytes: u64::MAX,
-            max_file_size: MAX_MMAP_FILE_SIZE,
+            max_file_size: MAX_FFFILE_SIZE,
             cached_count: AtomicUsize::new(0),
             cached_bytes: AtomicU64::new(0),
         }
@@ -696,6 +934,12 @@ impl ContentCacheBudget {
         }
     }
 
+    // Byte budget
+    pub fn is_exhausted(&self) -> bool {
+        self.cached_count.load(Ordering::Relaxed) >= self.max_files
+            || self.cached_bytes.load(Ordering::Relaxed) >= self.max_bytes
+    }
+
     pub fn new_for_repo(file_count: usize) -> Self {
         let max_files = if file_count > 50_000 {
             5_000
@@ -716,7 +960,7 @@ impl ContentCacheBudget {
         Self {
             max_files,
             max_bytes,
-            max_file_size: MAX_MMAP_FILE_SIZE,
+            max_file_size: MAX_FFFILE_SIZE,
             cached_count: AtomicUsize::new(0),
             cached_bytes: AtomicU64::new(0),
         }
@@ -758,43 +1002,3 @@ impl Default for ContentCacheBudget {
         Self::new_for_repo(30_000)
     }
 }
-
-#[cfg(test)]
-impl FileItem {
-    /// Leaks a single-file arena so the pointer stays valid forever.
-    pub fn new_for_test(
-        rel_path: &str,
-        size: u64,
-        modified: u64,
-        git_status: Option<git2::Status>,
-        is_binary: bool,
-    ) -> Self {
-        let (item, _arena) =
-            Self::new_for_test_with_arena(rel_path, size, modified, git_status, is_binary);
-        item
-    }
-
-    pub(crate) fn new_for_test_with_arena(
-        rel_path: &str,
-        size: u64,
-        modified: u64,
-        git_status: Option<git2::Status>,
-        is_binary: bool,
-    ) -> (Self, ArenaPtr) {
-        let filename_start = rel_path
-            .rfind(std::path::is_separator)
-            .map(|i| i + 1)
-            .unwrap_or(0) as u16;
-        let mut item = Self::new_raw(filename_start, size, modified, git_status, is_binary);
-        let paths = [rel_path.to_string()];
-        let (store, strings) = crate::simd_path::build_chunked_path_store_from_strings(
-            &paths,
-            std::slice::from_ref(&item),
-        );
-        let cs = strings.into_iter().next().unwrap();
-        let arena = store.as_arena_ptr();
-        item.set_path(cs);
-        std::mem::forget(store);
-        (item, arena)
-    }
-}

crates/fff-core/src/walk/mod.rs

@@ -0,0 +1,148 @@
+//! Filesystem traversal backend. Selects one implementation at compile time:
+//! - `zlob`: zlob's native parallel walker (requires the Zig toolchain).
+//! - `ripgrep`: the `ignore` crate (ripgrep's walker), used by default.
+//!
+//! Both expose [`walk_collect_files`] with identical semantics so the rest of
+//! the crate stays backend-agnostic.
+
+use crate::types::FileItem;
+use std::path::Path;
+
+#[cfg(feature = "zlob")]
+mod zlob;
+#[cfg(feature = "zlob")]
+pub(crate) use zlob::walk_collect_files;
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+mod ripgrep;
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) use ripgrep::walk_collect_files;
+
+pub(crate) struct WalkOutput {
+    pub(crate) pairs: Vec<(FileItem, String)>,
+    /// Every non-ignored directory the walk visited, relative, ending with /
+    pub(crate) dirs: Vec<String>,
+    pub(crate) ignore_rules: Option<WalkIgnoreRules>,
+}
+
+pub(crate) struct WalkIgnoreRules {
+    #[cfg(feature = "zlob")]
+    inner: ::zlob::walk::WalkerOutcomeRules,
+    #[cfg(not(feature = "zlob"))]
+    _never: std::convert::Infallible,
+}
+
+// SAFETY: the underlying storage is immutable, heap-owned, and thread-safe to
+// read from concurrently (mirrors zlob's `IgnoreRules: Send + Sync`).
+unsafe impl Send for WalkIgnoreRules {}
+unsafe impl Sync for WalkIgnoreRules {}
+
+impl std::fmt::Debug for WalkIgnoreRules {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("WalkIgnoreRules")
+    }
+}
+
+// In ripgrep builds `WalkIgnoreRules` is never constructed (the `_never`
+// field is uninhabited), so its methods are legitimately dead there.
+#[cfg_attr(not(feature = "zlob"), allow(dead_code))]
+impl WalkIgnoreRules {
+    /// Returns `true` if the provided path is ignored by the collected rule set
+    ///
+    /// `relative_path` has to be relative to the walker's provided base path
+    pub(crate) fn is_ignored(&self, relative_path: &Path) -> bool {
+        #[cfg(feature = "zlob")]
+        {
+            self.inner
+                .rules()
+                .is_some_and(|rules| rules.is_ignored(relative_path))
+        }
+        #[cfg(not(feature = "zlob"))]
+        {
+            let _ = relative_path;
+            match self._never {}
+        }
+    }
+
+    // The old `is_ignored_untrusted` variant was folded away when zlob's
+    // ignore matcher moved to full ancestor enumeration — trailing-slash
+    // sniffing on the input is now sufficient for external queries.
+}
+
+#[cfg(test)]
+mod tests {
+    use super::walk_collect_files;
+    use std::fs;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    // Backend-agnostic parity check: both the zlob and ripgrep walkers must
+    // respect .gitignore, skip hidden files in a git repo, and surface the
+    // expected file set with a correct synced count.
+    #[test]
+    fn collects_files_respecting_gitignore() {
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join(".git")).unwrap();
+        fs::create_dir(root.join("src")).unwrap();
+        fs::create_dir(root.join("target")).unwrap();
+        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();
+        fs::write(root.join("Cargo.toml"), "x").unwrap();
+        fs::write(root.join("debug.log"), "").unwrap();
+        fs::write(root.join("src/main.rs"), "fn main() {}").unwrap();
+        fs::write(root.join("target/out.bin"), "bin").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, true, false, 1, &counter).unwrap();
+
+        let mut names: Vec<String> = out.pairs.into_iter().map(|(_, rel)| rel).collect();
+        names.sort();
+
+        assert!(names.contains(&"Cargo.toml".to_string()));
+        assert!(names.iter().any(|n| n.ends_with("main.rs")));
+        // target/ and *.log are gitignored; .git/ is skipped.
+        assert!(!names.iter().any(|n| n.contains("target")));
+        assert!(!names.iter().any(|n| n.ends_with(".log")));
+        assert!(!names.iter().any(|n| n.contains(".git/")));
+        assert_eq!(counter.load(Ordering::Relaxed), names.len());
+    }
+
+    // Non-git roots prune known non-code directories (node_modules).
+    #[test]
+    fn prunes_non_code_dirs_for_non_git_root() {
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join("node_modules")).unwrap();
+        fs::write(root.join("node_modules/lib.js"), "x").unwrap();
+        fs::write(root.join("index.js"), "x").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, false, false, 1, &counter).unwrap();
+        let names: Vec<String> = out.pairs.into_iter().map(|(_, rel)| rel).collect();
+
+        assert!(names.iter().any(|n| n.ends_with("index.js")));
+        assert!(!names.iter().any(|n| n.contains("node_modules")));
+    }
+
+    // Only the zlob backend surfaces reusable ignore rules; they must match
+    // the same tree the walk respected.
+    #[cfg(feature = "zlob")]
+    #[test]
+    fn surfaces_reusable_ignore_rules() {
+        use std::path::Path;
+
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join(".git")).unwrap();
+        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();
+        fs::write(root.join("Cargo.toml"), "x").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, true, false, 1, &counter).unwrap();
+
+        let rules = out.ignore_rules.expect("zlob surfaces ignore rules");
+        assert!(rules.is_ignored(Path::new("target/")));
+        assert!(rules.is_ignored(Path::new("debug.log")));
+        assert!(!rules.is_ignored(Path::new("Cargo.toml")));
+    }
+}

crates/fff-core/src/walk/ripgrep.rs

@@ -0,0 +1,87 @@
+use crate::ignore::non_git_repo_overrides;
+use crate::types::FileItem;
+use crate::walk::WalkOutput;
+use crate::watch::is_git_file;
+use ignore::WalkBuilder;
+use std::path::Path;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+#[tracing::instrument(skip_all, name = "ripgrep walker", level = "info")]
+pub(crate) fn walk_collect_files(
+    base_path: &Path,
+    is_git_repo: bool,
+    follow_symlinks: bool,
+    threads: usize,
+    synced_files_count: &Arc<AtomicUsize>,
+) -> crate::Result<WalkOutput> {
+    let mut walk_builder = WalkBuilder::new(base_path);
+    walk_builder
+        // this is a very important guard for the user opening ~/ or other root non-git dir
+        .hidden(!is_git_repo)
+        .git_ignore(true)
+        .git_exclude(true)
+        .git_global(true)
+        .ignore(true)
+        .follow_links(follow_symlinks)
+        .threads(threads);
+
+    if !is_git_repo && let Some(overrides) = non_git_repo_overrides(base_path) {
+        walk_builder.overrides(overrides);
+    }
+
+    let walker = walk_builder.build_parallel();
+
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected =
+        parking_lot::Mutex::new((Vec::<(FileItem, String)>::new(), Vec::<String>::new()));
+    walker.run(|| {
+        let collected = &collected;
+        let counter = Arc::clone(synced_files_count);
+        let base_path = base_path.to_path_buf();
+
+        Box::new(move |result| {
+            let Ok(entry) = result else {
+                return ignore::WalkState::Continue;
+            };
+
+            if entry.file_type().is_some_and(|ft| ft.is_file()) {
+                let path = entry.path();
+
+                // Ignore walkers sometimes surface files inside `.git/`
+                // when the base is itself a git repo — skip them.
+                if is_git_file(path) {
+                    return ignore::WalkState::Continue;
+                }
+
+                let metadata = entry.metadata().ok();
+                let (file_item, rel_path) =
+                    FileItem::new_from_walk(path, &base_path, None, metadata.as_ref());
+
+                collected.lock().0.push((file_item, rel_path));
+                counter.fetch_add(1, Ordering::Relaxed);
+            } else if entry.depth() > 0 && entry.file_type().is_some_and(|ft| ft.is_dir()) {
+                let path = entry.path();
+                if !is_git_file(path)
+                    && let Ok(rel) = path.strip_prefix(&base_path)
+                {
+                    let mut rel = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy())
+                        .into_owned();
+                    rel.push('/');
+                    collected.lock().1.push(rel);
+                }
+            }
+            ignore::WalkState::Continue
+        })
+    });
+
+    let (pairs, dirs) = collected.into_inner();
+    Ok(WalkOutput {
+        pairs,
+        dirs,
+        ignore_rules: None,
+    })
+}

crates/fff-core/src/walk/zlob.rs

@@ -0,0 +1,123 @@
+use crate::file_picker::is_known_binary_extension_basename;
+use crate::ignore::IGNORED_DIRS;
+use crate::types::FileItem;
+use crate::walk::{WalkIgnoreRules, WalkOutput};
+use parking_lot::Mutex;
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use zlob::walk::{WalkBuilder, WalkFlags, WalkMetadata, WalkState};
+
+const PROGRESS_STEP: usize = 13;
+
+#[tracing::instrument(skip_all, name = "zlob walker", level = "info")]
+pub(crate) fn walk_collect_files(
+    base_path: &Path,
+    is_git_repo: bool,
+    follow_symlinks: bool,
+    threads: usize,
+    synced_files_count: &Arc<AtomicUsize>,
+) -> crate::Result<WalkOutput> {
+    // gitignore on; skip hidden on non-git roots (so `~/` doesn't recurse into
+    // ~/.cache, ~/.config, etc.); optionally follow symlinks.
+    let mut flags = WalkFlags::GITIGNORE;
+    if !is_git_repo {
+        flags |= WalkFlags::SKIP_HIDDEN;
+    }
+    if follow_symlinks {
+        flags |= WalkFlags::FOLLOW_SYMLINKS;
+    }
+
+    let mut builder = WalkBuilder::new(base_path)
+        .map_err(|e| crate::Error::WalkFailed(format!("WalkBuilder::new: {e:?}")))?;
+    builder
+        .options(flags)
+        .threads(threads)
+        // Bulk-fetch the only metadata FileItem needs; zlob never stats more.
+        .metadata(WalkMetadata::SIZE | WalkMetadata::MTIME);
+
+    if !is_git_repo
+        && !IGNORED_DIRS.is_empty()
+        && let Err(e) = builder.extra_ignore(IGNORED_DIRS)
+    {
+        // Interior NUL in one of the extra_ignore patterns would fail
+        // here — treat as if no extras were supplied rather than
+        // aborting the whole walk.
+        tracing::warn!(?e, "zlob extra_ignore rejected; walking without it");
+    }
+
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected = Mutex::new((Vec::new(), Vec::new()));
+
+    let outcome = match builder.run(|entry| {
+        if !entry.is_file() {
+            // unlike ripgrep walker zlob doesnt show .git files
+            if entry.is_dir() {
+                let rel_bytes = entry.relative_path_bytes();
+                if !rel_bytes.is_empty() {
+                    let mut rel = String::from_utf8_lossy(rel_bytes).into_owned();
+                    rel.push('/');
+                    collected.lock().1.push(rel);
+                }
+            }
+
+            return WalkState::Continue;
+        }
+
+        let rel_bytes = entry.relative_path_bytes();
+
+        // `basename()` returns `&str` for files only.
+        let basename = entry.basename().unwrap_or("");
+        let is_binary = is_known_binary_extension_basename(basename);
+
+        let size = entry.size().unwrap_or(0);
+        // zlob reports mtime in ns since the Unix epoch; FileItem wants secs.
+        let modified = entry
+            .modified_ns()
+            .map(|ns| (ns / 1_000_000_000).max(0) as u64)
+            .unwrap_or(0);
+
+        let basename_offset = entry.basename_offset_in_relative();
+        // zlob emits '/'-separated relative paths, which is fff's canonical
+        // internal form on every platform — store them verbatim.
+        let rel_str = String::from_utf8_lossy(rel_bytes).into_owned();
+        let item = FileItem::new_raw(basename_offset, size, modified, None, is_binary);
+
+        let mut guard = collected.lock();
+        guard.0.push((item, rel_str));
+        let n = guard.0.len();
+        drop(guard);
+
+        if n % PROGRESS_STEP == 0 {
+            synced_files_count.store(n, Ordering::Relaxed);
+        }
+
+        WalkState::Continue
+    }) {
+        Ok(outcome) => outcome,
+        Err(e) => {
+            // Preserve whatever we collected before the failure so the caller
+            // can still surface a partial index instead of nothing.
+            tracing::error!(?e, "zlob walk failed");
+            return Err(crate::Error::WalkFailed(format!("{e:?}")));
+        }
+    };
+
+    let (pairs, dirs) = collected.into_inner();
+    // Always report the exact final total regardless of the last step.
+    synced_files_count.store(pairs.len(), Ordering::Relaxed);
+
+    // Retain the ignore rules only when the walk actually gathered some
+    // (git roots with .gitignore/.ignore). Otherwise callers fall back.
+    let ignore_rules = outcome
+        .rules()
+        .is_some()
+        .then(|| WalkIgnoreRules { inner: outcome });
+
+    Ok(WalkOutput {
+        pairs,
+        dirs,
+        ignore_rules,
+    })
+}

crates/fff-core/src/watcher/mod.rs

@@ -0,0 +1,9 @@
+mod background_watcher;
+pub use background_watcher::*;
+
+mod watch;
+pub use watch::*;
+
+// The harness reads rescan counters, which release builds compile out.
+#[cfg(all(test, rescan_stats))]
+mod rescan_tests;

crates/fff-core/src/watcher/rescan_tests.rs

@@ -0,0 +1,621 @@
+use std::path::PathBuf;
+use std::process::Command;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use notify::Event;
+use notify::EventKind;
+use notify::event::{
+    AccessKind, AccessMode, CreateKind, DataChange, Flag, ModifyKind, RemoveKind, RenameMode,
+};
+use notify_debouncer_full::DebouncedEvent;
+use tempfile::TempDir;
+
+use super::handle_debounced_events;
+use crate::constants::MAX_OVERFLOW_FILES;
+use crate::file_picker::{FFFMode, FilePicker, FilePickerOptions};
+use crate::git_status_worker::GitStatusWorker;
+use crate::rescan_stats::{RescanReason, RescanStats};
+use crate::shared::{SharedFilePicker, SharedFrecency};
+
+#[test]
+fn saving_an_indexed_file_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/main.rs", "fn main() { println!(); }");
+    let delta = f.feed([modify(f.path("src/main.rs"))]);
+
+    f.assert_no_rescan(&delta, "saving a tracked file");
+}
+
+#[test]
+fn editor_atomic_save_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // write-to-temp + rename-over-target, the way vim/VSCode/IntelliJ save.
+    f.write("src/main.rs", "fn main() { println!(); }");
+    let target = f.path("src/main.rs");
+    let temp = f.path("src/.main.rs.swp");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Create(CreateKind::File)).add_path(temp.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::From)))
+                .add_path(temp.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::To)))
+                .add_path(target.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Remove(RemoveKind::File)).add_path(temp),
+            Instant::now(),
+        ),
+    ]);
+
+    f.assert_no_rescan(&delta, "an atomic editor save");
+    assert!(f.is_indexed("src/main.rs"), "target must stay indexed");
+}
+
+#[test]
+fn creating_and_deleting_files_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/added.rs", "pub fn added() {}");
+    let created = f.feed([create(f.path("src/added.rs"))]);
+    f.assert_no_rescan(&created, "creating a file");
+    assert!(f.is_indexed("src/added.rs"));
+
+    f.remove("src/added.rs");
+    let removed = f.feed([remove_file(f.path("src/added.rs"))]);
+    f.assert_no_rescan(&removed, "deleting a file");
+    assert!(!f.is_indexed("src/added.rs"));
+}
+
+#[test]
+fn deleting_a_directory_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.write("src/nested/a.rs", "");
+    f.write("src/nested/b.rs", "");
+    f.index();
+
+    std::fs::remove_dir_all(f.path("src/nested")).unwrap();
+    let delta = f.feed([DebouncedEvent::new(
+        Event::new(EventKind::Remove(RemoveKind::Folder)).add_path(f.path("src/nested")),
+        Instant::now(),
+    )]);
+
+    f.assert_no_rescan(&delta, "deleting a directory");
+    assert!(!f.is_indexed("src/nested/a.rs"));
+    assert!(f.is_indexed("src/main.rs"));
+}
+
+#[test]
+fn read_only_access_events_are_ignored() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // fff's own preview + grep reads generate these; reacting to them would
+    // make the picker rescan whenever the user scrolls the result list.
+    let path = f.path("src/main.rs");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Read)).add_path(path.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Open(AccessMode::Read)))
+                .add_path(path.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Close(AccessMode::Read))).add_path(path),
+            Instant::now(),
+        ),
+    ]);
+
+    f.assert_no_rescan(&delta, "read-only access events");
+}
+
+#[test]
+fn recreating_the_same_paths_does_not_consume_overflow_capacity() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Recreated paths must reuse their overflow slots.
+    for _ in 0..8 {
+        for i in 0..200 {
+            let rel = format!("gen/out{i}.rs");
+            f.write(&rel, "generated");
+            f.feed([create(f.path(&rel))]);
+        }
+        for i in 0..200 {
+            let rel = format!("gen/out{i}.rs");
+            f.remove(&rel);
+            f.feed([remove_file(f.path(&rel))]);
+        }
+    }
+
+    let delta = f.all_rescans();
+    f.assert_no_rescan(&delta, "1600 create/delete cycles over 200 stable paths");
+    assert!(
+        f.overflow_len() <= 200,
+        "each path must claim one overflow slot at most, got {}",
+        f.overflow_len()
+    );
+}
+
+#[test]
+fn writes_inside_a_gitignored_directory_stay_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\nnode_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let mut events = Vec::new();
+    for i in 0..64 {
+        let rel = format!("target/debug/artifact{i}.o");
+        f.write(&rel, "binary");
+        events.push(create(f.path(&rel)));
+    }
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "build output written into an ignored directory");
+}
+
+#[test]
+fn ignored_event_batch_above_index_capacity_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "node_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|i| {
+            let rel = format!("node_modules/pkg/file{i}.js");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "ignored events above the index capacity");
+    assert_eq!(f.overflow_len(), 0);
+}
+
+#[test]
+fn repeated_edits_above_index_capacity_stay_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "repeated edits above the index capacity");
+    assert_eq!(f.overflow_len(), 0);
+}
+
+#[test]
+fn ignore_file_inside_an_ignored_directory_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "node_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let ignore_files =
+        ["left-pad", "lodash", "typescript"].map(|pkg| format!("node_modules/{pkg}/.gitignore"));
+
+    for rel in &ignore_files {
+        f.write(rel, "dist\n");
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| create(f.path(rel))));
+    f.assert_no_rescan(&delta, "creating ignored .gitignore files");
+
+    for rel in &ignore_files {
+        f.write(rel, "build\n");
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| modify(f.path(rel))));
+    f.assert_no_rescan(&delta, "modifying ignored .gitignore files");
+
+    for rel in &ignore_files {
+        f.remove(rel);
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| remove_file(f.path(rel))));
+    f.assert_no_rescan(&delta, "removing ignored .gitignore files");
+}
+
+#[test]
+fn ignore_file_inside_an_indexed_directory_triggers_a_rescan() {
+    let f = Fixture::with_git();
+    f.write("src/.gitignore", ".gitignore\ngenerated/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/.gitignore", ".gitignore\ngenerated/\nbuild/\n");
+    let delta = f.feed([modify(f.path("src/.gitignore"))]);
+
+    assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
+}
+
+#[test]
+fn git_internal_churn_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let git_dir = f.path(".git");
+    let delta = f.feed([
+        create(git_dir.join("index.lock")),
+        modify(git_dir.join("index")),
+        remove_file(git_dir.join("index.lock")),
+        modify(git_dir.join("HEAD")),
+        modify(git_dir.join("logs/HEAD")),
+        modify(git_dir.join("COMMIT_EDITMSG")),
+        modify(git_dir.join("refs/heads/main")),
+    ]);
+
+    f.assert_no_rescan(&delta, "git writing its own metadata");
+}
+
+#[test]
+fn changing_the_root_ignore_file_triggers_a_rescan() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write(".gitignore", "target/\nsrc/\n");
+    let delta = f.feed([modify(f.path(".gitignore"))]);
+
+    assert_eq!(
+        delta.count(RescanReason::IgnoreFileChanged),
+        1,
+        "the indexed set depends on the root ignore rules, got {delta}"
+    );
+}
+
+#[test]
+fn kernel_event_loss_on_a_directory_triggers_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let delta = f.feed([DebouncedEvent::new(
+        Event::new(EventKind::Modify(ModifyKind::Any))
+            .add_path(f.path("src"))
+            .set_flag(Flag::Rescan),
+        Instant::now(),
+    )]);
+
+    assert_eq!(
+        delta.count(RescanReason::KernelEventLoss),
+        1,
+        "a dropped-events flag over a directory means unknown subtree state, got {delta}"
+    );
+}
+
+#[test]
+fn new_files_above_index_capacity_trigger_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|i| {
+            let rel = format!("src/bulk{i}.rs");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    assert_eq!(
+        delta.count(RescanReason::IndexUpdateRejected),
+        1,
+        "new files above the overflow region cannot be applied incrementally, got {delta}"
+    );
+}
+
+#[test]
+fn batch_at_the_overflow_boundary_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES)
+        .map(|i| {
+            let rel = format!("src/bulk{i}.rs");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "a batch exactly at the overflow limit");
+}
+
+#[test]
+fn event_batch_at_four_times_index_capacity_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES * 4)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "an event batch exactly at the event limit");
+}
+
+#[test]
+fn event_batch_above_four_times_index_capacity_triggers_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES * 4 + 1)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    assert_eq!(
+        delta.count(RescanReason::EventBatchOverflow),
+        1,
+        "an event batch above four times the index capacity must rescan, got {delta}"
+    );
+}
+
+#[test]
+fn repeated_triggers_inside_the_cooldown_collapse_to_one_rescan() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Repeated batches during the cooldown must share one walk.
+    for round in 0..50 {
+        f.write(".gitignore", &format!("target/\n# round {round}\n"));
+        f.feed([modify(f.path(".gitignore"))]);
+    }
+
+    let stats = f.all_rescans();
+    assert_eq!(
+        stats.total, 1,
+        "50 triggers inside the cooldown must collapse to a single walk, got {stats}"
+    );
+    assert_eq!(
+        stats.throttled, 49,
+        "every suppressed request must be accounted for, got {stats}"
+    );
+}
+
+#[test]
+fn an_explicit_request_is_never_throttled() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Burn the cooldown with a watcher trigger, then confirm a user-initiated
+    // refresh still goes through.
+    f.write(".gitignore", "target/\nsrc/\n");
+    f.feed([modify(f.path(".gitignore"))]);
+
+    for _ in 0..3 {
+        f.picker.trigger_full_rescan_async(&f.frecency).unwrap();
+    }
+
+    let stats = f.all_rescans();
+    assert_eq!(
+        stats.count(RescanReason::Explicit),
+        3,
+        "explicit refreshes must bypass the throttle, got {stats}"
+    );
+    assert_eq!(stats.count_throttled(RescanReason::Explicit), 0);
+}
+
+#[test]
+fn events_after_a_suppressed_kernel_rescan_are_still_applied() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/added.rs", "pub fn added() {}");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content)))
+                .add_path(f.path("src/main.rs"))
+                .set_flag(Flag::Rescan),
+            Instant::now(),
+        ),
+        create(f.path("src/added.rs")),
+    ]);
+
+    f.assert_no_rescan(&delta, "a dropped-events flag over a single tracked file");
+    assert!(
+        f.is_indexed("src/added.rs"),
+        "suppressing the rescan must not drop the rest of the batch"
+    );
+}
+
+#[test]
+fn a_throttled_ignore_file_event_is_still_applied_incrementally() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Burn the cooldown: deleting .gitignore admits a full rescan.
+    f.remove(".gitignore");
+    let delta = f.feed([remove_file(f.path(".gitignore"))]);
+    assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
+    f.picker.wait_for_indexing_complete(Duration::from_secs(10));
+
+    // Recreating it inside the cooldown throttles the rescan, but the file
+    // itself must re-enter the index via the incremental fallback.
+    f.write(".gitignore", "target/\n__ignored_x/\n");
+    let delta = f.feed([create(f.path(".gitignore"))]);
+    assert_eq!(delta.total, 0, "the rescan must be throttled, got {delta}");
+    assert_eq!(delta.count_throttled(RescanReason::IgnoreFileChanged), 1);
+    assert!(
+        f.is_indexed(".gitignore"),
+        "a throttled ignore-file event must still index the file itself"
+    );
+}
+
+struct Fixture {
+    base: PathBuf,
+    picker: SharedFilePicker,
+    frecency: SharedFrecency,
+    git_workdir: Option<PathBuf>,
+    git_worker: Arc<GitStatusWorker>,
+    // Dropped last so background work started by a triggered rescan still
+    // sees the tree it was asked to walk.
+    _tmp: TempDir,
+}
+
+impl Fixture {
+    fn new() -> Self {
+        Self::build(false)
+    }
+
+    fn with_git() -> Self {
+        Self::build(true)
+    }
+
+    fn build(git: bool) -> Self {
+        let tmp = tempfile::tempdir().unwrap();
+        let base = crate::path_utils::canonicalize(tmp.path()).unwrap();
+        let git_workdir = git.then(|| {
+            let status = Command::new("git")
+                .args(["init", "-b", "main"])
+                .current_dir(&base)
+                .output()
+                .expect("git init");
+            assert!(status.status.success(), "git init failed");
+            base.clone()
+        });
+
+        Self {
+            base,
+            picker: SharedFilePicker::default(),
+            frecency: SharedFrecency::noop(),
+            git_workdir,
+            git_worker: GitStatusWorker::new(),
+            _tmp: tmp,
+        }
+    }
+
+    fn index(&self) {
+        let mut picker = FilePicker::new(FilePickerOptions {
+            base_path: self.base.to_string_lossy().into_owned(),
+            watch: false,
+            ..Default::default()
+        })
+        .unwrap();
+        picker.collect_files().unwrap();
+        self.picker.rebase_watches(&self.base);
+        *self.picker.write().unwrap() = Some(picker);
+    }
+
+    fn feed(&self, events: impl IntoIterator<Item = DebouncedEvent>) -> RescanStats {
+        let before = self.picker.rescan_stats();
+        handle_debounced_events(
+            FFFMode::Neovim,
+            events.into_iter().collect(),
+            &self.base,
+            &self.git_workdir,
+            &self.picker,
+            &self.frecency,
+            &self.git_worker,
+        );
+
+        self.picker.rescan_stats().since(&before)
+    }
+
+    fn assert_no_rescan(&self, delta: &RescanStats, what: &str) {
+        assert_eq!(delta.total, 0, "{what} must not trigger a rescan: {delta}");
+    }
+
+    fn path(&self, rel: &str) -> PathBuf {
+        self.base.join(rel)
+    }
+
+    fn write(&self, rel: &str, contents: &str) {
+        let path = self.path(rel);
+        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+        std::fs::write(path, contents).unwrap();
+    }
+
+    fn remove(&self, rel: &str) {
+        std::fs::remove_file(self.path(rel)).unwrap();
+    }
+
+    fn is_indexed(&self, rel: &str) -> bool {
+        let guard = self.picker.read().unwrap();
+        guard
+            .as_ref()
+            .and_then(|p| p.get_file_by_path(self.path(rel)))
+            .is_some_and(|file| !file.is_deleted())
+    }
+
+    fn all_rescans(&self) -> RescanStats {
+        self.picker.rescan_stats()
+    }
+
+    fn overflow_len(&self) -> usize {
+        let guard = self.picker.read().unwrap();
+        guard
+            .as_ref()
+            .map(|p| p.get_overflow_files().len())
+            .unwrap_or(0)
+    }
+}
+
+impl Drop for Fixture {
+    fn drop(&mut self) {
+        // A test that intentionally triggers a rescan leaves a walk running on
+        // the background pool; let it finish before the tree disappears.
+        self.picker
+            .wait_for_indexing_complete(Duration::from_secs(10));
+    }
+}
+
+fn event(kind: EventKind, path: PathBuf) -> DebouncedEvent {
+    DebouncedEvent::new(Event::new(kind).add_path(path), Instant::now())
+}
+
+fn create(path: PathBuf) -> DebouncedEvent {
+    event(EventKind::Create(CreateKind::File), path)
+}
+
+fn modify(path: PathBuf) -> DebouncedEvent {
+    event(
+        EventKind::Modify(ModifyKind::Data(DataChange::Content)),
+        path,
+    )
+}
+
+fn remove_file(path: PathBuf) -> DebouncedEvent {
+    event(EventKind::Remove(RemoveKind::File), path)
+}

crates/fff-core/tests/bigram_overlay_coherence_test.rs

@@ -13,6 +13,8 @@
 //!    exists but is dead code — never called from the grep path.
 //!    See: grep.rs lines ~1787-1855.
 
+mod overflow_frecency_segfault;
+
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -21,7 +23,9 @@ use tempfile::TempDir;
 
 use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions};
 use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query};
-use fff_search::{FilePickerOptions, PaginationArgs, QueryParser, SharedFrecency, SharedPicker};
+use fff_search::{
+    FilePickerOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency,
+};
 
 /// Stress test: 200 base files, 3 rounds of edits + deletes. New files
 /// are tracked but NOT verified via grep (see Group 3 for that bug).
@@ -82,7 +86,7 @@ fn bigram_overlay_coherence_stress_base_edits_and_deletes() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(name)).is_some(),
+                    picker.handle_create_or_modify(base.join(name)).is_some(),
                     "round {round}: modify({name}) should succeed"
                 );
             }
@@ -146,7 +150,7 @@ fn bigram_overlay_coherence_long_session_incremental_edits() {
             {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
-                picker.on_create_or_modify(base.join(name));
+                picker.handle_create_or_modify(base.join(name));
             }
             latest_tokens[file_idx] = new_token;
         }
@@ -223,7 +227,7 @@ fn bigram_overlay_coherence_resurrect_tombstoned_file() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&target_path).is_some());
+        assert!(picker.handle_create_or_modify(&target_path).is_some());
     }
 
     {
@@ -273,7 +277,7 @@ fn bigram_overlay_coherence_proves_contribution_for_modified_base() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(&target_path);
+        picker.handle_create_or_modify(&target_path);
     }
 
     {
@@ -282,12 +286,6 @@ fn bigram_overlay_coherence_proves_contribution_for_modified_base() {
 
         let with_overlay = grep_count(picker, unique);
         assert_eq!(with_overlay, 1, "overlay should find the new token");
-
-        let without_overlay = grep_without_overlay_count(picker, unique);
-        assert_eq!(
-            without_overlay, 0,
-            "without overlay, bigram should exclude the file (stale bigrams)"
-        );
     }
 
     stop_picker(&shared_picker);
@@ -331,7 +329,7 @@ fn bigram_overlay_coherence_rapid_create_delete_same_base_path() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(&volatile_path);
+            picker.handle_create_or_modify(&volatile_path);
         }
 
         {
@@ -374,7 +372,7 @@ fn bigram_overlay_coherence_rapid_create_delete_same_base_path() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(&volatile_path);
+            picker.handle_create_or_modify(&volatile_path);
         }
     }
 
@@ -432,7 +430,7 @@ fn bigram_overlay_coherence_overflow_files_searchable_via_grep() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&new_path).is_some());
+        assert!(picker.handle_create_or_modify(&new_path).is_some());
     }
 
     // Overflow file is tracked.
@@ -497,15 +495,16 @@ fn bigram_overlay_coherence_mixed_tombstones_and_overflow() {
         write_file_with_token(base, &name, &token);
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join(&name));
+        picker.handle_create_or_modify(base.join(&name));
         new_tokens.push(token);
     }
 
     {
+        // assert
         let guard = shared_picker.read().unwrap();
         let picker = guard.as_ref().unwrap();
 
-        // Tombstones work: deleted tokens are gone.
+        // deleted tokens are gon
         for token in &deleted_tokens {
             assert_eq!(
                 grep_count(picker, token),
@@ -517,16 +516,16 @@ fn bigram_overlay_coherence_mixed_tombstones_and_overflow() {
         // Surviving base files still findable.
         for (name, token) in &repo_files[15..] {
             assert!(
-                grep_count(picker, token) >= 1,
+                grep_count(picker, token) == 1,
                 "surviving {name} token should be findable"
             );
         }
 
-        // BUG: Overflow files not findable via grep.
+        // New tokens needs to be findable
         for token in &new_tokens {
             assert!(
-                grep_count(picker, token) >= 1,
-                "BUG: overflow token {token} should be findable but bigram skips overflow"
+                grep_count(picker, token) == 1,
+                "BUG: overflow token {token} should be findable"
             );
         }
     }
@@ -570,7 +569,7 @@ fn bigram_overlay_coherence_full_stress_loop_with_overflow() {
             write_file_with_token(base, name, &new_token);
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
             dead_tokens.push(old_token.clone());
             *old_token = new_token;
         }
@@ -582,7 +581,7 @@ fn bigram_overlay_coherence_full_stress_loop_with_overflow() {
             write_file_with_token(base, &name, &token);
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(&name));
+            picker.handle_create_or_modify(base.join(&name));
             live_overflow.push((name, token));
         }
 
@@ -645,7 +644,7 @@ fn bigram_overlay_coherence_overflow_file_edit_and_delete() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(&path);
+            picker.handle_create_or_modify(&path);
         }
         overflow_files.push((path, token));
     }
@@ -670,7 +669,7 @@ fn bigram_overlay_coherence_overflow_file_edit_and_delete() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(path);
+            picker.handle_create_or_modify(path);
         }
         edited_tokens.push(new_token);
     }
@@ -686,14 +685,19 @@ fn bigram_overlay_coherence_overflow_file_edit_and_delete() {
         }
     }
 
-    // Verify 5 overflow remain.
+    // Verify 5 overflow remain live (tombstones still occupy slots by design —
+    // StableVec never shifts, so get_overflow_files().len() stays at 10).
     {
         let guard = shared_picker.read().unwrap();
         let picker = guard.as_ref().unwrap();
+        let live = picker
+            .get_overflow_files()
+            .iter()
+            .filter(|f| !f.is_deleted())
+            .count();
         assert_eq!(
-            picker.get_overflow_files().len(),
-            5,
-            "should have 5 overflow files after deleting 5"
+            live, 5,
+            "should have 5 live overflow files after deleting 5"
         );
     }
 
@@ -734,7 +738,7 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         edited_tokens.push(token);
     }
@@ -747,25 +751,23 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(&name));
+            picker.handle_create_or_modify(base.join(&name));
         }
         new_tokens.push(token);
     }
 
     // Phase 2: Commit and rescan.
     git_add_and_commit(base, "batch edit");
-
-    {
-        let mut guard = shared_picker.write().unwrap();
-        let picker = guard.as_mut().unwrap();
-        picker
-            .trigger_rescan(&shared_frecency)
-            .expect("trigger_rescan should succeed");
-    }
+    shared_picker
+        .trigger_full_rescan_async(&shared_frecency)
+        .expect("rescan should succeed");
 
     // After trigger_rescan, sync_data is replaced (and bigram_index dropped
     // with it). Wait for the synchronous scan to finish.
-    wait_for_scan(&shared_picker);
+    assert!(
+        shared_picker.wait_for_scan(Duration::from_secs(15)),
+        "Timed out waiting for scan to complete"
+    );
 
     // Verify the file list is refreshed: all base_count + 5 files should
     // be present as base files (not overflow, since they're committed).
@@ -800,14 +802,6 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
                 with >= 1,
                 "post-rescan: edited token {token} should be findable"
             );
-
-            // The content is now in the base index, so it should be
-            // findable even without the overlay.
-            let without = grep_without_overlay_count(picker, token);
-            assert!(
-                without >= 1,
-                "post-rescan: {token} should be in base index (without overlay: {without})"
-            );
         }
 
         for token in &new_tokens {
@@ -850,7 +844,7 @@ fn bigram_overlay_coherence_full_lifecycle_seed_edit_commit_rescan_edit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         phase1_tokens.push(token);
     }
@@ -890,18 +884,14 @@ fn bigram_overlay_coherence_full_lifecycle_seed_edit_commit_rescan_edit() {
 
     // -- Phase 2: Commit and rescan --
     git_add_and_commit(base, "phase 1 changes");
+    shared_picker
+        .trigger_full_rescan_async(&shared_frecency)
+        .expect("rescan should succeed");
 
-    {
-        let mut guard = shared_picker.write().unwrap();
-        let picker = guard.as_mut().unwrap();
-        picker
-            .trigger_rescan(&shared_frecency)
-            .expect("rescan should succeed");
-    }
-
-    // After rescan, bigram is dropped with old FileSync. Grep falls back
-    // to full search, which is correct.
-    wait_for_scan(&shared_picker);
+    assert!(
+        shared_picker.wait_for_scan(Duration::from_secs(15)),
+        "Timed out waiting for scan to complete"
+    );
 
     // Phase1 tokens should still be findable (now in base index).
     {
@@ -933,7 +923,7 @@ fn bigram_overlay_coherence_full_lifecycle_seed_edit_commit_rescan_edit() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         phase3_tokens.push(token);
     }
@@ -1012,7 +1002,7 @@ fn bigram_overlay_coherence_nested_directory_edits() {
         {
             let mut guard = shared_picker.write().unwrap();
             let picker = guard.as_mut().unwrap();
-            picker.on_create_or_modify(base.join(name));
+            picker.handle_create_or_modify(base.join(name));
         }
         edited.push(token);
     }
@@ -1130,7 +1120,7 @@ fn bigram_overlay_coherence_fuzzy_search_base_overflow_and_deleted() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join("controller_admin.rs"));
+        picker.handle_create_or_modify(base.join("controller_admin.rs"));
     }
 
     // Fuzzy search should find the new overflow file.
@@ -1177,7 +1167,7 @@ fn bigram_overlay_coherence_fuzzy_search_after_rescan() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join("router_grpc.rs"));
+        picker.handle_create_or_modify(base.join("router_grpc.rs"));
     }
 
     let web_path = base.join("router_web.rs");
@@ -1191,14 +1181,13 @@ fn bigram_overlay_coherence_fuzzy_search_after_rescan() {
     // Commit and rescan.
     git_add_and_commit(base, "add grpc, remove web");
 
-    {
-        let mut guard = shared_picker.write().unwrap();
-        let picker = guard.as_mut().unwrap();
-        picker
-            .trigger_rescan(&shared_frecency)
-            .expect("rescan should succeed");
-    }
-    wait_for_scan(&shared_picker);
+    shared_picker
+        .trigger_full_rescan_async(&shared_frecency)
+        .expect("rescan should succeed");
+    assert!(
+        shared_picker.wait_for_scan(Duration::from_secs(15)),
+        "Timed out waiting for scan to complete"
+    );
 
     // After rescan, fuzzy search should reflect the committed state.
     {
@@ -1243,7 +1232,7 @@ fn bigram_overlay_coherence_fuzzy_and_grep_combined() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join(edit_name));
+        picker.handle_create_or_modify(base.join(edit_name));
     }
 
     // Add an overflow file with a distinctive name.
@@ -1255,7 +1244,7 @@ fn bigram_overlay_coherence_fuzzy_and_grep_combined() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        picker.on_create_or_modify(base.join("unique_overflow_widget.rs"));
+        picker.handle_create_or_modify(base.join("unique_overflow_widget.rs"));
     }
 
     // Delete a base file.
@@ -1298,15 +1287,17 @@ fn bigram_overlay_coherence_fuzzy_and_grep_combined() {
 
         // Fuzzy: deleted file not in results.
         let fuzzy_del = fuzzy_search_paths(picker, del_name);
+        let del_file = Path::new(del_name).file_name().unwrap().to_str().unwrap();
         assert!(
-            !fuzzy_del.iter().any(|p| p.contains(del_name)),
+            !fuzzy_del.iter().any(|p| p.contains(del_file)),
             "fuzzy should not find deleted file"
         );
 
         // Fuzzy: edited file still findable by name.
         let fuzzy_edit = fuzzy_search_paths(picker, edit_name);
+        let edit_file = Path::new(edit_name).file_name().unwrap().to_str().unwrap();
         assert!(
-            fuzzy_edit.iter().any(|p| p.contains(edit_name)),
+            fuzzy_edit.iter().any(|p| p.contains(edit_file)),
             "fuzzy should find edited file by name"
         );
     }
@@ -1336,37 +1327,10 @@ fn grep_count(picker: &FilePicker, query: &str) -> usize {
     picker.grep(&parsed, &grep_opts()).matches.len()
 }
 
-fn grep_without_overlay_count(picker: &FilePicker, query: &str) -> usize {
-    let parsed = parse_grep_query(query);
-    picker
-        .grep_without_overlay(&parsed, &grep_opts())
-        .matches
-        .len()
-}
-
 /// Wait for scanning to finish (no bigram requirement).
 /// Use after `trigger_rescan` which replaces sync_data but does not
 /// rebuild the bigram index.
-fn wait_for_scan(shared_picker: &SharedPicker) {
-    let deadline = std::time::Instant::now() + Duration::from_secs(15);
-    loop {
-        std::thread::sleep(Duration::from_millis(50));
-        let ready = shared_picker
-            .read()
-            .ok()
-            .map(|guard| guard.as_ref().map_or(false, |p| !p.is_scan_active()))
-            .unwrap_or(false);
-        if ready {
-            break;
-        }
-        assert!(
-            std::time::Instant::now() < deadline,
-            "Timed out waiting for scan to complete"
-        );
-    }
-}
-
-fn wait_for_bigram(shared_picker: &SharedPicker) {
+fn wait_for_bigram(shared_picker: &SharedFilePicker) {
     let deadline = std::time::Instant::now() + Duration::from_secs(10);
     loop {
         std::thread::sleep(Duration::from_millis(50));
@@ -1389,7 +1353,7 @@ fn wait_for_bigram(shared_picker: &SharedPicker) {
     }
 }
 
-fn stop_picker(shared_picker: &SharedPicker) {
+fn stop_picker(shared_picker: &SharedFilePicker) {
     if let Ok(mut guard) = shared_picker.write() {
         if let Some(ref mut picker) = *guard {
             picker.stop_background_monitor();
@@ -1426,8 +1390,8 @@ fn git_add_and_commit(dir: &Path, msg: &str) {
     git_run(dir, &["commit", "-m", msg]);
 }
 
-fn make_picker(base: &Path) -> (SharedPicker, SharedFrecency) {
-    let shared_picker = SharedPicker::default();
+fn make_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) {
+    let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
 
     FilePicker::new_with_shared_state(
@@ -1659,7 +1623,7 @@ fn bigram_overlay_coherence_fuzzy_grep_finds_overflow_files() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&new_path).is_some());
+        assert!(picker.handle_create_or_modify(&new_path).is_some());
         assert_eq!(picker.get_overflow_files().len(), 1);
     }
 

crates/fff-core/tests/bigram_overlay_integration.rs

@@ -7,7 +7,7 @@ use tempfile::TempDir;
 
 use fff_search::file_picker::{FFFMode, FilePicker};
 use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query};
-use fff_search::{FilePickerOptions, SharedFrecency, SharedPicker};
+use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency};
 
 /// Create a temp directory with some initial files, run the full picker lifecycle,
 /// then modify a file and verify grep finds the new content.
@@ -25,7 +25,7 @@ fn modified_file_findable_via_overlay() {
     .unwrap();
     fs::write(base.join("gamma.txt"), "yet another file\nmore lines\n").unwrap();
 
-    let shared_picker = SharedPicker::default();
+    let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
 
     FilePicker::new_with_shared_state(
@@ -36,6 +36,7 @@ fn modified_file_findable_via_overlay() {
             enable_mmap_cache: true,
             enable_content_indexing: true,
             mode: FFFMode::Neovim,
+            watch: false, // we drive events manually
             ..Default::default()
         },
     )
@@ -112,7 +113,7 @@ fn modified_file_findable_via_overlay() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        let result = picker.on_create_or_modify(&modified_path);
+        let result = picker.handle_create_or_modify(&modified_path);
         assert!(
             result.is_some(),
             "on_create_or_modify should return the file"
@@ -143,21 +144,6 @@ fn modified_file_findable_via_overlay() {
         );
     }
 
-    // Prove the overlay is actually doing something: without it, the bigram
-    // index would filter out beta.txt and the search would miss the needle.
-    {
-        let guard = shared_picker.read().unwrap();
-        let picker = guard.as_ref().unwrap();
-        let parsed = parse_grep_query("UNIQUE_NEEDLE");
-        let opts = grep_opts();
-        let result = picker.grep_without_overlay(&parsed, &opts);
-        assert_eq!(
-            result.matches.len(),
-            0,
-            "Without overlay, bigram prefiltering should exclude the modified file"
-        );
-    }
-
     // Cleanup: stop background watcher.
     if let Ok(mut guard) = shared_picker.write() {
         if let Some(ref mut picker) = *guard {
@@ -175,7 +161,7 @@ fn deleted_file_excluded_via_overlay() {
     fs::write(base.join("keep.txt"), "keep this content\n").unwrap();
     fs::write(base.join("remove.txt"), "DELETEME_TOKEN is here\n").unwrap();
 
-    let shared_picker = SharedPicker::default();
+    let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
 
     FilePicker::new_with_shared_state(
@@ -186,6 +172,7 @@ fn deleted_file_excluded_via_overlay() {
             enable_mmap_cache: true,
             enable_content_indexing: true,
             mode: FFFMode::Neovim,
+            watch: false, // we drive events manually
             ..Default::default()
         },
     )
@@ -244,7 +231,7 @@ fn new_file_findable_after_add() {
 
     fs::write(base.join("existing.txt"), "original content\n").unwrap();
 
-    let shared_picker = SharedPicker::default();
+    let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
 
     FilePicker::new_with_shared_state(
@@ -255,6 +242,7 @@ fn new_file_findable_after_add() {
             enable_mmap_cache: true,
             enable_content_indexing: true,
             mode: FFFMode::Neovim,
+            watch: false, // we drive events manually
             ..Default::default()
         },
     )
@@ -270,7 +258,7 @@ fn new_file_findable_after_add() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        let result = picker.on_create_or_modify(&new_path);
+        let result = picker.handle_create_or_modify(&new_path);
         assert!(
             result.is_some(),
             "on_create_or_modify should return the new file"
@@ -314,7 +302,7 @@ fn modified_file_findable_via_regex_overlay() {
     )
     .unwrap();
 
-    let shared_picker = SharedPicker::default();
+    let shared_picker = SharedFilePicker::default();
     let shared_frecency = SharedFrecency::default();
 
     FilePicker::new_with_shared_state(
@@ -325,6 +313,7 @@ fn modified_file_findable_via_regex_overlay() {
             enable_mmap_cache: true,
             enable_content_indexing: true,
             mode: FFFMode::Neovim,
+            watch: false, // we drive events manually
             ..Default::default()
         },
     )
@@ -346,7 +335,7 @@ fn modified_file_findable_via_regex_overlay() {
     {
         let mut guard = shared_picker.write().unwrap();
         let picker = guard.as_mut().unwrap();
-        assert!(picker.on_create_or_modify(&modified_path).is_some());
+        assert!(picker.handle_create_or_modify(&modified_path).is_some());
     }
 
     // Regex grep should find the modified file through the overlay.
@@ -397,7 +386,7 @@ fn grep_for<'a>(picker: &'a FilePicker, query: &str) -> fff_search::grep::GrepRe
     picker.grep(&parsed, &grep_opts())
 }
 
-fn wait_for_bigram(shared_picker: &SharedPicker) {
+fn wait_for_bigram(shared_picker: &SharedFilePicker) {
     let deadline = std::time::Instant::now() + Duration::from_secs(30);
     loop {
         std::thread::sleep(Duration::from_millis(50));

crates/fff-core/tests/dir_index_consistency_test.rs

@@ -0,0 +1,301 @@
+use fff_search::file_picker::{FFFMode, FilePicker};
+use fff_search::{
+    DirSearchConfig, FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser,
+    SharedFilePicker, SharedFrecency,
+};
+use std::fs;
+use std::path::Path;
+use std::time::{Duration, Instant};
+use tempfile::TempDir;
+
+fn make_watched_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) {
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::noop();
+
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().into_owned(),
+            enable_mmap_cache: false,
+            enable_content_indexing: false,
+            mode: FFFMode::Neovim,
+            watch: true,
+            ..Default::default()
+        },
+    )
+    .expect("FilePicker::new_with_shared_state");
+
+    assert!(
+        shared_picker.wait_for_scan(Duration::from_secs(30)),
+        "initial scan did not complete"
+    );
+    assert!(
+        shared_picker.wait_for_watcher(Duration::from_secs(30)),
+        "watcher did not install"
+    );
+    // macOS FSEvents streams need a beat before they deliver reliably
+    std::thread::sleep(Duration::from_millis(300));
+
+    (shared_picker, shared_frecency)
+}
+
+fn search_dirs(picker: &SharedFilePicker, query: &str) -> Vec<String> {
+    let guard = picker.read().expect("picker read lock");
+    let p = guard.as_ref().expect("picker initialized");
+    let parser = QueryParser::new(DirSearchConfig);
+    let parsed = parser.parse(query);
+    let results = p.fuzzy_search_directories(
+        &parsed,
+        FuzzySearchOptions {
+            pagination: PaginationArgs {
+                offset: 0,
+                limit: 100,
+            },
+            ..Default::default()
+        },
+    );
+    results.items.iter().map(|d| d.relative_path(p)).collect()
+}
+
+fn wait_until<F: Fn() -> bool>(cond: F, timeout: Duration) -> bool {
+    let deadline = Instant::now() + timeout;
+    while Instant::now() < deadline {
+        if cond() {
+            return true;
+        }
+        std::thread::sleep(Duration::from_millis(50));
+    }
+    cond()
+}
+
+#[test]
+fn removed_directory_disappears_from_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("doomed/nested")).unwrap();
+    fs::write(base.join("doomed/a.rs"), "x").unwrap();
+    fs::write(base.join("doomed/nested/b.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(
+        search_dirs(&picker, "doomed")
+            .iter()
+            .any(|d| d.starts_with("doomed")),
+        "sanity: dir indexed after scan"
+    );
+
+    fs::remove_dir_all(base.join("doomed")).unwrap();
+
+    assert!(
+        wait_until(
+            || !search_dirs(&picker, "doomed")
+                .iter()
+                .any(|d| d.starts_with("doomed")),
+            Duration::from_secs(10)
+        ),
+        "removed dir must disappear from dir search, got: {:?}",
+        search_dirs(&picker, "doomed")
+    );
+}
+
+#[test]
+fn moved_out_directory_disappears_from_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let trash = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("doomed/nested")).unwrap();
+    fs::write(base.join("doomed/a.rs"), "x").unwrap();
+    fs::write(base.join("doomed/nested/b.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(
+        search_dirs(&picker, "doomed")
+            .iter()
+            .any(|d| d.starts_with("doomed")),
+        "sanity: dir indexed after scan"
+    );
+
+    fs::rename(base.join("doomed"), trash.path().join("doomed")).unwrap();
+
+    assert!(
+        wait_until(
+            || !search_dirs(&picker, "doomed")
+                .iter()
+                .any(|d| d.starts_with("doomed")),
+            Duration::from_secs(10)
+        ),
+        "moved-out dir must disappear from dir search, got: {:?}",
+        search_dirs(&picker, "doomed")
+    );
+}
+
+#[test]
+fn moved_in_directory_appears_in_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let staging = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let incoming = staging.path().join("arrived");
+    fs::create_dir_all(incoming.join("nested")).unwrap();
+    fs::write(incoming.join("a.rs"), "x").unwrap();
+    fs::write(incoming.join("nested/b.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(search_dirs(&picker, "arrived").is_empty(), "sanity");
+
+    fs::rename(&incoming, base.join("arrived")).unwrap();
+
+    assert!(
+        wait_until(
+            || {
+                let dirs = search_dirs(&picker, "arrived");
+                dirs.iter().any(|d| d.starts_with("arrived"))
+            },
+            Duration::from_secs(10)
+        ),
+        "moved-in dir must appear in dir search, got: {:?}",
+        search_dirs(&picker, "arrived")
+    );
+}
+
+#[test]
+fn new_file_in_new_directory_surfaces_the_dir() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+    assert!(search_dirs(&picker, "brandnew").is_empty(), "sanity");
+
+    fs::create_dir_all(base.join("brandnew")).unwrap();
+    fs::write(base.join("brandnew/file.rs"), "x").unwrap();
+
+    assert!(
+        wait_until(
+            || search_dirs(&picker, "brandnew")
+                .iter()
+                .any(|d| d.starts_with("brandnew")),
+            Duration::from_secs(10)
+        ),
+        "new dir must appear in dir search, got: {:?}",
+        search_dirs(&picker, "brandnew")
+    );
+}
+
+#[test]
+fn deleting_last_file_keeps_directory_visible() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("lonely")).unwrap();
+    fs::write(base.join("lonely/only.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+
+    // the file goes away but the directory itself still exists on disk
+    fs::remove_file(base.join("lonely/only.rs")).unwrap();
+
+    assert!(
+        wait_until(
+            || {
+                let guard = picker.read().unwrap();
+                let p = guard.as_ref().unwrap();
+                p.get_file_by_path(base.join("lonely/only.rs"))
+                    .is_none_or(|f| f.is_deleted())
+            },
+            Duration::from_secs(10)
+        ),
+        "file removal must be applied"
+    );
+    assert!(
+        search_dirs(&picker, "lonely")
+            .iter()
+            .any(|d| d.starts_with("lonely")),
+        "dir still exists on disk and must stay searchable"
+    );
+}
+
+#[test]
+fn recreated_directory_reappears_in_dir_search() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("phoenix")).unwrap();
+    fs::write(base.join("phoenix/a.rs"), "x").unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+
+    fs::remove_dir_all(base.join("phoenix")).unwrap();
+    assert!(
+        wait_until(
+            || !search_dirs(&picker, "phoenix")
+                .iter()
+                .any(|d| d.starts_with("phoenix")),
+            Duration::from_secs(10)
+        ),
+        "dir must disappear after removal"
+    );
+
+    fs::create_dir_all(base.join("phoenix")).unwrap();
+    fs::write(base.join("phoenix/a.rs"), "x").unwrap();
+
+    assert!(
+        wait_until(
+            || search_dirs(&picker, "phoenix")
+                .iter()
+                .any(|d| d.starts_with("phoenix")),
+            Duration::from_secs(10)
+        ),
+        "recreated dir must reappear in dir search, got: {:?}",
+        search_dirs(&picker, "phoenix")
+    );
+}
+
+/// Regression for #725: a dir that is EMPTY at scan time must be indexed —
+/// searchable in dir search and watched so later file creations are seen.
+#[test]
+fn empty_directory_at_scan_is_searchable_and_watched() {
+    let tmp = TempDir::new().unwrap();
+    let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
+    fs::create_dir_all(base.join("commands")).unwrap();
+    fs::write(base.join("keep.rs"), "x").unwrap();
+
+    let (picker, _frecency) = make_watched_picker(&base);
+
+    assert!(
+        search_dirs(&picker, "commands")
+            .iter()
+            .any(|d| d.starts_with("commands")),
+        "empty dir must be searchable right after the scan, got: {:?}",
+        search_dirs(&picker, "commands")
+    );
+
+    // The empty dir must reuse its scan-built DirItem when a file lands in it
+    // and the watcher must have registered a watch on it (the #725 repro).
+    fs::write(base.join("commands/review.md"), "# review").unwrap();
+    assert!(
+        wait_until(
+            || {
+                let guard = picker.read().unwrap();
+                let p = guard.as_ref().unwrap();
+                p.get_file_by_path(base.join("commands/review.md"))
+                    .is_some()
+            },
+            Duration::from_secs(10)
+        ),
+        "file created in a scan-time-empty dir must be indexed"
+    );
+
+    let guard = picker.read().unwrap();
+    let p = guard.as_ref().unwrap();
+    let commands_dirs = p
+        .get_dirs()
+        .iter()
+        .filter(|d| d.relative_path(p).starts_with("commands"))
+        .count();
+    assert_eq!(commands_dirs, 1, "no duplicate DirItem for the empty dir");
+}

crates/fff-core/tests/drop_during_post_scan.rs

@@ -0,0 +1,161 @@
+// Regression pinning: dropping a picker during poset scan off-lock time
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+use tempfile::TempDir;
+
+use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions};
+use fff_search::{FilePickerOptions, QueryParser, SharedFilePicker, SharedFrecency};
+
+fn seed_files(dir: &Path, count: usize) {
+    for i in 0..count {
+        let subdir = dir.join(format!("dir_{}", i / 20));
+        fs::create_dir_all(&subdir).unwrap();
+        fs::write(
+            subdir.join(format!("file_{i}.rs")),
+            format!("pub fn func_{i}() {{ /* token_{i} */ }}\n"),
+        )
+        .unwrap();
+    }
+}
+
+fn git_init(dir: &Path) {
+    let run = |args: &[&str]| {
+        Command::new("git")
+            .args(args)
+            .current_dir(dir)
+            .env("GIT_AUTHOR_NAME", "test")
+            .env("GIT_AUTHOR_EMAIL", "t@t")
+            .env("GIT_COMMITTER_NAME", "test")
+            .env("GIT_COMMITTER_EMAIL", "t@t")
+            .output()
+            .unwrap();
+    };
+    run(&["init"]);
+    run(&["add", "-A"]);
+    run(&["commit", "-m", "init"]);
+}
+
+fn make_picker(base: &Path) -> (SharedFilePicker, SharedFrecency) {
+    let sp = SharedFilePicker::default();
+    let sf = SharedFrecency::default();
+    FilePicker::new_with_shared_state(
+        sp.clone(),
+        sf.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: true,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("init");
+    (sp, sf)
+}
+
+/// Drop picker immediately after scan starts — scan thread will find
+/// the picker gone and exit cleanly.
+#[test]
+fn drop_picker_during_walk_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 500);
+    git_init(tmp.path());
+
+    let (sp, _sf) = make_picker(tmp.path());
+    // Don't wait — drop immediately while walk is likely in progress
+    drop(sp);
+
+    // If we get here without SIGSEGV, the test passes.
+    std::thread::sleep(Duration::from_millis(200));
+}
+
+/// Drop picker while post-scan indexing is running. The snapshot holds
+/// Arc clones that keep the buffers alive.
+#[test]
+fn drop_picker_during_post_scan_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 500);
+    git_init(tmp.path());
+
+    let (sp, _sf) = make_picker(tmp.path());
+
+    // Wait for walk to finish (files are searchable) but post-scan is
+    // still running (bigram not yet built).
+    sp.wait_for_scan(Duration::from_secs(10));
+
+    // At this point post_scan_indexing_active is likely true.
+    // Drop the picker — this releases the picker's Arc clones, but the
+    // post-scan snapshot's clones keep the buffers alive.
+    if let Ok(mut guard) = sp.write() {
+        guard.take(); // drop the FilePicker
+    }
+
+    // Give post-scan threads time to run against the "dead" picker.
+    // They must not segfault.
+    std::thread::sleep(Duration::from_secs(2));
+}
+
+/// Drop picker from a second thread while the first thread is doing
+/// fuzzy searches. Verifies no segfault from interleaved access.
+#[test]
+fn drop_picker_concurrent_with_search_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 500);
+    git_init(tmp.path());
+
+    let (sp, _sf) = make_picker(tmp.path());
+    sp.wait_for_scan(Duration::from_secs(10));
+
+    let sp_clone = sp.clone();
+    let running = Arc::new(AtomicBool::new(true));
+    let running_clone = running.clone();
+
+    // Searcher thread: continuously queries while the picker lives
+    let searcher = std::thread::spawn(move || {
+        let parser = QueryParser::default();
+        while running_clone.load(Ordering::Relaxed) {
+            if let Ok(guard) = sp_clone.read() {
+                if let Some(picker) = guard.as_ref() {
+                    let query = parser.parse("func");
+                    let _ = picker.fuzzy_search(&query, None, FuzzySearchOptions::default());
+                }
+            }
+            std::thread::sleep(Duration::from_millis(1));
+        }
+    });
+
+    // Let searches run for a bit, then drop
+    std::thread::sleep(Duration::from_millis(100));
+    if let Ok(mut guard) = sp.write() {
+        guard.take();
+    }
+    std::thread::sleep(Duration::from_millis(100));
+
+    running.store(false, Ordering::Relaxed);
+    searcher.join().unwrap();
+}
+
+/// Repeated init + wait + clean-drop cycle. This is the pattern that
+/// SIGSEGV'd on the pre-refactor code in the benchmark.
+#[test]
+fn repeated_init_and_drop_no_segfault() {
+    let tmp = TempDir::new().unwrap();
+    seed_files(tmp.path(), 200);
+    git_init(tmp.path());
+
+    for _ in 0..5 {
+        let (sp, _sf) = make_picker(tmp.path());
+        sp.wait_for_scan(Duration::from_secs(10));
+        sp.wait_for_indexing_complete(Duration::from_secs(30));
+        if let Ok(mut guard) = sp.write()
+            && let Some(mut picker) = guard.take()
+        {
+            picker.stop_background_monitor();
+        }
+    }
+}

crates/fff-core/tests/fuzz_file_operations.rs

@@ -1,14 +1,3 @@
-//! Randomized file-system mutation stress test.
-//!
-//! Seeds a directory with ~40 files across diverse content domains, builds the
-//! picker + bigram index, then runs 20 rounds of randomized create / edit /
-//! delete / rename / read-only operations. After every round the test verifies
-//! that plain-text grep, regex grep, and fuzzy file search all return correct
-//! results for every live and dead file.
-//!
-//! Uses a seeded RNG (`SmallRng::seed_from_u64`) for deterministic
-//! reproduction.
-
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -20,7 +9,9 @@ use rand::{RngCore, SeedableRng};
 
 use fff_search::file_picker::{FFFMode, FilePicker, FuzzySearchOptions};
 use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query};
-use fff_search::{FilePickerOptions, PaginationArgs, QueryParser, SharedFrecency, SharedPicker};
+use fff_search::{
+    FilePickerOptions, PaginationArgs, QueryParser, SharedFilePicker, SharedFrecency,
+};
 
 const DOMAINS: &[&str] = &[
     r#"
@@ -308,7 +299,7 @@ fn fuzz_file_operations_stress() {
     git_init_and_commit(base);
     t_git += t0.elapsed();
 
-    let shared_picker = SharedPicker::default();
+    let shared_picker = SharedFilePicker::default();
 
     FilePicker::new_with_shared_state(
         shared_picker.clone(),
@@ -377,7 +368,7 @@ fn fuzz_file_operations_stress() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(name)).is_some(),
+                    picker.handle_create_or_modify(base.join(name)).is_some(),
                     "round {round}: on_create_or_modify({name}) should succeed for edit"
                 );
             }
@@ -397,7 +388,7 @@ fn fuzz_file_operations_stress() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(&name)).is_some(),
+                    picker.handle_create_or_modify(base.join(&name)).is_some(),
                     "round {round}: on_create_or_modify({name}) should succeed for create"
                 );
             }
@@ -451,7 +442,9 @@ fn fuzz_file_operations_stress() {
                 let mut guard = shared_picker.write().unwrap();
                 let picker = guard.as_mut().unwrap();
                 assert!(
-                    picker.on_create_or_modify(base.join(&new_name)).is_some(),
+                    picker
+                        .handle_create_or_modify(base.join(&new_name))
+                        .is_some(),
                     "round {round}: on_create_or_modify({new_name}) should succeed for rename"
                 );
             }
@@ -711,7 +704,7 @@ fn fuzzy_search_paths(picker: &FilePicker, query: &str) -> Vec<String> {
         .collect()
 }
 
-fn wait_for_bigram(shared_picker: &SharedPicker) {
+fn wait_for_bigram(shared_picker: &SharedFilePicker) {
     let deadline = std::time::Instant::now() + Duration::from_secs(10);
     loop {
         std::thread::sleep(Duration::from_millis(50));
@@ -764,3 +757,98 @@ fn git_init_and_commit(dir: &Path) {
     git_run(dir, &["add", "-A"]);
     git_run(dir, &["commit", "-m", "initial"]);
 }
+
+/// Proves that dropping the picker while post-scan (warmup + bigram build)
+/// is actively iterating raw pointers does NOT segfault. The Drop impl
+/// sets `cancelled`, waits for `post_scan_indexing_active` to clear, and
+/// only then frees the backing Vec.
+///
+/// Runs 10 iterations to exercise the race window reliably.
+#[test]
+fn drop_during_post_scan_does_not_crash() {
+    let mut caught_active = 0u32;
+
+    for round in 0..10 {
+        let tmp = TempDir::new().unwrap();
+        let base = tmp.path();
+
+        // Create enough files so bigram build takes measurable time
+        for i in 0..2000 {
+            let dir = base.join(format!("d_{:02}", i % 20));
+            fs::create_dir_all(&dir).unwrap();
+            let content = format!(
+                "fn func_{i}() {{ let x = {i}; println!(\"{{x}}\"); }}\n\
+                 const T_{i}: &str = \"TOKEN_{i}\";\n"
+            );
+            fs::write(dir.join(format!("f_{i:04}.rs")), content).unwrap();
+        }
+
+        git_init_and_commit(base);
+
+        let shared_picker = SharedFilePicker::default();
+
+        FilePicker::new_with_shared_state(
+            shared_picker.clone(),
+            SharedFrecency::noop(),
+            FilePickerOptions {
+                base_path: base.to_string_lossy().to_string(),
+                enable_mmap_cache: true,
+                enable_content_indexing: true,
+                watch: false,
+                mode: FFFMode::Neovim,
+                ..Default::default()
+            },
+        )
+        .unwrap();
+
+        // Wait for scan but NOT for bigram — drop while post-scan is active
+        shared_picker.wait_for_scan(Duration::from_secs(10));
+
+        // Poll until post_scan_indexing_active is true (bigram started)
+        let deadline = std::time::Instant::now() + Duration::from_secs(5);
+        let mut was_active = false;
+        loop {
+            if let Ok(guard) = shared_picker.read() {
+                if let Some(picker) = guard.as_ref() {
+                    if picker.is_post_scan_active() {
+                        was_active = true;
+                        break;
+                    }
+                }
+            }
+            if std::time::Instant::now() > deadline {
+                break;
+            }
+            std::thread::sleep(Duration::from_millis(1));
+        }
+
+        if was_active {
+            caught_active += 1;
+        }
+
+        // Drop the picker while post_scan_indexing_active is set.
+        // Take it out of the shared handle first, then drop outside the lock —
+        // Drop spins until post-scan finishes, which needs the write lock for
+        // bigram install, so we can't hold it during Drop.
+        let old_picker = shared_picker.write().unwrap().take();
+        drop(old_picker); // Drop fires here — spins until post-scan exits
+
+        assert!(
+            shared_picker.read().unwrap().is_none(),
+            "round {round}: picker should be None after drop"
+        );
+    }
+
+    // The primary invariant — dropping while post-scan may be active must not
+    // crash — is exercised every round regardless. Catching the active window
+    // is timing-dependent: with a fast walker/scan the post-scan phase can
+    // complete before the poll observes it, especially on loaded CI runners.
+    // So we only warn (not fail) if no round observed it.
+    if caught_active == 0 {
+        eprintln!(
+            "warning: never observed post_scan_indexing_active=true; \
+             drop-safety was still exercised in all rounds ({caught_active}/10)"
+        );
+    }
+    eprintln!("Caught post-scan active in {caught_active}/10 rounds");
+}

crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions

@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 2c9d1ea2efbf6161f84b69598e884dbf1bde6039c70625adde0374817e20e2ea
+cc 1ac0f8f02b160dce13ca3f3630266abd24bd32b4e36d72a6a0a5365139ded3a8