fix: LMDB stale readers and automatic compactions (#468)
* fix: LMDB stale readers and automatic compactions closes https://github.com/dmtrKovalenko/fff/issues/460 * chore: Update docs for - fix: LMDB stale readers and automatic compactions * chore: Fix CI - typos, rustfmt, stylua, lls
This commit is contained in:
committed by
GitHub
parent
a1efd5e011
commit
1104a8deaf
@@ -58,9 +58,17 @@ test-setup:
|
||||
test-rust:
|
||||
cargo test --workspace --features zlob --exclude fff-nvim
|
||||
|
||||
# neovim instance swallows internal crashes and doesn't rise the the error exiting silently
|
||||
# so check the stdout in case the sigsegv coming out of fff was printed (actual regression)
|
||||
test-lua: test-setup build
|
||||
nvim --headless -u tests/minimal_init.lua \
|
||||
-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1
|
||||
@output=$$(nvim --headless -u tests/minimal_init.lua \
|
||||
-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1); \
|
||||
echo "$$output"; \
|
||||
if echo "$$output" | grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)"; then \
|
||||
echo ""; \
|
||||
echo "FAIL: native crash detected during lua tests"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
test-version: test-setup
|
||||
nvim --headless -u tests/minimal_init.lua \
|
||||
|
||||
@@ -226,7 +226,6 @@ pub unsafe extern "C" fn fff_create_instance2(
|
||||
if let Err(e) = shared_frecency.init(tracker) {
|
||||
return FffResult::err(&format!("Failed to acquire frecency lock: {}", e));
|
||||
}
|
||||
let _ = shared_frecency.spawn_gc(frecency_path.clone());
|
||||
}
|
||||
Err(e) => return FffResult::err(&format!("Failed to init frecency db: {}", e)),
|
||||
}
|
||||
|
||||
@@ -9,16 +9,23 @@ pub struct DbHealth {
|
||||
pub disk_size: u64,
|
||||
/// Entry counts by table name
|
||||
pub entry_counts: Vec<(&'static str, u64)>,
|
||||
/// Set to `false` if can not acquire the write lock
|
||||
pub healthy: bool,
|
||||
}
|
||||
|
||||
pub trait DbHealthChecker {
|
||||
fn get_env(&self) -> &heed::Env;
|
||||
fn is_healthy(&self) -> bool;
|
||||
/// Entries per database, each group has a static string label
|
||||
fn count_entries(&self) -> Result<Vec<(&'static str, u64)>>;
|
||||
|
||||
/// Health summary of the database, returns summary struct
|
||||
fn get_health(&self) -> Result<DbHealth> {
|
||||
let env = self.get_env();
|
||||
|
||||
let size = env.real_disk_size().map_err(crate::error::Error::EnvOpen)?;
|
||||
let size = env
|
||||
.real_disk_size()
|
||||
.map_err(crate::error::Error::GenericDbError)?;
|
||||
let path = env.path().to_string_lossy().to_string();
|
||||
let entry_counts = self.count_entries()?;
|
||||
|
||||
@@ -26,6 +33,7 @@ pub trait DbHealthChecker {
|
||||
path,
|
||||
disk_size: size,
|
||||
entry_counts,
|
||||
healthy: self.is_healthy(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+115
-107
@@ -1,13 +1,10 @@
|
||||
use super::db_healthcheck::DbHealthChecker;
|
||||
use super::lmdb::{LmdbStore, is_map_full};
|
||||
use super::lmdb::{DbHealth, LmdbStore, is_map_full};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::file_picker::FFFMode;
|
||||
use crate::git::is_modified_status;
|
||||
use crate::shared::SharedFrecency;
|
||||
use heed::types::{Bytes, SerdeBincode};
|
||||
use heed::{Database, Env};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::{collections::VecDeque, path::Path};
|
||||
|
||||
@@ -24,6 +21,7 @@ const AI_MAX_HISTORY_DAYS: f64 = 7.0; // Only consider accesses within 7 days
|
||||
pub struct FrecencyTracker {
|
||||
env: Env,
|
||||
db: Database<Bytes, SerdeBincode<VecDeque<u64>>>,
|
||||
health: DbHealth,
|
||||
}
|
||||
|
||||
const MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [
|
||||
@@ -48,22 +46,53 @@ impl DbHealthChecker for FrecencyTracker {
|
||||
&self.env
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
self.health.is_healthy()
|
||||
}
|
||||
|
||||
fn count_entries(&self) -> Result<Vec<(&'static str, u64)>> {
|
||||
let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let count = self.db.len(&rtxn).map_err(Error::DbRead)?;
|
||||
let rtxn = self
|
||||
.env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
let count = self.db.len(&rtxn).map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
Ok(vec![("absolute_frecency_entries", count)])
|
||||
}
|
||||
}
|
||||
|
||||
impl LmdbStore for FrecencyTracker {
|
||||
const MAX_DBS: u32 = 0;
|
||||
const LABEL: &'static str = "frecency";
|
||||
// 10 MiB hard ceiling. Owner's db after years of use is ~560 KiB, so this
|
||||
// leaves ~18× headroom while capping runaway growth (see GH issue #437).
|
||||
const MAP_SIZE: usize = 10 * 1024 * 1024;
|
||||
const MAX_DBS: u32 = 0;
|
||||
// Nuke the db when it exceeds 8 MiB on disk — leaves a small margin under
|
||||
// MAP_SIZE so we don't hit MDB_MAP_FULL before the open-time erase fires.
|
||||
const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
|
||||
const SIZE_CAP_BYTES: u64 = 12 * 1024 * 1024;
|
||||
|
||||
fn env(&self) -> &Env {
|
||||
&self.env
|
||||
}
|
||||
|
||||
fn health(&self) -> &DbHealth {
|
||||
&self.health
|
||||
}
|
||||
|
||||
fn purge_stale_data(env: &Env) -> Result<()> {
|
||||
let (deleted, pruned) = Self::purge_stale_entries(env)?;
|
||||
if deleted > 0 || pruned > 0 {
|
||||
tracing::info!(deleted, pruned, "Frecency GC purged entries");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FrecencyTracker {
|
||||
@@ -74,11 +103,10 @@ impl FrecencyTracker {
|
||||
|
||||
pub fn open(db_path: impl AsRef<Path>) -> Result<Self> {
|
||||
let db_path = db_path.as_ref();
|
||||
let env = Self::open_env(db_path)?;
|
||||
let (env, health) = Self::open_env(db_path)?;
|
||||
|
||||
let db = Self::open_database_safe(&env, None)?;
|
||||
|
||||
Ok(FrecencyTracker { db, env })
|
||||
Ok(FrecencyTracker { db, env, health })
|
||||
}
|
||||
|
||||
#[deprecated(
|
||||
@@ -90,102 +118,41 @@ impl FrecencyTracker {
|
||||
Self::open(db_path)
|
||||
}
|
||||
|
||||
/// Spawns a background thread to purge stale frecency entries and compact the database.
|
||||
/// Run it once in a while to purge old pages and keep DB file size reasonable.
|
||||
///
|
||||
/// It's okay to not join this thread since it acquires locks for the db access
|
||||
///
|
||||
/// ```
|
||||
/// use fff_search::frecency::FrecencyTracker;
|
||||
/// use fff_search::SharedFrecency;
|
||||
/// let shared_frecency: SharedFrecency = Default::default();
|
||||
/// let _ = FrecencyTracker::spawn_gc(shared_frecency, "/path/to/frecency_db".into()).ok();
|
||||
/// ```
|
||||
pub fn spawn_gc(
|
||||
shared: SharedFrecency,
|
||||
db_path: String,
|
||||
) -> Result<std::thread::JoinHandle<()>> {
|
||||
Ok(std::thread::Builder::new()
|
||||
.name("fff-frecency-gc".into())
|
||||
.spawn(move || Self::run_frecency_gc(shared, db_path))?)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(shared), fields(db_path = %db_path))]
|
||||
fn run_frecency_gc(shared: SharedFrecency, db_path: String) {
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
let (deleted, pruned) = {
|
||||
let guard = match shared.read() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
tracing::debug!("Failed to acquire read lock: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(ref tracker) = *guard else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Clear stale readers here (on a background thread) rather than in
|
||||
// open_env — clear_stale_readers needs the writer mutex which can
|
||||
// block indefinitely on a stuck lock if called on the main thread.
|
||||
if let Err(e) = tracker.env.clear_stale_readers() {
|
||||
tracing::debug!("clear_stale_readers failed: {e}");
|
||||
}
|
||||
|
||||
match tracker.purge_stale_entries() {
|
||||
Ok(result) => result,
|
||||
Err(e) => {
|
||||
tracing::debug!("Purge failed: {e}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if deleted > 0 || pruned > 0 {
|
||||
tracing::info!(deleted, pruned, elapsed = ?start.elapsed(), "Frecency GC purged entries");
|
||||
}
|
||||
|
||||
let data_path = PathBuf::from(&db_path).join("data.mdb");
|
||||
let file_size = fs::metadata(&data_path).map(|m| m.len()).unwrap_or(0);
|
||||
if file_size > <Self as LmdbStore>::SIZE_CAP_BYTES {
|
||||
tracing::warn!(
|
||||
size = file_size,
|
||||
cap = <Self as LmdbStore>::SIZE_CAP_BYTES,
|
||||
"Frecency DB exceeds size cap — will be erased on next open"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes entries where all timestamps are older than MAX_HISTORY_DAYS,
|
||||
/// and prunes stale timestamps from entries that still have recent ones.
|
||||
/// Returns (deleted_count, pruned_count).
|
||||
fn purge_stale_entries(&self) -> Result<(usize, usize)> {
|
||||
let now = self.get_now();
|
||||
fn purge_stale_entries(env: &Env) -> Result<(usize, usize)> {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64);
|
||||
|
||||
// Collect entries to delete or update
|
||||
let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let db: Database<Bytes, SerdeBincode<VecDeque<u64>>> = Self::open_database_safe(env, None)?;
|
||||
|
||||
let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
let mut to_delete: Vec<Vec<u8>> = Vec::new();
|
||||
let mut to_update: Vec<(Vec<u8>, VecDeque<u64>)> = Vec::new();
|
||||
|
||||
let iter = self.db.iter(&rtxn).map_err(Error::DbRead)?;
|
||||
let iter = db.iter(&rtxn).map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
for result in iter {
|
||||
let (key, accesses) = result.map_err(Error::DbRead)?;
|
||||
let (key, accesses) = result.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
// Timestamps are chronologically ordered (oldest at front).
|
||||
// Find the first timestamp that is still within the retention window.
|
||||
// Timestamps chronologically ordered (oldest at front).
|
||||
let fresh_start = accesses.iter().position(|&ts| ts >= cutoff_time);
|
||||
match fresh_start {
|
||||
None => {
|
||||
// All timestamps are stale — delete the entire entry
|
||||
to_delete.push(key.to_vec());
|
||||
}
|
||||
Some(0) => {
|
||||
// All timestamps are fresh — nothing to do
|
||||
}
|
||||
None => to_delete.push(key.to_vec()),
|
||||
Some(0) => {}
|
||||
Some(start) => {
|
||||
// Some timestamps are stale — keep only the fresh ones
|
||||
let pruned: VecDeque<u64> = accesses.iter().skip(start).copied().collect();
|
||||
to_update.push((key.to_vec(), pruned));
|
||||
}
|
||||
@@ -197,17 +164,29 @@ impl FrecencyTracker {
|
||||
return Ok((0, 0));
|
||||
}
|
||||
|
||||
// Apply all changes in a single write transaction
|
||||
let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
|
||||
let mut wtxn = env.write_txn().map_err(|source| Error::DbStartWriteTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
for key in &to_delete {
|
||||
self.db.delete(&mut wtxn, key).map_err(Error::DbWrite)?;
|
||||
db.delete(&mut wtxn, key).map_err(|source| Error::DbWrite {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
}
|
||||
|
||||
for (key, accesses) in &to_update {
|
||||
self.db
|
||||
.put(&mut wtxn, key, accesses)
|
||||
.map_err(Error::DbWrite)?;
|
||||
db.put(&mut wtxn, key, accesses)
|
||||
.map_err(|source| Error::DbWrite {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
}
|
||||
wtxn.commit().map_err(Error::DbCommit)?;
|
||||
wtxn.commit().map_err(|source| Error::DbCommit {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
Ok((to_delete.len(), to_update.len()))
|
||||
}
|
||||
@@ -215,9 +194,24 @@ impl FrecencyTracker {
|
||||
fn get_accesses(&self, path: &Path) -> Result<Option<VecDeque<u64>>> {
|
||||
let key_hash = Self::path_to_hash_bytes(path)?;
|
||||
|
||||
let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let result = self.db.get(&rtxn, &key_hash).map_err(Error::DbRead)?;
|
||||
rtxn.commit().map_err(Error::DbCommit)?;
|
||||
let rtxn = self
|
||||
.env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
let result = self
|
||||
.db
|
||||
.get(&rtxn, &key_hash)
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
rtxn.commit().map_err(|source| Error::DbCommit {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -265,9 +259,16 @@ impl FrecencyTracker {
|
||||
accesses.push_back(now);
|
||||
tracing::debug!(?path, accesses = accesses.len(), "Tracking access");
|
||||
|
||||
let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
|
||||
let mut wtxn = self
|
||||
.env
|
||||
.write_txn()
|
||||
.map_err(|source| Error::DbStartWriteTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
if let Err(e) = self.db.put(&mut wtxn, &key_hash, &accesses) {
|
||||
if is_map_full(&e) {
|
||||
self.health.mark_unhealthy("MDB_MAP_FULL on put");
|
||||
tracing::error!(
|
||||
?path,
|
||||
"Frecency DB hit MDB_MAP_FULL; dropping write — db will be \
|
||||
@@ -275,19 +276,26 @@ impl FrecencyTracker {
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::DbWrite(e));
|
||||
return Err(Error::DbWrite {
|
||||
db: Self::LABEL,
|
||||
source: e,
|
||||
});
|
||||
}
|
||||
|
||||
wtxn.commit()
|
||||
.inspect_err(|e| {
|
||||
if is_map_full(e) {
|
||||
self.health.mark_unhealthy("MDB_MAP_FULL on commit");
|
||||
tracing::error!(
|
||||
?path,
|
||||
"Frecency DB hit MDB_MAP_FULL on commit; dropping write"
|
||||
);
|
||||
}
|
||||
})
|
||||
.map_err(Error::DbCommit)
|
||||
.map_err(|source| Error::DbCommit {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_access_score(&self, file_path: &Path, mode: FFFMode) -> i64 {
|
||||
|
||||
+157
-17
@@ -1,16 +1,110 @@
|
||||
use heed::{Database, Env, EnvOpenOptions};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use heed::{Database, Env, EnvOpenOptions};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
pub(crate) fn is_map_full(err: &heed::Error) -> bool {
|
||||
matches!(err, heed::Error::Mdb(heed::MdbError::MapFull))
|
||||
}
|
||||
|
||||
#[repr(u8)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum DbHealthState {
|
||||
Pending = 0,
|
||||
Healthy = 1,
|
||||
Degraded = 2,
|
||||
}
|
||||
|
||||
impl DbHealthState {
|
||||
fn from_u8(v: u8) -> Self {
|
||||
debug_assert!(v <= 2);
|
||||
|
||||
match v {
|
||||
0 => Self::Pending,
|
||||
1 => Self::Healthy,
|
||||
_ => Self::Degraded,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct DbHealth(Arc<AtomicU8>);
|
||||
|
||||
impl DbHealth {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self(Arc::new(AtomicU8::new(DbHealthState::Pending as u8)))
|
||||
}
|
||||
|
||||
pub(crate) fn is_healthy(&self) -> bool {
|
||||
// Pending counts as unhealthy: if the GC thread never flipped to
|
||||
// Healthy, something's wrong (deadlocked clear_stale_readers, stuck
|
||||
// writer mutex, etc.) and we want that surfaced to the user.
|
||||
DbHealthState::from_u8(self.0.load(Ordering::Acquire)) == DbHealthState::Healthy
|
||||
}
|
||||
|
||||
pub(crate) fn mark_healthy(&self) {
|
||||
let _ = self.0.compare_exchange(
|
||||
DbHealthState::Pending as u8,
|
||||
DbHealthState::Healthy as u8,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn mark_unhealthy(&self, reason: &'static str) {
|
||||
let prev = self.0.swap(DbHealthState::Degraded as u8, Ordering::AcqRel);
|
||||
if DbHealthState::from_u8(prev) != DbHealthState::Degraded {
|
||||
tracing::error!(reason, "LMDB tracker marked unhealthy");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns a background thread that is ensuring that the environment that was previously
|
||||
/// open is safe, accessible and doesn't have a corrupted lock.md file. If it does this thread will
|
||||
/// hang indefinitely but we will have the information that the database is in failure mode
|
||||
pub(crate) fn spawn_lmdb_gc<T: LmdbStore>(shared: Arc<RwLock<Option<T>>>) {
|
||||
let thread_shared = shared.clone();
|
||||
let spawn_result = thread::Builder::new()
|
||||
.name("fff-lmdb-gc".into())
|
||||
.spawn(move || {
|
||||
// Holding a read guard blocks `destroy` / re-init's write
|
||||
// guard until this thread finishes — natural serialization.
|
||||
let guard = match thread_shared.read() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
tracing::debug!("gc: read lock poisoned: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(ref tracker) = *guard else {
|
||||
return; // destroyed before we started
|
||||
};
|
||||
let env = tracker.env();
|
||||
|
||||
if let Err(e) = T::purge_stale_data(env) {
|
||||
tracing::debug!("purge_stale_data failed: {e}");
|
||||
}
|
||||
|
||||
tracker.health().mark_healthy();
|
||||
});
|
||||
|
||||
if let Err(e) = spawn_result {
|
||||
tracing::debug!(?e, "failed to spawn fff-lmdb-gc thread");
|
||||
// No thread = mark healthy now so healthcheck isn't stuck Pending.
|
||||
if let Ok(guard) = shared.read()
|
||||
&& let Some(ref tracker) = *guard
|
||||
{
|
||||
tracker.health().mark_healthy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrent `mdb_env_open` calls on the same path can race on macOS
|
||||
// this is for some reason fixabtly by simple retry of the open
|
||||
fn is_transient_env_open_error(err: &heed::Error) -> bool {
|
||||
@@ -23,7 +117,9 @@ fn is_transient_env_open_error(err: &heed::Error) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) trait LmdbStore {
|
||||
pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
|
||||
/// Short label used to defferintiate different instances of this trait
|
||||
const LABEL: &'static str;
|
||||
/// LMDB map size in bytes. Must be a multiple of the OS page size.
|
||||
const MAP_SIZE: usize;
|
||||
/// Number of named sub-databases. `0` for single-db envs.
|
||||
@@ -31,14 +127,30 @@ pub(crate) trait LmdbStore {
|
||||
/// Hard cap on `data.mdb` size.
|
||||
const SIZE_CAP_BYTES: u64;
|
||||
|
||||
/// Borrow the env in the read lock
|
||||
fn env(&self) -> &Env;
|
||||
/// Borrow the health flag from the tracker.
|
||||
fn health(&self) -> &DbHealth;
|
||||
|
||||
/// Override to purge stale rows, compact, etc. Default no-op. Runs on
|
||||
/// the GC thread while a read lock is held against the shared handle,
|
||||
/// so destroy / re-init naturally wait for it.
|
||||
fn purge_stale_data(_env: &Env) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open the LMDB env. Returns env + a `DbHealth` starting in Pending;
|
||||
/// the GC thread spawned by `spawn_gc` flips it to Healthy. Write
|
||||
/// paths flip it to Degraded on MDB_MAP_FULL.
|
||||
#[tracing::instrument]
|
||||
fn open_env(db_path: &Path) -> Result<Env> {
|
||||
fn open_env(db_path: &Path) -> Result<(Env, DbHealth)> {
|
||||
Self::erase_if_oversized(db_path);
|
||||
fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
|
||||
let db = Self::LABEL;
|
||||
|
||||
const MAX_ATTEMPTS: u32 = 8;
|
||||
let mut attempt = 0u32;
|
||||
loop {
|
||||
let env = loop {
|
||||
let result = unsafe {
|
||||
let mut opts = EnvOpenOptions::new();
|
||||
opts.map_size(Self::MAP_SIZE);
|
||||
@@ -49,7 +161,7 @@ pub(crate) trait LmdbStore {
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(env) => return Ok(env),
|
||||
Ok(env) => break env,
|
||||
Err(e) if is_transient_env_open_error(&e) && attempt + 1 < MAX_ATTEMPTS => {
|
||||
attempt += 1;
|
||||
tracing::debug!(
|
||||
@@ -61,9 +173,29 @@ pub(crate) trait LmdbStore {
|
||||
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
Err(e) => return Err(Error::EnvOpen(e)),
|
||||
Err(e) => return Err(Error::EnvOpen { db, source: e }),
|
||||
}
|
||||
};
|
||||
|
||||
// Reclaim reader slots left behind by prior processes that died
|
||||
// without cleanup. Must run before we start any read txns (which
|
||||
// open_database_safe does) — otherwise we may hit MDB_READERS_FULL
|
||||
// on a fresh env just because lock.mdb still has stale entries
|
||||
// from a previous crash.
|
||||
//
|
||||
// This is the one LMDB maintenance call we run on the caller's
|
||||
// thread. If the lock file is genuinely wedged this will block
|
||||
// forever, but the alternative — never getting past init — is
|
||||
// worse and the bg-thread trick doesn't solve it anyway.
|
||||
match env.clear_stale_readers() {
|
||||
Ok(cleared) if cleared > 0 => {
|
||||
tracing::warn!(cleared, "reclaimed stale LMDB reader slots at open");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::debug!("clear_stale_readers at open failed: {e}"),
|
||||
}
|
||||
|
||||
Ok((env, DbHealth::new()))
|
||||
}
|
||||
|
||||
/// Open or create a database without blocking on the LMDB writer mutex
|
||||
@@ -73,27 +205,35 @@ pub(crate) trait LmdbStore {
|
||||
KC: 'static,
|
||||
DC: 'static,
|
||||
{
|
||||
let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let maybe_db: Option<Database<KC, DC>> =
|
||||
env.open_database(&rtxn, name).map_err(Error::DbOpen)?;
|
||||
let db = Self::LABEL;
|
||||
let rtxn = env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn { db, source })?;
|
||||
let maybe_db: Option<Database<KC, DC>> = env
|
||||
.open_database(&rtxn, name)
|
||||
.map_err(|source| Error::DbOpen { db, source })?;
|
||||
|
||||
// do not drop the DB here
|
||||
rtxn.commit().map_err(Error::DbCommit)?;
|
||||
rtxn.commit()
|
||||
.map_err(|source| Error::DbCommit { db, source })?;
|
||||
|
||||
match maybe_db {
|
||||
Some(db) => Ok(db),
|
||||
Some(handle) => Ok(handle),
|
||||
None => {
|
||||
// First time: create the database (requires write lock).
|
||||
// unfortunately this CAN be deadlocking and this is what we see happens
|
||||
// if the other part of the code is segfaulting, so the only rule to prevent this
|
||||
// write the good code mf, okay?
|
||||
let mut wtxn = env.write_txn().map_err(Error::DbStartWriteTxn)?;
|
||||
let db = env
|
||||
let mut wtxn = env
|
||||
.write_txn()
|
||||
.map_err(|source| Error::DbStartWriteTxn { db, source })?;
|
||||
let handle = env
|
||||
.create_database(&mut wtxn, name)
|
||||
.map_err(Error::DbCreate)?;
|
||||
.map_err(|source| Error::DbCreate { db, source })?;
|
||||
|
||||
wtxn.commit().map_err(Error::DbCommit)?;
|
||||
Ok(db)
|
||||
wtxn.commit()
|
||||
.map_err(|source| Error::DbCommit { db, source })?;
|
||||
Ok(handle)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::db_healthcheck::DbHealthChecker;
|
||||
use super::lmdb::{LmdbStore, is_map_full};
|
||||
use super::lmdb::{DbHealth, LmdbStore, is_map_full};
|
||||
use crate::error::Error;
|
||||
use heed::types::{Bytes, SerdeBincode};
|
||||
use heed::{Database, Env};
|
||||
@@ -34,6 +34,7 @@ pub struct QueryTracker {
|
||||
query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
|
||||
// Database for project_path -> VecDeque<HistoryEntry> mappings (grep)
|
||||
grep_query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
|
||||
health: DbHealth,
|
||||
}
|
||||
|
||||
impl DbHealthChecker for QueryTracker {
|
||||
@@ -41,15 +42,40 @@ impl DbHealthChecker for QueryTracker {
|
||||
&self.env
|
||||
}
|
||||
|
||||
fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
|
||||
let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
fn is_healthy(&self) -> bool {
|
||||
self.health.is_healthy()
|
||||
}
|
||||
|
||||
let count_queries = self.query_file_db.len(&rtxn).map_err(Error::DbRead)?;
|
||||
let count_histories = self.query_history_db.len(&rtxn).map_err(Error::DbRead)?;
|
||||
let count_grep_histories = self
|
||||
.grep_query_history_db
|
||||
fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
|
||||
let rtxn = self
|
||||
.env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
let count_queries = self
|
||||
.query_file_db
|
||||
.len(&rtxn)
|
||||
.map_err(Error::DbRead)?;
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
let count_histories = self
|
||||
.query_history_db
|
||||
.len(&rtxn)
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
let count_grep_histories =
|
||||
self.grep_query_history_db
|
||||
.len(&rtxn)
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
Ok(vec![
|
||||
("query_file_entries", count_queries),
|
||||
@@ -60,12 +86,19 @@ impl DbHealthChecker for QueryTracker {
|
||||
}
|
||||
|
||||
impl LmdbStore for QueryTracker {
|
||||
const LABEL: &'static str = "query";
|
||||
// 10 MiB hard ceiling. Same reasoning as FrecencyTracker (GH issue #437).
|
||||
const MAP_SIZE: usize = 10 * 1024 * 1024;
|
||||
const MAX_DBS: u32 = 16;
|
||||
// Nuke at 4 MiB — query history is bounded per-project but query→file
|
||||
// associations grow unbounded over typing time.
|
||||
const SIZE_CAP_BYTES: u64 = 4 * 1024 * 1024;
|
||||
const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
|
||||
|
||||
fn env(&self) -> &Env {
|
||||
&self.env
|
||||
}
|
||||
|
||||
fn health(&self) -> &DbHealth {
|
||||
&self.health
|
||||
}
|
||||
}
|
||||
|
||||
impl QueryTracker {
|
||||
@@ -76,7 +109,7 @@ impl QueryTracker {
|
||||
|
||||
pub fn open(db_path: impl AsRef<Path>) -> Result<Self, Error> {
|
||||
let db_path = db_path.as_ref();
|
||||
let env = Self::open_env(db_path)?;
|
||||
let (env, health) = Self::open_env(db_path)?;
|
||||
|
||||
let query_file_db = Self::open_database_safe(&env, Some("query_file_associations"))?;
|
||||
let query_history_db = Self::open_database_safe(&env, Some("query_history"))?;
|
||||
@@ -87,6 +120,7 @@ impl QueryTracker {
|
||||
query_file_db,
|
||||
query_history_db,
|
||||
grep_query_history_db,
|
||||
health,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -137,7 +171,10 @@ impl QueryTracker {
|
||||
) -> Result<(), Error> {
|
||||
let mut history = db
|
||||
.get(wtxn, project_key)
|
||||
.map_err(Error::DbRead)?
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?
|
||||
.unwrap_or_default();
|
||||
|
||||
history.push_back(HistoryEntry {
|
||||
@@ -149,7 +186,10 @@ impl QueryTracker {
|
||||
}
|
||||
|
||||
db.put(wtxn, project_key, &history)
|
||||
.map_err(Error::DbWrite)?;
|
||||
.map_err(|source| Error::DbWrite {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -161,11 +201,17 @@ impl QueryTracker {
|
||||
project_key: &[u8; 32],
|
||||
offset: usize,
|
||||
) -> Result<Option<String>, Error> {
|
||||
let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
let mut history = db
|
||||
.get(&rtxn, project_key)
|
||||
.map_err(Error::DbRead)?
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?
|
||||
.unwrap_or_default();
|
||||
|
||||
// history is FIFO, last element is most recent
|
||||
@@ -188,12 +234,21 @@ impl QueryTracker {
|
||||
let file_path_buf = file_path.to_path_buf();
|
||||
|
||||
let query_key = Self::create_query_key(project_path, query)?;
|
||||
let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
|
||||
let mut wtxn = self
|
||||
.env
|
||||
.write_txn()
|
||||
.map_err(|source| Error::DbStartWriteTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
let mut entry = self
|
||||
.query_file_db
|
||||
.get(&wtxn, &query_key)
|
||||
.map_err(Error::DbRead)?
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?
|
||||
.unwrap_or_else(|| QueryMatchEntry {
|
||||
file_path: file_path_buf.clone(),
|
||||
open_count: 0,
|
||||
@@ -225,6 +280,7 @@ impl QueryTracker {
|
||||
|
||||
if let Err(e) = self.query_file_db.put(&mut wtxn, &query_key, &entry) {
|
||||
if is_map_full(&e) {
|
||||
self.health.mark_unhealthy("MDB_MAP_FULL on put");
|
||||
tracing::error!(
|
||||
?query,
|
||||
"Query tracker DB hit MDB_MAP_FULL; dropping write — db will \
|
||||
@@ -232,7 +288,10 @@ impl QueryTracker {
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::DbWrite(e));
|
||||
return Err(Error::DbWrite {
|
||||
db: Self::LABEL,
|
||||
source: e,
|
||||
});
|
||||
}
|
||||
|
||||
// Update query history database
|
||||
@@ -240,9 +299,12 @@ impl QueryTracker {
|
||||
if let Err(e) =
|
||||
Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now)
|
||||
{
|
||||
if let Error::DbWrite(ref inner) = e
|
||||
if let Error::DbWrite {
|
||||
source: ref inner, ..
|
||||
} = e
|
||||
&& is_map_full(inner)
|
||||
{
|
||||
self.health.mark_unhealthy("MDB_MAP_FULL on history append");
|
||||
tracing::error!(?query, "Query tracker DB map full while appending history");
|
||||
return Ok(());
|
||||
}
|
||||
@@ -251,10 +313,14 @@ impl QueryTracker {
|
||||
|
||||
if let Err(e) = wtxn.commit() {
|
||||
if is_map_full(&e) {
|
||||
self.health.mark_unhealthy("MDB_MAP_FULL on commit");
|
||||
tracing::error!(?query, "Query tracker DB map full on commit");
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::DbCommit(e));
|
||||
return Err(Error::DbCommit {
|
||||
db: Self::LABEL,
|
||||
source: e,
|
||||
});
|
||||
}
|
||||
|
||||
tracing::debug!(?query, ?file_path, "Tracked query completion");
|
||||
@@ -268,12 +334,21 @@ impl QueryTracker {
|
||||
min_combo_count: u32,
|
||||
) -> Result<Option<QueryMatchEntry>, Error> {
|
||||
let query_key = Self::create_query_key(project_path, query)?;
|
||||
let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let rtxn = self
|
||||
.env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
let last_match = self
|
||||
.query_file_db
|
||||
.get(&rtxn, &query_key)
|
||||
.map_err(Error::DbRead)?;
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
Ok(last_match.filter(|entry| entry.open_count >= min_combo_count))
|
||||
}
|
||||
@@ -287,13 +362,21 @@ impl QueryTracker {
|
||||
) -> Result<i32, Error> {
|
||||
let query_key = Self::create_query_key(project_path, query)?;
|
||||
tracing::debug!(?query_key, "HASH");
|
||||
let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
|
||||
let rtxn = self
|
||||
.env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
match self
|
||||
.query_file_db
|
||||
.get(&rtxn, &query_key)
|
||||
.map_err(Error::DbRead)?
|
||||
{
|
||||
.map_err(|source| Error::DbRead {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})? {
|
||||
Some(entry) => {
|
||||
// Check if the file path matches and return boost
|
||||
if entry.file_path == file_path && entry.open_count >= 2 {
|
||||
@@ -322,7 +405,13 @@ impl QueryTracker {
|
||||
pub fn track_grep_query(&mut self, query: &str, project_path: &Path) -> Result<(), Error> {
|
||||
let now = self.get_now();
|
||||
let project_key = Self::create_project_key(project_path)?;
|
||||
let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
|
||||
let mut wtxn = self
|
||||
.env
|
||||
.write_txn()
|
||||
.map_err(|source| Error::DbStartWriteTxn {
|
||||
db: Self::LABEL,
|
||||
source,
|
||||
})?;
|
||||
|
||||
if let Err(e) = Self::append_to_history(
|
||||
&self.grep_query_history_db,
|
||||
@@ -331,9 +420,13 @@ impl QueryTracker {
|
||||
query,
|
||||
now,
|
||||
) {
|
||||
if let Error::DbWrite(ref inner) = e
|
||||
if let Error::DbWrite {
|
||||
source: ref inner, ..
|
||||
} = e
|
||||
&& is_map_full(inner)
|
||||
{
|
||||
self.health
|
||||
.mark_unhealthy("MDB_MAP_FULL on grep history append");
|
||||
tracing::error!(?query, "Grep query history DB map full; dropping write");
|
||||
return Ok(());
|
||||
}
|
||||
@@ -342,10 +435,14 @@ impl QueryTracker {
|
||||
|
||||
if let Err(e) = wtxn.commit() {
|
||||
if is_map_full(&e) {
|
||||
self.health.mark_unhealthy("MDB_MAP_FULL on commit");
|
||||
tracing::error!(?query, "Grep query history DB map full on commit");
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::DbCommit(e));
|
||||
return Err(Error::DbCommit {
|
||||
db: Self::LABEL,
|
||||
source: e,
|
||||
});
|
||||
}
|
||||
|
||||
tracing::debug!(?query, "Tracked grep query");
|
||||
|
||||
@@ -26,26 +26,63 @@ pub enum Error {
|
||||
path: std::path::PathBuf,
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("Failed to open frecency database env: {0}")]
|
||||
EnvOpen(#[source] heed::Error),
|
||||
#[error("Failed to create frecency database: {0}")]
|
||||
DbCreate(#[source] heed::Error),
|
||||
#[error("Failed to open frecency database: {0}")]
|
||||
DbOpen(#[source] heed::Error),
|
||||
#[error("Failed to clear stale readers for frecency database: {0}")]
|
||||
DbClearStaleReaders(#[source] heed::Error),
|
||||
#[error("Something is wrong with the local db instance: {0}")]
|
||||
GenericDbError(#[from] heed::Error),
|
||||
#[error("Failed to open {db} database env: {source}")]
|
||||
EnvOpen {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to create {db} database: {source}")]
|
||||
DbCreate {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to open {db} database: {source}")]
|
||||
DbOpen {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to clear stale readers for {db} database: {source}")]
|
||||
DbClearStaleReaders {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
|
||||
#[error("Failed to start read transaction for frecency database: {0}")]
|
||||
DbStartReadTxn(#[source] heed::Error),
|
||||
#[error("Failed to start write transaction for frecency database: {0}")]
|
||||
DbStartWriteTxn(#[source] heed::Error),
|
||||
|
||||
#[error("Failed to read from frecency database: {0}")]
|
||||
DbRead(#[source] heed::Error),
|
||||
#[error("Failed to write to frecency database: {0}")]
|
||||
DbWrite(#[source] heed::Error),
|
||||
#[error("Failed to commit write transaction to frecency database: {0}")]
|
||||
DbCommit(#[source] heed::Error),
|
||||
#[error("Failed to start read transaction for {db} database: {source}")]
|
||||
DbStartReadTxn {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to start write transaction for {db} database: {source}")]
|
||||
DbStartWriteTxn {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to read from {db} database: {source}")]
|
||||
DbRead {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to write to {db} database: {source}")]
|
||||
DbWrite {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to commit write transaction to {db} database: {source}")]
|
||||
DbCommit {
|
||||
db: &'static str,
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error("Failed to start file system watcher: {0}")]
|
||||
FileSystemWatch(#[from] notify::Error),
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::dbs::lmdb::spawn_lmdb_gc;
|
||||
use crate::error::Error;
|
||||
use crate::file_picker::FilePicker;
|
||||
use crate::frecency::FrecencyTracker;
|
||||
@@ -292,19 +293,20 @@ impl SharedFrecency {
|
||||
self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
|
||||
}
|
||||
|
||||
/// Initialize the frecency tracker. No-op if this is a disabled instance.
|
||||
pub fn init(&self, tracker: FrecencyTracker) -> Result<(), Error> {
|
||||
if !self.enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let mut guard = self.write()?;
|
||||
*guard = Some(tracker);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Spawn a background GC thread for this frecency tracker.
|
||||
pub fn spawn_gc(&self, db_path: String) -> crate::Result<std::thread::JoinHandle<()>> {
|
||||
FrecencyTracker::spawn_gc(self.clone(), db_path)
|
||||
{
|
||||
let mut guard = self.write()?;
|
||||
*guard = Some(tracker);
|
||||
}
|
||||
|
||||
// GC holds a read guard on this lock, so destroy / re-init wait
|
||||
// for it naturally — no join handle, no race against file removal.
|
||||
spawn_lmdb_gc(self.inner.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop the in-memory tracker and delete the on-disk database directory.
|
||||
@@ -371,17 +373,22 @@ impl SharedQueryTracker {
|
||||
self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
|
||||
}
|
||||
|
||||
/// Initialize the query tracker. No-op if this is a disabled instance.
|
||||
/// Initialize the query tracker + spawn GC in the background.
|
||||
/// No-op if this is a disabled instance.
|
||||
pub fn init(&self, tracker: QueryTracker) -> Result<(), Error> {
|
||||
if !self.enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let mut guard = self.write()?;
|
||||
*guard = Some(tracker);
|
||||
{
|
||||
let mut guard = self.write()?;
|
||||
*guard = Some(tracker);
|
||||
}
|
||||
|
||||
spawn_lmdb_gc(self.inner.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop the in-memory tracker and delete the on-disk database directory.
|
||||
///Drop the in-memory tracker and delete the on-disk database directory.
|
||||
///
|
||||
/// Acquires the write lock, ensuring all readers (including any active mmap
|
||||
/// access) are finished before the LMDB environment is closed and the files
|
||||
|
||||
@@ -238,7 +238,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
match FrecencyTracker::open(&frecency_db_path) {
|
||||
Ok(tracker) => {
|
||||
let _ = shared_frecency.init(tracker);
|
||||
let _ = shared_frecency.spawn_gc(frecency_db_path);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Warning: Failed to init frecency db: {}", e);
|
||||
|
||||
+10
-16
@@ -35,24 +35,16 @@ pub fn init_db(
|
||||
_: &Lua,
|
||||
(frecency_db_path, history_db_path, _use_unsafe_no_lock): (String, String, bool),
|
||||
) -> LuaResult<bool> {
|
||||
let mut frecency = FRECENCY.write().into_lua_result()?;
|
||||
if frecency.is_some() {
|
||||
*frecency = None;
|
||||
}
|
||||
*frecency = Some(FrecencyTracker::open(&frecency_db_path).into_lua_result()?);
|
||||
// Route through SharedFrecency::init / SharedQueryTracker::init so the
|
||||
// GC thread gets spawned + bound to the shared handle's RwLock.
|
||||
FRECENCY
|
||||
.init(FrecencyTracker::open(&frecency_db_path).into_lua_result()?)
|
||||
.into_lua_result()?;
|
||||
tracing::info!("Frecency database initialized at {}", frecency_db_path);
|
||||
drop(frecency);
|
||||
|
||||
// Spawn background GC to purge stale entries without blocking startup
|
||||
let _ = FRECENCY.spawn_gc(frecency_db_path);
|
||||
|
||||
let mut query_tracker = QUERY_TRACKER.write().into_lua_result()?;
|
||||
if query_tracker.is_some() {
|
||||
*query_tracker = None;
|
||||
}
|
||||
|
||||
*query_tracker = Some(QueryTracker::open(&history_db_path).into_lua_result()?);
|
||||
|
||||
QUERY_TRACKER
|
||||
.init(QueryTracker::open(&history_db_path).into_lua_result()?)
|
||||
.into_lua_result()?;
|
||||
tracing::info!("Query tracker database initialized at {}", history_db_path);
|
||||
Ok(true)
|
||||
}
|
||||
@@ -739,6 +731,7 @@ pub fn health_check(lua: &Lua, test_path: Option<String>) -> LuaResult<LuaValue>
|
||||
let healthcheck_table = lua.create_table()?;
|
||||
healthcheck_table.set("path", health.path)?;
|
||||
healthcheck_table.set("disk_size", health.disk_size)?;
|
||||
healthcheck_table.set("healthy", health.healthy)?;
|
||||
for (name, count) in health.entry_counts {
|
||||
healthcheck_table.set(name, count)?;
|
||||
}
|
||||
@@ -767,6 +760,7 @@ pub fn health_check(lua: &Lua, test_path: Option<String>) -> LuaResult<LuaValue>
|
||||
let healthcheck_table = lua.create_table()?;
|
||||
healthcheck_table.set("path", health.path)?;
|
||||
healthcheck_table.set("disk_size", health.disk_size)?;
|
||||
healthcheck_table.set("healthy", health.healthy)?;
|
||||
for (name, count) in health.entry_counts {
|
||||
healthcheck_table.set(name, count)?;
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
*fff.nvim.txt*
|
||||
For Neovim >= 0.10.0 Last change: 2026 May 11
|
||||
For Neovim >= 0.10.0 Last change: 2026 May 12
|
||||
|
||||
==============================================================================
|
||||
Table of Contents *fff.nvim-table-of-contents*
|
||||
|
||||
+83
-81
@@ -13,6 +13,60 @@ local function fetch_rust_checkhealth(rust_module, test_path)
|
||||
return result, nil
|
||||
end
|
||||
|
||||
-- Report health for a rust-side LMDB section (frecency / query_tracker).
|
||||
-- opts.populate(section, db_info) copies DB-specific counters into health_section.
|
||||
-- opts.format_healthy_msg(db_info) returns the message for the healthy path;
|
||||
-- the unhealthy path is handled uniformly here as a critical error.
|
||||
local function report_db_health(rust_section, health_section, messages, label, opts)
|
||||
if not rust_section then return end
|
||||
|
||||
health_section.initialized = rust_section.initialized
|
||||
health_section.error = rust_section.error
|
||||
|
||||
if not rust_section.initialized then
|
||||
table.insert(messages, {
|
||||
level = 'info',
|
||||
msg = label .. ' not initialized (will initialize on first use)',
|
||||
})
|
||||
return
|
||||
end
|
||||
|
||||
local db_info = rust_section.db_healthcheck
|
||||
if not db_info then
|
||||
if rust_section.db_healthcheck_error then
|
||||
table.insert(messages, {
|
||||
level = 'warn',
|
||||
msg = label .. ' initialized but health check failed: ' .. rust_section.db_healthcheck_error,
|
||||
})
|
||||
else
|
||||
table.insert(messages, { level = 'ok', msg = label .. ' initialized' })
|
||||
end
|
||||
return
|
||||
end
|
||||
|
||||
health_section.db_path = db_info.path
|
||||
health_section.disk_size = db_info.disk_size
|
||||
health_section.healthy = db_info.healthy
|
||||
if opts.populate then opts.populate(health_section, db_info) end
|
||||
|
||||
if db_info.healthy == false then
|
||||
table.insert(messages, {
|
||||
level = 'error',
|
||||
msg = string.format(
|
||||
'%s UNRESPONSIVE — CRITICAL ERROR (see logs; try removing %s/lock.mdb to unblock)',
|
||||
label,
|
||||
db_info.path or 'unknown'
|
||||
),
|
||||
})
|
||||
return
|
||||
end
|
||||
|
||||
table.insert(messages, {
|
||||
level = 'ok',
|
||||
msg = opts.format_healthy_msg(db_info),
|
||||
})
|
||||
end
|
||||
|
||||
--- Check snacks.nvim image preview availability
|
||||
--- @return table image_preview_info
|
||||
local function check_image_preview()
|
||||
@@ -77,6 +131,7 @@ function M.run(opts)
|
||||
db_path = nil,
|
||||
disk_size = nil,
|
||||
entries = nil,
|
||||
healthy = nil,
|
||||
error = nil,
|
||||
},
|
||||
query_tracker = {
|
||||
@@ -85,6 +140,7 @@ function M.run(opts)
|
||||
disk_size = nil,
|
||||
query_file_entries = nil,
|
||||
query_history_entries = nil,
|
||||
healthy = nil,
|
||||
error = nil,
|
||||
},
|
||||
},
|
||||
@@ -175,6 +231,7 @@ function M.run(opts)
|
||||
|
||||
if rust_health.file_picker.initialized then
|
||||
local status = rust_health.file_picker.is_scanning and 'scanning' or 'ready'
|
||||
|
||||
table.insert(health.messages, {
|
||||
level = 'ok',
|
||||
msg = string.format(
|
||||
@@ -192,88 +249,33 @@ function M.run(opts)
|
||||
end
|
||||
end
|
||||
|
||||
-- Frecency database status
|
||||
if rust_health.frecency then
|
||||
health.rust.frecency.initialized = rust_health.frecency.initialized
|
||||
health.rust.frecency.error = rust_health.frecency.error
|
||||
report_db_health(rust_health.frecency, health.rust.frecency, health.messages, 'Frecency database', {
|
||||
populate = function(section, db_info) section.entries = db_info.absolute_frecency_entries end,
|
||||
format_healthy_msg = function(db_info)
|
||||
return string.format(
|
||||
'Frecency database operational (%d entries, %s, path: %s)',
|
||||
db_info.absolute_frecency_entries or 0,
|
||||
utils.format_file_size(db_info.disk_size or 0),
|
||||
db_info.path or 'unknown'
|
||||
)
|
||||
end,
|
||||
})
|
||||
|
||||
if rust_health.frecency.initialized then
|
||||
local db_info = rust_health.frecency.db_healthcheck
|
||||
if db_info then
|
||||
health.rust.frecency.db_path = db_info.path
|
||||
health.rust.frecency.disk_size = db_info.disk_size
|
||||
health.rust.frecency.entries = db_info.absolute_frecency_entries
|
||||
|
||||
table.insert(health.messages, {
|
||||
level = 'ok',
|
||||
msg = string.format(
|
||||
'Frecency database initialized (%d entries, %s, path: %s)',
|
||||
db_info.absolute_frecency_entries or 0,
|
||||
utils.format_file_size(db_info.disk_size or 0),
|
||||
db_info.path or 'unknown'
|
||||
),
|
||||
})
|
||||
elseif rust_health.frecency.db_healthcheck_error then
|
||||
table.insert(health.messages, {
|
||||
level = 'warn',
|
||||
msg = 'Frecency database initialized but health check failed: '
|
||||
.. rust_health.frecency.db_healthcheck_error,
|
||||
})
|
||||
else
|
||||
table.insert(health.messages, {
|
||||
level = 'ok',
|
||||
msg = 'Frecency database initialized',
|
||||
})
|
||||
end
|
||||
else
|
||||
table.insert(health.messages, {
|
||||
level = 'info',
|
||||
msg = 'Frecency database not initialized (will initialize on first use)',
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
if rust_health.query_tracker then
|
||||
health.rust.query_tracker.initialized = rust_health.query_tracker.initialized
|
||||
health.rust.query_tracker.error = rust_health.query_tracker.error
|
||||
|
||||
if rust_health.query_tracker.initialized then
|
||||
local db_info = rust_health.query_tracker.db_healthcheck
|
||||
if db_info then
|
||||
health.rust.query_tracker.db_path = db_info.path
|
||||
health.rust.query_tracker.disk_size = db_info.disk_size
|
||||
health.rust.query_tracker.query_file_entries = db_info.query_file_entries
|
||||
health.rust.query_tracker.query_history_entries = db_info.query_history_entries
|
||||
|
||||
table.insert(health.messages, {
|
||||
level = 'ok',
|
||||
msg = string.format(
|
||||
'Query tracker initialized (%d query-file mappings, %d history entries, %s, path: %s)',
|
||||
db_info.query_file_entries or 0,
|
||||
db_info.query_history_entries or 0,
|
||||
utils.format_file_size(db_info.disk_size or 0),
|
||||
db_info.path or 'unknown'
|
||||
),
|
||||
})
|
||||
elseif rust_health.query_tracker.db_healthcheck_error then
|
||||
table.insert(health.messages, {
|
||||
level = 'warn',
|
||||
msg = 'Query tracker initialized but health check failed: '
|
||||
.. rust_health.query_tracker.db_healthcheck_error,
|
||||
})
|
||||
else
|
||||
table.insert(health.messages, {
|
||||
level = 'ok',
|
||||
msg = 'Query tracker initialized',
|
||||
})
|
||||
end
|
||||
else
|
||||
table.insert(health.messages, {
|
||||
level = 'info',
|
||||
msg = 'Query tracker not initialized (will initialize on first use)',
|
||||
})
|
||||
end
|
||||
end
|
||||
report_db_health(rust_health.query_tracker, health.rust.query_tracker, health.messages, 'Query cache database', {
|
||||
populate = function(section, db_info)
|
||||
section.query_file_entries = db_info.query_file_entries
|
||||
section.query_history_entries = db_info.query_history_entries
|
||||
end,
|
||||
format_healthy_msg = function(db_info)
|
||||
return string.format(
|
||||
'Query cache database operational (%d query-file mappings, %d history entries, %s, path: %s)',
|
||||
db_info.query_file_entries or 0,
|
||||
db_info.query_history_entries or 0,
|
||||
utils.format_file_size(db_info.disk_size or 0),
|
||||
db_info.path or 'unknown'
|
||||
)
|
||||
end,
|
||||
})
|
||||
else
|
||||
health.ok = false
|
||||
table.insert(health.messages, {
|
||||
|
||||
@@ -262,8 +262,13 @@ local function compute_layout(config)
|
||||
total_width = width,
|
||||
-- Top/bottom preview with prompt-top has a 2-row chrome over-subtraction in
|
||||
-- calculate_layout_dimensions (BORDER_SIZE is subtracted twice). Compensate at fullscreen.
|
||||
total_height = (is_fullscreen and prompt_position == 'top'
|
||||
and (preview_position == 'top' or preview_position == 'bottom')) and height + 2 or height,
|
||||
total_height = (
|
||||
is_fullscreen
|
||||
and prompt_position == 'top'
|
||||
and (preview_position == 'top' or preview_position == 'bottom')
|
||||
)
|
||||
and height + 2
|
||||
or height,
|
||||
start_col = col,
|
||||
start_row = row,
|
||||
preview_position = preview_position,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
-- Sanity test: open fresh (non-existent) LMDB dbs, run a couple of picker
|
||||
-- calls, then close. Verifies the health_check returns `healthy = true` on a
|
||||
-- freshly-initialized tracker — i.e. the GC thread actually ran and flipped
|
||||
-- the flag out of Pending.
|
||||
--
|
||||
-- Run with:
|
||||
-- nvim -l tests/fresh_db_open_test.lua
|
||||
--
|
||||
-- Exit code 0 = success, non-zero = failure (error printed).
|
||||
|
||||
local function die(msg)
|
||||
io.stderr:write('FAIL: ' .. msg .. '\n')
|
||||
os.exit(1)
|
||||
end
|
||||
|
||||
local function ok(msg) print('ok ' .. msg) end
|
||||
|
||||
-- Resolve plugin dir and add to runtimepath. `arg[0]` is the script path
|
||||
-- under `nvim -l`, while `<sfile>` is not set in that mode.
|
||||
local script_path = arg and arg[0] or debug.getinfo(1, 'S').source:sub(2)
|
||||
local plugin_dir = vim.fn.fnamemodify(vim.fn.resolve(script_path), ':h:h')
|
||||
vim.opt.runtimepath:prepend(plugin_dir)
|
||||
|
||||
-- Force brand new db paths so we exercise the fresh-open code path
|
||||
local tmp_frecency = vim.fn.tempname() .. '_fresh_frec'
|
||||
local tmp_history = vim.fn.tempname() .. '_fresh_hist'
|
||||
|
||||
vim.fn.delete(tmp_frecency, 'rf')
|
||||
vim.fn.delete(tmp_history, 'rf')
|
||||
|
||||
local fff_rust = require('fff.rust')
|
||||
|
||||
-- Init dbs at the fresh paths
|
||||
local init_ok = fff_rust.init_db(tmp_frecency, tmp_history, true)
|
||||
if not init_ok then die('init_db returned false') end
|
||||
ok('init_db(fresh paths)')
|
||||
|
||||
-- Init the picker rooted at the plugin dir so there's something to scan
|
||||
local picker_ok = fff_rust.init_file_picker(plugin_dir)
|
||||
if not picker_ok then die('init_file_picker returned false') end
|
||||
ok('init_file_picker(plugin_dir)')
|
||||
|
||||
fff_rust.wait_for_initial_scan(10000)
|
||||
ok('initial scan complete')
|
||||
|
||||
-- Actually run a search so the picker touches the frecency/query dbs.
|
||||
-- Signature: (query, max_threads, current_file, combo_boost, min_combo, page_index, page_size)
|
||||
local results = fff_rust.fuzzy_search_files('lib.rs', 4, nil, 0, nil, nil, nil)
|
||||
if type(results) ~= 'table' then die('fuzzy_search_files did not return a table') end
|
||||
ok(string.format('fuzzy_search_files returned %d results', #(results.items or results)))
|
||||
|
||||
-- Give the GC thread up to ~2s to flip Pending -> Healthy
|
||||
---@type any
|
||||
local health = nil
|
||||
local deadline = vim.loop.now() + 2000
|
||||
while vim.loop.now() < deadline do
|
||||
health = fff_rust.health_check(plugin_dir)
|
||||
local frec = health and health.frecency and health.frecency.db_healthcheck
|
||||
local qt = health and health.query_tracker and health.query_tracker.db_healthcheck
|
||||
if frec and qt and frec.healthy == true and qt.healthy == true then break end
|
||||
vim.wait(50)
|
||||
end
|
||||
|
||||
if not health then die('health_check returned nil') end
|
||||
ok('health_check returned a table')
|
||||
|
||||
local frec = health.frecency and health.frecency.db_healthcheck
|
||||
local qt = health.query_tracker and health.query_tracker.db_healthcheck
|
||||
|
||||
if not frec then die('frecency db_healthcheck missing from health result') end
|
||||
if not qt then die('query_tracker db_healthcheck missing from health result') end
|
||||
|
||||
if frec.healthy ~= true then die('frecency.healthy expected true, got ' .. tostring(frec.healthy)) end
|
||||
ok('frecency.healthy = true')
|
||||
|
||||
if qt.healthy ~= true then die('query_tracker.healthy expected true, got ' .. tostring(qt.healthy)) end
|
||||
ok('query_tracker.healthy = true')
|
||||
|
||||
-- Confirm the db dirs now exist on disk (proves we actually opened them)
|
||||
if vim.fn.isdirectory(tmp_frecency) ~= 1 then die('frecency dir missing after init: ' .. tmp_frecency) end
|
||||
if vim.fn.isdirectory(tmp_history) ~= 1 then die('history dir missing after init: ' .. tmp_history) end
|
||||
ok('db directories exist on disk')
|
||||
|
||||
-- Cleanup
|
||||
pcall(fff_rust.stop_background_monitor)
|
||||
pcall(fff_rust.cleanup_file_picker)
|
||||
pcall(fff_rust.destroy_frecency_db)
|
||||
pcall(fff_rust.destroy_query_db)
|
||||
vim.fn.delete(tmp_frecency, 'rf')
|
||||
vim.fn.delete(tmp_history, 'rf')
|
||||
ok('cleanup complete')
|
||||
|
||||
print('\nALL CHECKS PASSED')
|
||||
os.exit(0)
|
||||
Reference in New Issue
Block a user