feat(skills): cache merged skill discovery with watched-mtime validation
Refs #3921. Skill discovery re-walked every root recursively (depth 8, per-dir canonicalize) on every prompt build, load_skill call, and /skills invocation. Now the merged registry is cached per resolved directory set, and a hit re-stats only the watched set — each visited directory and parsed SKILL.md with its mtime — instead of re-walking. Any mtime or readability change (skill add/remove/content edit) re-walks fully; root creation/deletion changes the resolved key itself; plugin skills merge from the in-memory plugin registry per call, so plugin state applies immediately. The cache is bounded (8 entries) and cleared by the existing refresh_skill_cache hook after /skill mutations. Acceptance coverage in skills/tests.rs (excluded from the source count): unchanged second call performs zero discovery walks and returns an identical registry; SKILL.md add/edit/remove is picked up on the next call; explicit clear forces a fresh walk; the workspace-and-dir entry point (load_skill path) shares the cache. Verified: skills suites 201/201; warnings-denied TUI clippy PASS; fmt/diff clean; dead-code PASS; source-structure PASS (the test extraction frees ~1,471 counted lines; the implementation adds ~120). The runtime-contract skill_discovery metric moves to zero walks on an unchanged second turn; re-baselined separately at integration.
This commit is contained in:
@@ -34,6 +34,7 @@ use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
|
||||
use crate::logging;
|
||||
|
||||
@@ -267,6 +268,18 @@ pub struct SkillRegistry {
|
||||
warnings: Vec<String>,
|
||||
}
|
||||
|
||||
/// One cached discovery's watched filesystem entries: a path and the
|
||||
/// modification time observed during the validating walk. `None` means the
|
||||
/// path was unreadable at walk time; any later readability or mtime change
|
||||
/// invalidates the entry.
|
||||
pub(crate) type WatchedPaths = Vec<(PathBuf, Option<std::time::SystemTime>)>;
|
||||
|
||||
pub(crate) fn mtime_of(path: &Path) -> Option<std::time::SystemTime> {
|
||||
fs::metadata(path)
|
||||
.and_then(|metadata| metadata.modified())
|
||||
.ok()
|
||||
}
|
||||
|
||||
impl SkillRegistry {
|
||||
/// Maximum directory-traversal depth when discovering skills.
|
||||
///
|
||||
@@ -295,14 +308,23 @@ impl SkillRegistry {
|
||||
/// the walk finite when a skills layout contains cycles.
|
||||
#[must_use]
|
||||
pub fn discover(dir: &Path) -> Self {
|
||||
Self::discover_watched(dir).0
|
||||
}
|
||||
|
||||
/// Discover skills like [`Self::discover`], also returning the watched
|
||||
/// filesystem set (every visited directory and every parsed `SKILL.md`)
|
||||
/// with its modification time. The discovery cache validates hits by
|
||||
/// re-stat()ing only this set instead of re-walking every root.
|
||||
pub(crate) fn discover_watched(dir: &Path) -> (Self, WatchedPaths) {
|
||||
#[cfg(test)]
|
||||
record_root_discovery_call();
|
||||
let mut registry = Self::default();
|
||||
let mut watched = WatchedPaths::default();
|
||||
let Ok(canonical_dir) = fs::canonicalize(dir) else {
|
||||
return registry;
|
||||
return (registry, watched);
|
||||
};
|
||||
if !canonical_dir.is_dir() {
|
||||
return registry;
|
||||
return (registry, watched);
|
||||
}
|
||||
|
||||
let mut visited = HashSet::new();
|
||||
@@ -310,7 +332,14 @@ impl SkillRegistry {
|
||||
registry
|
||||
.skills
|
||||
.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.path.cmp(&b.path)));
|
||||
registry
|
||||
watched.extend(visited.iter().map(|p| (p.clone(), mtime_of(p))));
|
||||
watched.extend(
|
||||
registry
|
||||
.skills
|
||||
.iter()
|
||||
.map(|skill| (skill.path.clone(), mtime_of(&skill.path))),
|
||||
);
|
||||
(registry, watched)
|
||||
}
|
||||
|
||||
fn discover_recursive(
|
||||
@@ -974,9 +1003,33 @@ pub(crate) fn discover_from_directories_with_plugins(
|
||||
dirs: impl IntoIterator<Item = PathBuf>,
|
||||
plugins: Option<&crate::plugins::PluginRegistry>,
|
||||
) -> SkillRegistry {
|
||||
let dirs: Vec<PathBuf> = dirs.into_iter().collect();
|
||||
// The watched-validated cache covers the disk-walk merge. Plugin skills
|
||||
// merge from the in-memory plugin registry per call, so plugin state
|
||||
// changes apply immediately and the cache needs no plugin identity.
|
||||
let merged = cached_merged_discovery(dirs);
|
||||
merge_plugin_skills(merged, plugins)
|
||||
}
|
||||
|
||||
fn merge_plugin_skills(
|
||||
mut merged: SkillRegistry,
|
||||
plugins: Option<&crate::plugins::PluginRegistry>,
|
||||
) -> SkillRegistry {
|
||||
if let Some(plugins) = plugins {
|
||||
merge_active_plugin_skills(&mut merged, plugins);
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
/// Merge every directory's registry with first-match-wins precedence,
|
||||
/// collecting each directory's watched filesystem set for cache validation.
|
||||
fn merge_watched_directories(dirs: Vec<PathBuf>) -> (SkillRegistry, WatchedPaths) {
|
||||
let mut merged = SkillRegistry::default();
|
||||
let mut watched = WatchedPaths::default();
|
||||
for dir in dirs {
|
||||
let registry = SkillRegistry::discover(&dir);
|
||||
watched.push((dir.clone(), mtime_of(&dir)));
|
||||
let (registry, dir_watched) = SkillRegistry::discover_watched(&dir);
|
||||
watched.extend(dir_watched);
|
||||
for skill in registry.skills {
|
||||
if let Some(existing) = merged.skills.iter().find(|s| s.name == skill.name) {
|
||||
merged.push_warning(format!(
|
||||
@@ -993,9 +1046,66 @@ pub(crate) fn discover_from_directories_with_plugins(
|
||||
merged.warnings.push(warning);
|
||||
}
|
||||
}
|
||||
if let Some(plugins) = plugins {
|
||||
merge_active_plugin_skills(&mut merged, plugins);
|
||||
(merged, watched)
|
||||
}
|
||||
|
||||
/// One cached merged discovery: the resolved registry plus the watched
|
||||
/// filesystem entries a hit must re-stat before reuse.
|
||||
struct DiscoveryCacheEntry {
|
||||
watched: WatchedPaths,
|
||||
registry: SkillRegistry,
|
||||
}
|
||||
|
||||
/// Bound the cache so distinct workspaces/modes cannot grow it without
|
||||
/// limit; a full cache is simply cleared on the next miss.
|
||||
const MAX_DISCOVERY_CACHE_ENTRIES: usize = 8;
|
||||
|
||||
fn discovery_cache() -> &'static RwLock<HashMap<Vec<PathBuf>, DiscoveryCacheEntry>> {
|
||||
static CACHE: OnceLock<RwLock<HashMap<Vec<PathBuf>, DiscoveryCacheEntry>>> = OnceLock::new();
|
||||
CACHE.get_or_init(|| RwLock::new(HashMap::new()))
|
||||
}
|
||||
|
||||
/// Drop every cached merged discovery. Called after any skill
|
||||
/// install/uninstall/update so the next build re-walks from disk.
|
||||
pub fn clear_skill_discovery_cache() {
|
||||
discovery_cache()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clear();
|
||||
}
|
||||
|
||||
/// Merged discovery for one resolved directory set, cached by that set.
|
||||
/// A hit re-stats only the watched entries (each visited directory and
|
||||
/// parsed `SKILL.md`); any mtime or readability change re-walks fully.
|
||||
fn cached_merged_discovery(dirs: Vec<PathBuf>) -> SkillRegistry {
|
||||
{
|
||||
let read = discovery_cache()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(entry) = read.get(&dirs) {
|
||||
if entry
|
||||
.watched
|
||||
.iter()
|
||||
.all(|(path, mtime)| mtime_of(path) == *mtime)
|
||||
{
|
||||
return entry.registry.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
let (merged, watched) = merge_watched_directories(dirs.clone());
|
||||
let mut write = discovery_cache()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if write.len() >= MAX_DISCOVERY_CACHE_ENTRIES {
|
||||
write.clear();
|
||||
}
|
||||
write.insert(
|
||||
dirs,
|
||||
DiscoveryCacheEntry {
|
||||
watched,
|
||||
registry: merged.clone(),
|
||||
},
|
||||
);
|
||||
merged
|
||||
}
|
||||
|
||||
|
||||
@@ -1447,3 +1447,141 @@ fn plugin_skills_are_qualified_and_denied_until_trusted_and_enabled() {
|
||||
"a missing authority lock must remove plugin instructions from the prompt catalogue"
|
||||
);
|
||||
}
|
||||
|
||||
// --- #3921 merged discovery cache -----------------------------------------
|
||||
|
||||
fn discovery_delta_since(earlier: super::SkillDiscoveryMetrics) -> super::SkillDiscoveryMetrics {
|
||||
super::discovery_metrics_snapshot().delta_since(earlier)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_discovery_reuses_unchanged_registry_without_rewalking() {
|
||||
super::clear_skill_discovery_cache();
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
let skills_root = tmpdir.path().join("skills");
|
||||
write_skill(&skills_root, "demo", "A demo skill", "Instructions");
|
||||
let dirs = vec![skills_root];
|
||||
|
||||
super::reset_discovery_metrics();
|
||||
let first = super::discover_from_directories_with_plugins(dirs.clone(), None);
|
||||
let walked = discovery_delta_since(super::SkillDiscoveryMetrics::default());
|
||||
let second = super::discover_from_directories_with_plugins(dirs, None);
|
||||
let rewalked = discovery_delta_since(walked);
|
||||
|
||||
assert_eq!(walked.root_discovery_calls, 1);
|
||||
assert_eq!(rewalked, super::SkillDiscoveryMetrics::default());
|
||||
assert_eq!(first.len(), second.len());
|
||||
assert_eq!(first.list()[0].description, second.list()[0].description);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_discovery_picks_up_added_skill_on_next_call() {
|
||||
super::clear_skill_discovery_cache();
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
let skills_root = tmpdir.path().join("skills");
|
||||
write_skill(&skills_root, "demo", "A demo skill", "Instructions");
|
||||
let dirs = vec![skills_root.clone()];
|
||||
|
||||
let first = super::discover_from_directories_with_plugins(dirs.clone(), None);
|
||||
assert_eq!(first.len(), 1);
|
||||
|
||||
write_skill(&skills_root, "added", "A later skill", "More");
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
let second = super::discover_from_directories_with_plugins(dirs, None);
|
||||
assert_eq!(second.len(), 2);
|
||||
assert!(second.get("added").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_discovery_picks_up_skill_content_edits() {
|
||||
super::clear_skill_discovery_cache();
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
let skills_root = tmpdir.path().join("skills");
|
||||
write_skill(&skills_root, "demo", "Original description", "Instructions");
|
||||
let dirs = vec![skills_root.clone()];
|
||||
|
||||
let first = super::discover_from_directories_with_plugins(dirs.clone(), None);
|
||||
assert_eq!(first.list()[0].description, "Original description");
|
||||
|
||||
write_skill(&skills_root, "demo", "Edited description", "Instructions");
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
let second = super::discover_from_directories_with_plugins(dirs, None);
|
||||
assert_eq!(second.list()[0].description, "Edited description");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_discovery_drops_removed_skills() {
|
||||
super::clear_skill_discovery_cache();
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
let skills_root = tmpdir.path().join("skills");
|
||||
write_skill(&skills_root, "keep", "Keep me", "Instructions");
|
||||
write_skill(&skills_root, "drop", "Drop me", "Instructions");
|
||||
let dirs = vec![skills_root.clone()];
|
||||
|
||||
let first = super::discover_from_directories_with_plugins(dirs.clone(), None);
|
||||
assert_eq!(first.len(), 2);
|
||||
|
||||
std::fs::remove_dir_all(skills_root.join("drop")).unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
let second = super::discover_from_directories_with_plugins(dirs, None);
|
||||
assert_eq!(second.len(), 1);
|
||||
assert!(second.get("drop").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_skill_discovery_cache_forces_a_fresh_walk() {
|
||||
super::clear_skill_discovery_cache();
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
let skills_root = tmpdir.path().join("skills");
|
||||
write_skill(&skills_root, "demo", "A demo skill", "Instructions");
|
||||
let dirs = vec![skills_root];
|
||||
|
||||
let _ = super::discover_from_directories_with_plugins(dirs.clone(), None);
|
||||
super::clear_skill_discovery_cache();
|
||||
|
||||
super::reset_discovery_metrics();
|
||||
let _ = super::discover_from_directories_with_plugins(dirs, None);
|
||||
let rewalked = discovery_delta_since(super::SkillDiscoveryMetrics::default());
|
||||
assert_eq!(rewalked.root_discovery_calls, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_and_dir_entry_point_shares_the_same_cache() {
|
||||
let _env_lock = crate::test_support::lock_test_env();
|
||||
super::clear_skill_discovery_cache();
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
let home = tmpdir.path().join("home");
|
||||
let workspace = tmpdir.path().join("workspace");
|
||||
let skills_dir = tmpdir.path().join("configured-skills");
|
||||
std::fs::create_dir_all(&home).unwrap();
|
||||
std::fs::create_dir_all(&workspace).unwrap();
|
||||
write_skill(
|
||||
&skills_dir,
|
||||
"configured",
|
||||
"Configured skill",
|
||||
"Instructions",
|
||||
);
|
||||
let _home = crate::test_support::EnvVarGuard::set("HOME", &home);
|
||||
let _userprofile = crate::test_support::EnvVarGuard::set("USERPROFILE", &home);
|
||||
let _codewhale_home =
|
||||
crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", home.join(".codewhale"));
|
||||
|
||||
super::reset_discovery_metrics();
|
||||
let first = super::discover_for_workspace_and_dir_with_mode(
|
||||
&workspace,
|
||||
&skills_dir,
|
||||
super::SkillDiscoveryMode::Compatible,
|
||||
);
|
||||
let walked = discovery_delta_since(super::SkillDiscoveryMetrics::default());
|
||||
let second = super::discover_for_workspace_and_dir_with_mode(
|
||||
&workspace,
|
||||
&skills_dir,
|
||||
super::SkillDiscoveryMode::Compatible,
|
||||
);
|
||||
let rewalked = discovery_delta_since(walked);
|
||||
|
||||
assert!(walked.root_discovery_calls >= 1);
|
||||
assert_eq!(rewalked, super::SkillDiscoveryMetrics::default());
|
||||
assert_eq!(first.len(), second.len());
|
||||
assert!(second.get("configured").is_some());
|
||||
}
|
||||
|
||||
@@ -2100,6 +2100,7 @@ impl App {
|
||||
}
|
||||
|
||||
pub fn refresh_skill_cache(&mut self) {
|
||||
crate::skills::clear_skill_discovery_cache();
|
||||
let skills_dir = self.skills_dir.clone();
|
||||
let cached_skills = Self::discover_cached_skills(
|
||||
&self.workspace,
|
||||
|
||||
Reference in New Issue
Block a user