feat(spec): gate resource capabilities and CommandIO slots (#609 item 4)

Registry membership only ever said a backend could be *built*. What it
does once mounted — how stale a listing may be, whether reads are cached,
whether a `du` is pushed down to the API — was a second hand-maintained
surface with no gate at all, which is how python served up-to-ten-minute
stale listings of a live postgres schema while typescript pinned 0.

`spec/*/resources.json` now carries two more tables, diffed by
`check_spec_parity.py` with the same one-fact-per-exemption rule the
command checks use:

- `capabilities`: per registry name, `index_ttl` / `caches_reads` /
  `supports_snapshot` / `sizes_always_known`, plus whether the class
  overrides `storage_id` and `statfs`.
- `command_io`: per backend command package, the wired `CommandIO` slots
  plus `local` / `max_glob_matches` / `max_du_entries`.

Python reads both off its classes and dataclasses. TypeScript reads them
from the source declarations (`scripts/resource_facts.ts`), because the
twins are instance fields and observing them at runtime would mean
constructing the resource — and construction is not inert:
`buildResource('github', {})` issues an HTTP request and `postgres` opens
a connection. A value the extractor cannot read as a literal is dumped as
`<expr:…>` rather than guessed.

The gate found 15 live divergences on its first run. Fixed here:

- `readRange` was wired on `disk` alone in typescript while python pushed
  the window down on twelve backends, so `head -c 100` on a large object
  downloaded the whole thing and sliced. Wired on the five whose read
  already takes an `{offset, size}` window (s3, gridfs, nextcloud, hf,
  databricks_volume) via a new `rangeOf` adapter, and `gen-specs` now
  refuses to emit when a reader takes a window with no slot to hand it —
  the twin of python's `test_read_range_optin.py`, which is why python
  was fully wired and typescript was not. The check keys on `offset` *and*
  `size`: postgres pairs `offset` with `limit` to mean SQL rows.
- python github ignored `SCOPE_ERROR` and refused globs at 10001 matches
  instead of 5001; the constant had no importers.

The remaining ten are documented in `parity_exceptions.json`, each naming
one resource and one key: box/dropbox `du`+`find` and the seven readers
with no window argument (T1-I), github `supports_snapshot` (snapshot
redesign), lancedb `caches_reads` (computed per URI), notion/hf `find`
(slot on one side, bespoke command on the other), ssh `append` (wired in
typescript, read by no builder on either side).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
bytecii
2026-08-09 11:14:25 -07:00
parent 8bd1d97069
commit 92a9196ac9
19 changed files with 3827 additions and 42 deletions
@@ -16,6 +16,7 @@ from functools import partial
from mirage.commands.builtin.generic_bind import CommandIO
from mirage.commands.builtin.utils.wrap import stream_from_bytes
from mirage.core.github.constants import SCOPE_ERROR
from mirage.core.github.read import read as _read
from mirage.core.github.readdir import readdir as _readdir
from mirage.core.github.stat import stat as _stat
@@ -33,6 +34,11 @@ IO = CommandIO(
stat=_stat,
is_mounted=lambda a: True,
local=False,
# A glob over a large repo walks the whole tree, so the refusal point
# is lower than the 10000 default every other backend takes. The
# typescript twin has always passed SCOPE_ERROR here; python left the
# constant with no importers and refused only at 10001.
max_glob_matches=SCOPE_ERROR,
)
resolve_glob = IO.resolve_glob
+32
View File
@@ -43,6 +43,38 @@ def test_registry_matches_the_committed_spec_manifest():
"rerun scripts/gen_specs.py")
def test_capabilities_match_the_committed_spec_manifest():
"""Every registry class's capability values must match the dump.
Membership alone says a backend can be built, not how it behaves once
mounted. These four values decide how stale a listing may be, whether
reads are cached, whether a snapshot records a fingerprint and whether
a size is knowable without fetching, and each was an independent hand
edit until the dump started carrying them.
"""
manifest = json.loads(SPEC_RESOURCES.read_text())["capabilities"]
live = {
name: {
"index_ttl": cls.index_ttl,
"caches_reads": cls.caches_reads,
"supports_snapshot": cls.SUPPORTS_SNAPSHOT,
"sizes_always_known": cls.SIZES_ALWAYS_KNOWN,
}
for name, cls in ((n, registry.resolve_class(e.resource_path))
for n, e in REGISTRY.items())
}
dumped = {
name: {
k: v
for k, v in entry.items() if k in live[name]
}
for name, entry in manifest.items()
}
assert live == dumped, ("resource capabilities drifted from "
"spec/python/resources.json; rerun "
"scripts/gen_specs.py")
def test_build_ram_returns_ram_resource():
from mirage.resource.ram import RAMResource
p = build_resource("ram")
+179 -17
View File
@@ -57,7 +57,158 @@ def meta_fields(py_meta: dict[str, Any], ts_meta: dict[str, Any]) -> list[str]:
return sorted((set(py_meta) | set(ts_meta)) - {"by_resource"})
def check_resources(language_only: set[str], expansions: dict[str, list[str]],
def load_resource_trees() -> dict[str, dict[str, Any]]:
"""The three ``resources.json`` payloads, or a SystemExit naming the
generator that has not been run."""
trees = {
"python": SPEC / "python" / "resources.json",
"node": SPEC / "typescript" / "node" / "resources.json",
"browser": SPEC / "typescript" / "browser" / "resources.json",
}
loaded: dict[str, dict[str, Any]] = {}
for tree, path in trees.items():
if not path.is_file():
raise SystemExit(f"missing {path}\nrun scripts/gen_specs.py and "
"typescript/scripts/gen-specs.ts first")
loaded[tree] = json.loads(path.read_text())
return loaded
def merge_variants(loaded: dict[str, dict[str, Any]], key: str,
language_only: set[str]) -> dict[str, Any]:
"""One typescript view of ``key``, node's entry winning over browser's.
A backend registered in both runtimes must describe itself the same
way, and ``compare_variants`` already fails when the two disagree
about a command. Where only one runtime carries a real entry — the
browser registers ``lancedb`` and ``email`` solely to explain that it
cannot serve them, so their capabilities dump as null — the runtime
that can actually mount the backend is the one worth comparing
against python.
Args:
loaded (dict[str, dict[str, Any]]): the three resource trees.
key (str): the payload key to merge, e.g. ``"capabilities"``.
language_only (set[str]): names with no counterpart to compare.
"""
out: dict[str, Any] = {}
for tree in ("node", "browser"):
for name, entry in loaded[tree].get(key, {}).items():
if name in language_only:
continue
if out.get(name) is None:
out[name] = entry
return out
def check_capabilities(loaded: dict[str, dict[str, Any]],
expansions: dict[str,
list[str]], language_only: set[str],
allowed: dict[str,
dict[str,
str]], used: set[str]) -> list[str]:
"""Per-resource behavior values: TTLs, caching, snapshot support.
Registry membership says a backend can be built; these say what it
does once mounted, and they are just as hand-maintained. Python kept
the 600 s ``index_ttl`` default for postgres and mongodb where
typescript pins 0, so a python mount could serve a ten-minute-stale
listing of a live schema while its typescript twin was exact.
Args:
loaded (dict[str, dict[str, Any]]): the three resource trees.
expansions (dict[str, list[str]]): python alias table.
language_only (set[str]): names present in one runtime only.
allowed (dict[str, dict[str, str]]): per-resource keys whose
divergence is documented, each mapped to its reason.
used (set[str]): collects the exemptions that fired.
"""
py: dict[str, Any] = {}
for name, entry in loaded["python"].get("capabilities", {}).items():
for alias in expansions.get(name, [name]):
py[alias] = entry
ts = merge_variants(loaded, "capabilities", language_only)
failures = _membership(py, ts, language_only, "capabilities")
for name in sorted(set(py) & set(ts)):
a, b = py[name], ts[name]
if b is None:
failures.append(f"capabilities[{name}]: python builds it, no "
f"typescript runtime does")
continue
exempt = allowed.get(name, {})
for key in sorted(set(a) | set(b)):
if a.get(key) == b.get(key):
continue
if key in exempt:
used.add(f"{name}:{key}")
continue
failures.append(f"capabilities[{name}].{key}: "
f"python={a.get(key)!r} typescript={b.get(key)!r}")
return failures
def check_command_io(loaded: dict[str, dict[str, Any]], aliases: dict[str,
str],
language_only: set[str], allowed: dict[str, dict[str,
str]],
used: set[str]) -> list[str]:
"""The wired ``CommandIO`` slots per backend.
The adapter's slot set is a hand-filled literal that nothing else
reads, so a backend can omit ``du`` or ``find`` and quietly fall back
to the capped readdir walk — a partial total and an exit 1 past the
cap — while its twin pushes the same query down to the API.
Args:
loaded (dict[str, dict[str, Any]]): the three resource trees.
aliases (dict[str, str]): python command-package name to the
typescript one where the directories differ.
language_only (set[str]): backends present in one runtime only.
allowed (dict[str, dict[str, str]]): per-backend keys whose
divergence is documented, each mapped to its reason.
used (set[str]): collects the exemptions that fired.
"""
py = {
aliases.get(name, name): entry
for name, entry in loaded["python"].get("command_io", {}).items()
}
ts = merge_variants(loaded, "command_io", language_only)
failures = _membership(py, ts, language_only, "command_io")
for name in sorted(set(py) & set(ts)):
a, b = py[name], ts[name]
exempt = allowed.get(name, {})
for key in sorted(set(a) | set(b)):
if a.get(key) == b.get(key):
continue
if key in exempt:
used.add(f"{name}:{key}")
continue
if key == "slots":
only_py = sorted(set(a["slots"]) - set(b["slots"]))
only_ts = sorted(set(b["slots"]) - set(a["slots"]))
failures.append(f"command_io[{name}].slots: "
f"python-only={only_py} "
f"typescript-only={only_ts}")
continue
failures.append(f"command_io[{name}].{key}: "
f"python={a.get(key)!r} typescript={b.get(key)!r}")
return failures
def _membership(py: dict[str, Any], ts: dict[str, Any],
language_only: set[str], label: str) -> list[str]:
only_py = sorted(set(py) - set(ts) - language_only)
only_ts = sorted(set(ts) - set(py) - language_only)
failures: list[str] = []
if only_py:
failures.append(f"{label} only in python: {only_py}")
if only_ts:
failures.append(f"{label} only in typescript: {only_ts}")
return failures
def check_resources(loaded: dict[str, dict[str, Any]], language_only: set[str],
expansions: dict[str, list[str]],
unconstructible: dict[str, dict[str, str]]) -> list[str]:
"""Registry membership, the surface the command specs cannot see.
@@ -69,6 +220,7 @@ def check_resources(language_only: set[str], expansions: dict[str, list[str]],
tables.
Args:
loaded (dict[str, dict[str, Any]]): the three resource trees.
language_only (set[str]): resources that exist in one runtime only.
expansions (dict[str, list[str]]): python alias table, so one
python name can stand for several typescript ones.
@@ -76,20 +228,6 @@ def check_resources(language_only: set[str], expansions: dict[str, list[str]],
register commands on purpose without a registry factory.
"""
failures: list[str] = []
trees = {
"python": SPEC / "python" / "resources.json",
"node": SPEC / "typescript" / "node" / "resources.json",
"browser": SPEC / "typescript" / "browser" / "resources.json",
}
loaded: dict[str, dict[str, list[str]]] = {}
for tree, path in trees.items():
if not path.is_file():
return [
f"missing {path}\nrun scripts/gen_specs.py and "
"typescript/scripts/gen-specs.ts first"
]
loaded[tree] = json.loads(path.read_text())
for tree, payload in loaded.items():
registry = set(payload["registry"])
allowed = unconstructible.get(tree, {})
@@ -287,12 +425,18 @@ def main() -> int:
dict[str,
str]] = exceptions["unconstructible_resources"]
allowed: dict[str, Any] = exceptions["commands"]
capability_exempt: dict[str,
dict[str,
str]] = exceptions["resource_capabilities"]
io_exempt: dict[str, dict[str, str]] = exceptions["command_io"]
io_aliases: dict[str, str] = exceptions["command_io_aliases"]["python"]
py_specs = load_dir(PYTHON)
ts_variants = [load_dir(p) for p in TYPESCRIPT]
failures: list[str] = []
used: set[str] = set()
used_facts: set[str] = set()
# Python has no runtime split, so its command set must equal the union
# of the two typescript variants; comparing against node alone would
@@ -307,9 +451,27 @@ def main() -> int:
if only_ts:
failures.append(f"commands only in typescript: {only_ts}")
trees = load_resource_trees()
failures.extend(compare_variants(ts_variants))
failures.extend(check_resources(language_only, expansions,
unconstructible))
failures.extend(
check_resources(trees, language_only, expansions, unconstructible))
failures.extend(
check_capabilities(trees, expansions, language_only, capability_exempt,
used_facts))
failures.extend(
check_command_io(trees, io_aliases, language_only, io_exempt,
used_facts))
declared = {
f"{name}:{key}"
for table in (capability_exempt, io_exempt)
for name, keys in table.items()
for key in keys
}
stale_facts = sorted(declared - used_facts)
if stale_facts:
failures.append(f"stale resource-fact exemptions in "
f"{EXCEPTIONS.name}, the divergence they cover is "
f"gone: {stale_facts}")
for name in sorted(set(py_specs) & set(ts_variants[0])):
py, ts = py_specs[name], ts_variants[0][name]
+64 -1
View File
@@ -22,15 +22,24 @@ from pathlib import Path
from typing import Any
import mirage.commands.builtin
from mirage.commands.builtin.generic_bind.adapter import CommandIO
from mirage.commands.config import RegisteredCommand
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import CommandSpec, Operand, Option
from mirage.resource.registry import REGISTRY
from mirage.resource.base import BaseResource
from mirage.resource.registry import REGISTRY, resolve_class
logger = logging.getLogger(__name__)
OUT = Path(__file__).resolve().parent.parent / "spec" / "python" / "general"
BUILTIN = Path(mirage.commands.builtin.__file__).resolve(
).parent # type: ignore[arg-type]
# Slots holding a configuration value rather than an operation. Everything
# else on the adapter is a wired operation, reported by name.
IO_VALUE_FIELDS = frozenset({"local", "max_glob_matches", "max_du_entries"})
def _walk_pkg(pkg: Any) -> list[str]:
"""Import every builtin command module, reporting the ones that failed.
@@ -178,6 +187,58 @@ def _emit_one(name: str, spec: Any, rcs: list[RegisteredCommand]) -> None:
json.dumps(payload, indent=2, sort_keys=True, default=_default) + "\n")
def _capabilities() -> dict[str, dict[str, Any]]:
"""Per-resource behavior values, read off the class, never an instance.
Registry membership only says a backend can be built. How it behaves
once mounted is a second hand-maintained surface that drifted just as
quietly: python kept the 600 s ``index_ttl`` default for postgres and
mongodb where typescript pins 0, so an ``ls`` of a live schema could
be ten minutes stale. ``storage_id`` and ``statfs`` are reported as
"does this class override the base" rather than by value, because the
base answers are per-instance identity and UNKNOWN.
"""
out: dict[str, dict[str, Any]] = {}
for name in sorted(REGISTRY):
cls = resolve_class(REGISTRY[name].resource_path)
out[name] = {
"index_ttl": cls.index_ttl,
"caches_reads": cls.caches_reads,
"supports_snapshot": cls.SUPPORTS_SNAPSHOT,
"sizes_always_known": cls.SIZES_ALWAYS_KNOWN,
"storage_id": cls.storage_id is not BaseResource.storage_id,
"statfs": cls.statfs is not BaseResource.statfs,
}
return out
def _command_io() -> dict[str, dict[str, Any]]:
"""The wired ``CommandIO`` slots per backend command package.
The adapter's slot set is a hand-filled literal that no gate reads, so
a backend can omit ``du`` or ``find`` and quietly fall back to the
capped readdir walk while its twin pushes the work down to the API.
Dumping the key set turns that omission into a spec diff.
"""
out: dict[str, dict[str, Any]] = {}
for path in sorted(BUILTIN.glob("*/io.py")):
backend = path.parent.name
mod = importlib.import_module(f"mirage.commands.builtin.{backend}.io")
io = getattr(mod, "IO", None)
if not isinstance(io, CommandIO):
continue
slots = sorted(f.name for f in fields(CommandIO)
if f.name not in IO_VALUE_FIELDS
and getattr(io, f.name) is not None)
out[backend] = {
"slots": slots,
"local": io.local,
"max_glob_matches": io.max_glob_matches,
"max_du_entries": io.max_du_entries,
}
return out
def _emit_resources(registry: dict[str, list[RegisteredCommand]]) -> None:
"""Dump the two resource-name sets the parity gate compares.
@@ -201,6 +262,8 @@ def _emit_resources(registry: dict[str, list[RegisteredCommand]]) -> None:
payload = {
"registry": sorted(REGISTRY),
"command_resources": sorted(command_resources),
"capabilities": _capabilities(),
"command_io": _command_io(),
}
path = OUT.parent / "resources.json"
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+29
View File
@@ -68,6 +68,33 @@ exemption. `python/tests/resource/test_registry.py` and
re-copying the name list, so neither can pin an omission the way the old
hand-written set pinned SharePoint's.
Two more tables record what a backend *does*, not just that it exists.
Membership never said whether a mount serves ten-minute-stale listings or
pushes a `du` down to the API, and both facts were hand-maintained on each
side.
`capabilities` carries, per registry name, `index_ttl`, `caches_reads`,
`supports_snapshot` and `sizes_always_known`, plus whether the class overrides
`storage_id` and `statfs` (booleans, since the base answers are per-instance
identity and UNKNOWN). Python reads them off the class. TypeScript reads them
from the class *declarations* (`scripts/resource_facts.ts`): the twins are
instance fields, so observing them at runtime would mean constructing the
resource, and construction is not inert — `buildResource('github', {})` issues
an HTTP request and `postgres` opens a connection. A value the extractor
cannot read as a literal is dumped verbatim as `<expr:…>` rather than guessed,
so it surfaces as a real mismatch instead of a plausible default. A browser
factory that exists only to explain that the runtime cannot serve the backend
(`lancedb`, `email`) dumps `null`, and the node entry is the one compared
against Python.
`command_io` carries, per backend command package, the wired `CommandIO` slot
names plus `local`, `max_glob_matches` and `max_du_entries`. The adapter's slot
set is a hand-filled literal nothing else reads, so a backend could omit `du`
or `find` and fall back to the capped readdir walk — a partial total and an
exit 1 past the cap — while its twin pushed the same query down to the API.
Where the two languages name the package differently, `command_io_aliases`
maps Python's name onto TypeScript's.
Divergences that are structural rather than bugs live in
`parity_exceptions.json`:
@@ -80,6 +107,8 @@ Divergences that are structural rather than bugs live in
- `commands` — per-command exemptions. `fields` mutes a whole top-level field;
`by_resource` names one resource and one metadata key, so an exemption
cannot quietly hide a second divergence on the same command.
- `resource_capabilities` and `command_io` — one resource and one key per
entry, same rule: an exemption covers the fact it names and nothing else.
The checker fails on a *stale* exception, so an entry cannot outlive the
divergence it documents. An exemption counts as used only when it actually
+63 -17
View File
@@ -1,31 +1,77 @@
{
"resource_expansions": {
"python": {
"hf_buckets": ["hf_buckets", "hf_datasets", "hf_models", "hf_spaces"]
"command_io": {
"box": {
"slots": "typescript pushes du and find down to the Box API while python takes the capped readdir walk (core/box/du/ is written but unwired); python range-reads while typescript's core read has no window. Both halves of T1-I."
},
"dropbox": {
"slots": "same as box: du/find are pushed down in typescript only, and the range read is python only."
},
"gdrive": {
"slots": "python range-reads; the typescript core read has no window argument."
},
"hf": {
"slots": "find is a slot in python and a bespoke command in typescript (see notion) \u2014 the same query, expressed at a different layer."
},
"notion": {
"slots": "find is a slot in python and a bespoke command in typescript (notion/find.ts). Both pass stat=None, so the -mtime behavior matches; only the layer differs."
},
"onedrive": {
"slots": "python range-reads; typescript has no core/onedrive/read.ts to add a window to."
},
"sharepoint": {
"slots": "python range-reads; typescript has no core/sharepoint/read.ts to add a window to."
},
"ssh": {
"slots": "python range-reads over SFTP; typescript's read is whole-file because of the ssh2 256 KiB packet cap. append is wired in typescript and read by no builder on either side \u2014 tee is read-modify-write everywhere \u2014 so the slot is pending a decision to wire it or drop it."
}
},
"language_only_resources": {
"opfs": "Browser-only OPFS backend. Python has no browser runtime, so there is nothing to mirror."
},
"unconstructible_resources": {
"command_io_aliases": {
"python": {
"history": "The /.bash_history view mount is created by the workspace itself and is never named in YAML or a snapshot, so it registers commands without a build_resource entry."
},
"node": {
"history": "See python: the history view mount is workspace-internal."
},
"browser": {
"history": "See python: the history view mount is workspace-internal.",
"databricks_volume": "Registers commands in the browser bundle but has no browser registry factory. Both it and jaeger are pure HTTP, so the omission looks accidental rather than a runtime limit — confirm the intended browser scope before wiring them up.",
"jaeger": "Registers commands in the browser bundle but has no browser registry factory. See databricks_volume."
"hf_buckets": "hf"
}
},
"commands": {
"grep": {
"by_resource": {
"github": ["has_provision"]
"github": [
"has_provision"
]
},
"reason": "TypeScript has no github grep provision. Python's is a bespoke cost model that walks the index to sum matched bytes and read ops, and it reaches IndexCacheStore internals, so porting it needs its own change rather than riding along with the parity gate."
}
},
"language_only_resources": {
"opfs": "Browser-only OPFS backend. Python has no browser runtime, so there is nothing to mirror."
},
"resource_capabilities": {
"github": {
"supports_snapshot": "TypeScript records the blob SHA as a snapshot fingerprint; python stamps the same SHA in core/github/stat.py but does not declare SUPPORTS_SNAPSHOT, so no drift check fires. Left to the snapshot redesign (#721) rather than flipped here, because turning it on changes what a python snapshot records."
},
"lancedb": {
"caches_reads": "TypeScript computes it per mount from the URI scheme (remote tables cache, local ones do not); python has no such branch and never caches. A value, not a table, so there is nothing to single-source."
}
},
"resource_expansions": {
"python": {
"hf_buckets": [
"hf_buckets",
"hf_datasets",
"hf_models",
"hf_spaces"
]
}
},
"unconstructible_resources": {
"browser": {
"databricks_volume": "Registers commands in the browser bundle but has no browser registry factory. Both it and jaeger are pure HTTP, so the omission looks accidental rather than a runtime limit \u2014 confirm the intended browser scope before wiring them up.",
"history": "See python: the history view mount is workspace-internal.",
"jaeger": "Registers commands in the browser bundle but has no browser registry factory. See databricks_volume."
},
"node": {
"history": "See python: the history view mount is workspace-internal."
},
"python": {
"history": "The /.bash_history view mount is created by the workspace itself and is never named in YAML or a snapshot, so it registers commands without a build_resource entry."
}
}
}
File diff suppressed because it is too large Load Diff
+799
View File
@@ -1,4 +1,803 @@
{
"capabilities": {
"aliyun": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"backblaze": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"box": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"ceph": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"chroma": {
"caches_reads": false,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"dify": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"digitalocean": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"discord": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"dropbox": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"email": null,
"gcs": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"gdocs": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"gdrive": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"github": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"github_ci": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"gmail": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"gsheets": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"gslides": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"lancedb": null,
"langfuse": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"linear": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"mem0": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"minio": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"mongodb": {
"caches_reads": false,
"index_ttl": 0,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"notion": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"oci": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"onedrive": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"opfs": {
"caches_reads": false,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"postgres": {
"caches_reads": false,
"index_ttl": 0,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"qdrant": {
"caches_reads": false,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"qingstor": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"r2": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"ram": {
"caches_reads": false,
"index_ttl": 0,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"s3": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"scaleway": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"seaweedfs": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"sharepoint": {
"caches_reads": true,
"index_ttl": 86400,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"slack": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": true,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"supabase": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"tencent": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
},
"trello": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": false
},
"wasabi": {
"caches_reads": true,
"index_ttl": 600,
"sizes_always_known": false,
"statfs": false,
"storage_id": false,
"supports_snapshot": true
}
},
"command_io": {
"box": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"copy",
"create",
"dir_copy",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"truncate",
"unlink",
"write"
]
},
"chroma": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"databricks_volume": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"copy",
"create",
"exists",
"is_mounted",
"mkdir",
"read_bytes",
"read_range",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"unlink",
"write"
]
},
"dify": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"discord": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"dropbox": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"copy",
"create",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"unlink",
"write"
]
},
"gdocs": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"gdrive": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"copy",
"create",
"dir_copy",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"truncate",
"unlink",
"write"
]
},
"github": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 5000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"github_ci": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"gmail": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"gsheets": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"gslides": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"history": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"jaeger": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"lancedb": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"langfuse": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"linear": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"mem0": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"mongodb": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"notion": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"onedrive": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"copy",
"create",
"dir_copy",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"truncate",
"unlink",
"write"
]
},
"opfs": {
"local": true,
"max_du_entries": 10000,
"max_glob_matches": 50000,
"slots": [
"append",
"copy",
"create",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"truncate",
"unlink",
"write"
]
},
"postgres": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"qdrant": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"ram": {
"local": true,
"max_du_entries": 10000,
"max_glob_matches": 50000,
"slots": [
"append",
"copy",
"create",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"set_attrs",
"stat",
"truncate",
"unlink",
"write"
]
},
"s3": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 5000,
"slots": [
"copy",
"create",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_range",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"truncate",
"unlink",
"write"
]
},
"sharepoint": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"copy",
"create",
"dir_copy",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
"read_stream",
"readdir",
"rename",
"rm_r",
"rmdir",
"stat",
"truncate",
"unlink",
"write"
]
},
"slack": {
"local": false,
"max_du_entries": 500,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
},
"trello": {
"local": false,
"max_du_entries": 10000,
"max_glob_matches": 10000,
"slots": [
"is_mounted",
"read_bytes",
"read_stream",
"readdir",
"stat"
]
}
},
"command_resources": [
"box",
"chroma",
File diff suppressed because it is too large Load Diff
@@ -26,11 +26,12 @@ import { stat as dbxStat } from '../../../core/databricks_volume/stat.ts'
import { readStream as dbxStream } from '../../../core/databricks_volume/stream.ts'
import { unlink as dbxUnlink } from '../../../core/databricks_volume/unlink.ts'
import { writeBytes as dbxWrite } from '../../../core/databricks_volume/write.ts'
import type { CommandIO } from '../generic_bind/index.ts'
import { type CommandIO, rangeOf } from '../generic_bind/index.ts'
export const DATABRICKS_VOLUME_IO: CommandIO<DatabricksVolumeAccessor> = {
readdir: dbxReaddir,
readBytes: dbxRead,
readRange: rangeOf(dbxRead),
readStream: dbxStream,
stat: dbxStat,
isMounted: () => true,
@@ -153,6 +153,32 @@ export function resolveGlobOf<A extends Accessor = Accessor>(ops: CommandIO<A>):
return makeResolveGlob(ops.readdir, ops.maxGlobMatches)
}
/**
* A `readRange` slot built from a backend read that already takes a byte
* window as its options argument.
*
* Without the slot the ops factory reads the whole object and slices, so
* `head -c 100` on a 2 GiB S3 key downloads 2 GiB. Python has pushed the
* window down on every one of these backends since the slot existed by
* pointing `read_range` at its own `read_bytes`; this is the same move,
* spelled for a read whose window arrives in an options object.
*
* Args:
* read: the backend's whole-file read, whose fourth argument is an
* `{offset?, size?}` window.
*/
export function rangeOf<A extends Accessor = Accessor>(
read: (
accessor: A,
path: PathSpec,
index: IndexCacheStore | undefined,
options: { offset?: number; size?: number },
) => Promise<Uint8Array>,
): NonNullable<CommandIO<A>['readRange']> {
return (accessor, path, index, offset, size) =>
read(accessor, path, index, size === null ? { offset } : { offset, size })
}
// Whether a path that failed with ENOENT is an implicit directory. Keyed
// backends (RAM/Redis/S3) have no directory entries: stat/read of a prefix
// that only exists through deeper keys raises ENOENT. The operand's own
@@ -19,6 +19,7 @@ export {
type DuOps,
makeResolveGlob,
overlaidStat,
rangeOf,
resolveGlobOf,
} from './adapter.ts'
export { type MakeGenericCommandsOptions, makeGenericCommands } from './factory.ts'
@@ -30,11 +30,12 @@ import { stream as s3Stream } from '../../../core/s3/stream.ts'
import { truncate as s3Truncate } from '../../../core/s3/truncate.ts'
import { unlink as s3Unlink } from '../../../core/s3/unlink.ts'
import { write as s3Write } from '../../../core/s3/write.ts'
import type { CommandIO } from '../generic_bind/index.ts'
import { type CommandIO, rangeOf } from '../generic_bind/index.ts'
export const S3_IO: CommandIO<S3Accessor> = {
readdir: s3Readdir,
readBytes: s3Read,
readRange: rangeOf(s3Read),
readStream: s3Stream,
stat: s3Stat,
isMounted: () => true,
+8 -1
View File
@@ -237,6 +237,7 @@ export {
metadataProvision,
overlaidStat,
pureProvision,
rangeOf,
resolveGlobOf,
withDefaultProvisions,
writeMetadataProvision,
@@ -294,7 +295,13 @@ export {
export { walkFind } from './core/generic/find.ts'
export { statGeneric } from './commands/builtin/generic/stat.ts'
export { diffGeneric } from './commands/builtin/generic/diff.ts'
export { duGeneric, parseDepth, parseDuFlags, runDu } from './commands/builtin/generic/du.ts'
export {
DEFAULT_MAX_DU_ENTRIES,
duGeneric,
parseDepth,
parseDuFlags,
runDu,
} from './commands/builtin/generic/du.ts'
export { treeGeneric } from './commands/builtin/generic/tree.ts'
export { lsGeneric } from './commands/builtin/generic/ls.ts'
export { fileGeneric } from './commands/builtin/generic/file.ts'
@@ -12,7 +12,7 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { CommandIO } from '@struktoai/mirage-core'
import { type CommandIO, rangeOf } from '@struktoai/mirage-core'
import type { GridFSAccessor } from '../../../accessor/gridfs.ts'
import { SCOPE_ERROR } from '../../../core/gridfs/constants.ts'
import { copy as gridfsCopy } from '../../../core/gridfs/copy.ts'
@@ -35,6 +35,7 @@ import { write as gridfsWrite } from '../../../core/gridfs/write.ts'
export const GRIDFS_IO: CommandIO<GridFSAccessor> = {
readdir: gridfsReaddir,
readBytes: gridfsRead,
readRange: rangeOf(gridfsRead),
readStream: gridfsStream,
stat: gridfsStat,
isMounted: () => true,
@@ -12,7 +12,7 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { CommandIO } from '@struktoai/mirage-core'
import { type CommandIO, rangeOf } from '@struktoai/mirage-core'
import type { HfAccessor } from '../../../accessor/hf.ts'
import { SCOPE_ERROR } from '../../../core/hf/constants.ts'
import { create as hfCreate } from '../../../core/hf/create.ts'
@@ -30,6 +30,7 @@ import { write as hfWrite } from '../../../core/hf/write.ts'
export const HF_IO: CommandIO<HfAccessor> = {
readdir: hfReaddir,
readBytes: hfRead,
readRange: rangeOf(hfRead),
readStream: hfStream,
stat: hfStat,
du: { size: hfDu, entries: hfDuAll },
@@ -1,4 +1,4 @@
import type { CommandIO } from '@struktoai/mirage-core'
import { type CommandIO, rangeOf } from '@struktoai/mirage-core'
import type { NextcloudAccessor } from '../../../accessor/nextcloud.ts'
import { SCOPE_ERROR } from '../../../core/nextcloud/constants.ts'
import { copy } from '../../../core/nextcloud/copy.ts'
@@ -25,6 +25,7 @@ export const NEXTCLOUD_IO: CommandIO<NextcloudAccessor> = {
maxGlobMatches: SCOPE_ERROR,
readdir,
readBytes: read,
readRange: rangeOf(read),
readStream: stream,
stat,
isMounted: () => true,
+46 -1
View File
@@ -22,6 +22,15 @@ import * as Node from '@struktoai/mirage-node'
import type { CommandSpec, Operand, Option, RegisteredCommand } from '@struktoai/mirage-core'
import {
type Capabilities,
type CommandIoFacts,
capabilitiesOf,
collectClasses,
commandIoFacts,
registryClasses,
} from './resource_facts.ts'
const { CommandSpec: SpecClass, Operand: OperandClass, Option: OptionClass } = Core
const __dirname = resolve(fileURLToPath(import.meta.url), '..')
@@ -238,10 +247,19 @@ function sortedStringify(value: unknown): string {
// command. A name in the second but not the first registers commands yet
// cannot be mounted by name, which is how chroma/dify/lancedb/qdrant stayed
// unconstructible in typescript while appearing in every command's `_meta`.
//
// `capabilities` and `command_io` carry the values behind those names.
// Registry membership only says a backend can be built; how it behaves is
// a second hand-maintained surface that drifted just as quietly — Python
// served ten-minute-stale listings of a live postgres schema because its
// `index_ttl` kept the 600 s default where typescript pinned 0, and box's
// `du` slot is wired on one side and absent on the other.
function emitResources(
name: string,
knownResources: string[],
registry: Record<string, RegisteredCommand[]>,
capabilities: Record<string, Capabilities | null>,
commandIo: Record<string, CommandIoFacts>,
): void {
const commandResources = new Set<string>()
for (const rcs of Object.values(registry)) {
@@ -250,12 +268,30 @@ function emitResources(
const payload = {
registry: [...knownResources].sort(),
command_resources: [...commandResources].sort(),
capabilities,
command_io: commandIo,
}
const path = resolve(SPEC_ROOT, name, 'resources.json')
writeFileSync(path, sortedStringify(payload) + '\n')
console.log(`emitted ${payload.registry.length} registry names to ${path}`)
}
// Every registry name's capability values, read from the class the entry
// constructs. A name whose class cannot be resolved is a hard error: a
// missing row would read as "no divergence here" in the parity gate.
function capabilitiesFor(
pkgs: readonly string[],
variantPkg: string,
): Record<string, Capabilities | null> {
const classes = collectClasses(PACKAGES, pkgs)
const names = registryClasses(resolve(PACKAGES, variantPkg, 'src', 'resource', 'registry.ts'))
const out: Record<string, Capabilities | null> = {}
for (const [resource, className] of [...names].sort(([a], [b]) => a.localeCompare(b))) {
out[resource] = className === null ? null : capabilitiesOf(className, classes)
}
return out
}
function emitVariant(
name: string,
pkgs: readonly string[],
@@ -274,7 +310,16 @@ function emitVariant(
writeFileSync(resolve(outDir, `${cmd}.json`), sortedStringify(payload) + '\n')
}
console.log(`emitted ${cmdNames.length} specs to ${outDir}`)
emitResources(name, knownResources, registry)
emitResources(
name,
knownResources,
registry,
capabilitiesFor(pkgs, pkgs[pkgs.length - 1] as string),
commandIoFacts(PACKAGES, pkgs, {
maxGlobMatches: Core.DEFAULT_MAX_GLOB_MATCHES,
maxDuEntries: Core.DEFAULT_MAX_DU_ENTRIES,
}),
)
}
function main(): void {
+503
View File
@@ -0,0 +1,503 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { existsSync, readdirSync } from 'node:fs'
import { resolve } from 'node:path'
import ts from 'typescript'
// Capability values and CommandIO slots are read from the source rather
// than from a live object on purpose. Python can introspect its resource
// classes because the values are class attributes, but the typescript
// twins are instance fields, so the only way to observe them at runtime
// is to construct the resource — and construction is not inert here:
// `buildResource('github', {})` issues an HTTP request and `postgres`
// opens a connection. A generator that reaches the network produces a
// different spec depending on who runs it, so the values come from the
// declarations instead.
const CAPABILITY_FIELDS = [
'indexTtl',
'cachesReads',
'supportsSnapshot',
'sizesAlwaysKnown',
] as const
// Slots that carry a configuration value rather than an operation. They
// are reported as values; every other key of the literal is a wired slot.
const IO_VALUE_FIELDS = new Set(['local', 'maxGlobMatches', 'maxDuEntries'])
const BASE_CLASS = 'BaseResource'
export interface Capabilities {
index_ttl: number | string
caches_reads: boolean | string
supports_snapshot: boolean | string
sizes_always_known: boolean | string
storage_id: boolean
statfs: boolean
}
export interface CommandIoFacts {
slots: string[]
local: boolean
max_glob_matches: number | null
max_du_entries: number | null
}
interface ClassInfo {
decl: ts.ClassDeclaration
source: ts.SourceFile
parent: string | undefined
}
function snake(name: string): string {
return name.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)
}
function parse(file: string): ts.SourceFile {
return ts.createSourceFile(file, ts.sys.readFile(file) ?? '', ts.ScriptTarget.ESNext, true)
}
function sourceFiles(dir: string): string[] {
if (!existsSync(dir)) return []
const out: string[] = []
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = resolve(dir, entry.name)
if (entry.isDirectory()) out.push(...sourceFiles(path))
else if (entry.name.endsWith('.ts') && !entry.name.endsWith('.test.ts')) out.push(path)
}
return out
}
// The literal a capability field is initialized with. Anything computed
// is reported verbatim as `<expr:Kind>` so the parity gate shows a real
// mismatch instead of a plausible-looking default: a value this cannot
// read is a value it must not guess.
function literalValue(node: ts.Expression | undefined): number | boolean | string {
if (node === undefined) return '<declared, no initializer>'
if (ts.isNumericLiteral(node)) return Number(node.text.replaceAll('_', ''))
if (node.kind === ts.SyntaxKind.TrueKeyword) return true
if (node.kind === ts.SyntaxKind.FalseKeyword) return false
if (ts.isPrefixUnaryExpression(node) && ts.isNumericLiteral(node.operand)) {
const value = Number(node.operand.text.replaceAll('_', ''))
return node.operator === ts.SyntaxKind.MinusToken ? -value : value
}
return `<expr:${ts.SyntaxKind[node.kind]}>`
}
function heritageName(decl: ts.ClassDeclaration): string | undefined {
for (const clause of decl.heritageClauses ?? []) {
if (clause.token !== ts.SyntaxKind.ExtendsKeyword) continue
const expr = clause.types[0]?.expression
if (expr !== undefined && ts.isIdentifier(expr)) return expr.text
}
return undefined
}
/**
* Every resource class a variant can reach, keyed by class name.
*
* Duplicate names across the scanned packages would make the extends walk
* ambiguous, so they are refused rather than resolved by import order.
*/
export function collectClasses(
packagesRoot: string,
pkgs: readonly string[],
): Map<string, ClassInfo> {
const out = new Map<string, ClassInfo>()
for (const pkg of pkgs) {
for (const file of sourceFiles(resolve(packagesRoot, pkg, 'src', 'resource'))) {
const source = parse(file)
ts.forEachChild(source, (node) => {
if (!ts.isClassDeclaration(node) || node.name === undefined) return
const name = node.name.text
const seen = out.get(name)
if (seen !== undefined) {
throw new Error(
`two resource classes named ${name}: ${seen.source.fileName} and ${file}; ` +
`the capability walk cannot tell which one a registry entry means`,
)
}
out.set(name, { decl: node, source, parent: heritageName(node) })
})
}
}
return out
}
function chain(className: string, classes: Map<string, ClassInfo>): ClassInfo[] {
const out: ClassInfo[] = []
const seen = new Set<string>()
let name: string | undefined = className
while (name !== undefined && !seen.has(name)) {
seen.add(name)
const info: ClassInfo | undefined = classes.get(name)
if (info === undefined) break
out.push(info)
name = info.parent
}
return out
}
function declaresMethod(info: ClassInfo, name: string): boolean {
return info.decl.members.some(
(m) =>
(ts.isMethodDeclaration(m) || ts.isPropertyDeclaration(m)) &&
m.name.getText(info.source) === name,
)
}
/**
* One class's capability values, resolved up its extends chain.
*
* The three boolean capabilities are optional members of the `Resource`
* interface with no `BaseResource` declaration, and every reader coerces
* with `=== true` (`resource/base.ts`), so a class that declares none of
* them is false — not undefined. `indexTtl` does have a `BaseResource`
* default and is picked up by the same walk.
*
* Args:
* className: the class the registry constructs.
* classes: every reachable resource class, from `collectClasses`.
*/
export function capabilitiesOf(className: string, classes: Map<string, ClassInfo>): Capabilities {
const ancestry = chain(className, classes)
if (ancestry.length === 0)
throw new Error(`no source declaration for resource class ${className}`)
const values: Record<string, number | boolean | string> = {}
for (const info of ancestry) {
for (const member of info.decl.members) {
if (!ts.isPropertyDeclaration(member)) continue
const name = member.name.getText(info.source)
if (!(CAPABILITY_FIELDS as readonly string[]).includes(name)) continue
if (name in values) continue
values[name] = literalValue(member.initializer)
}
}
const overrides = ancestry.filter((info) => info.decl.name?.text !== BASE_CLASS)
return {
index_ttl: values.indexTtl ?? 600,
caches_reads: values.cachesReads ?? false,
supports_snapshot: values.supportsSnapshot ?? false,
sizes_always_known: values.sizesAlwaysKnown ?? false,
storage_id: overrides.some((info) => declaresMethod(info, 'storageId')),
statfs: overrides.some((info) => declaresMethod(info, 'statfs')),
}
}
// The value of a named constant a slot was set to, followed one import
// hop. `maxGlobMatches: SCOPE_ERROR` is the whole reason this exists:
// reporting the name instead of 5000 would make the two languages differ
// on a value they agree about.
function resolveIdentifier(
source: ts.SourceFile,
name: string,
): number | boolean | string | undefined {
for (const statement of source.statements) {
if (ts.isVariableStatement(statement)) {
for (const decl of statement.declarationList.declarations) {
if (ts.isIdentifier(decl.name) && decl.name.text === name)
return literalValue(decl.initializer)
}
}
if (!ts.isImportDeclaration(statement)) continue
const bindings = statement.importClause?.namedBindings
if (bindings === undefined || !ts.isNamedImports(bindings)) continue
if (!bindings.elements.some((el) => el.name.text === name)) continue
const specifier = (statement.moduleSpecifier as ts.StringLiteral).text
if (!specifier.startsWith('.')) continue
const target = resolve(source.fileName, '..', specifier)
if (!existsSync(target)) continue
for (const inner of parse(target).statements) {
if (!ts.isVariableStatement(inner)) continue
for (const decl of inner.declarationList.declarations) {
if (ts.isIdentifier(decl.name) && decl.name.text === name)
return literalValue(decl.initializer)
}
}
}
return undefined
}
// The file a local name was imported from, and the name it has there.
// `import { read as s3Read }` binds `s3Read` locally to an exported
// `read`, so both halves are needed to find the declaration.
function importedFrom(
source: ts.SourceFile,
local: string,
): { file: string; exported: string } | undefined {
for (const statement of source.statements) {
if (!ts.isImportDeclaration(statement)) continue
const bindings = statement.importClause?.namedBindings
if (bindings === undefined || !ts.isNamedImports(bindings)) continue
for (const element of bindings.elements) {
if (element.name.text !== local) continue
const specifier = (statement.moduleSpecifier as ts.StringLiteral).text
if (!specifier.startsWith('.')) return undefined
const file = resolve(source.fileName, '..', specifier)
if (!existsSync(file)) return undefined
return { file, exported: (element.propertyName ?? element.name).text }
}
}
return undefined
}
// Whether a backend's whole-file read declares a fourth parameter, i.e.
// the `{offset, size}` window the `readRange` slot exists to hand it.
// Parameter count rather than arity, because optional and defaulted
// parameters do not show up in `Function.length` — `read(a, b, c?, opts =
// {})` reports 2 at runtime, so nothing observable at runtime can answer
// this question.
function takesWindow(source: ts.SourceFile, local: string): boolean {
const origin = importedFrom(source, local)
if (origin === undefined) return false
const declared = parse(origin.file)
for (const statement of declared.statements) {
if (!ts.isFunctionDeclaration(statement) || statement.name === undefined) continue
if (statement.name.text !== origin.exported) continue
const options = statement.parameters[3]?.type
if (options === undefined) return false
// The fourth parameter is not automatically a byte window — linear's
// is a `ReadFilter` of query terms — so the type has to declare an
// `offset` before this counts as a range the slot could carry.
if (ts.isTypeLiteralNode(options)) return declaresOffset(options.members)
if (!ts.isTypeReferenceNode(options) || !ts.isIdentifier(options.typeName)) return false
return declaresOffsetNamed(declared, options.typeName.text)
}
return false
}
// A byte window is `offset` *and* `size`, the pair python's own opt-in
// test keys on. `offset` alone is not enough: postgres pairs it with
// `limit` to mean a SQL row range, which no byte slot can carry.
function declaresOffset(members: ts.NodeArray<ts.TypeElement>): boolean {
const names = new Set(members.filter((m) => m.name !== undefined).map((m) => m.name?.getText()))
return names.has('offset') && names.has('size')
}
function declaresOffsetNamed(source: ts.SourceFile, name: string): boolean {
for (const statement of source.statements) {
if (ts.isInterfaceDeclaration(statement) && statement.name.text === name) {
return declaresOffset(statement.members)
}
}
const origin = importedFrom(source, name)
if (origin === undefined) return false
for (const statement of parse(origin.file).statements) {
if (ts.isInterfaceDeclaration(statement) && statement.name.text === origin.exported) {
return declaresOffset(statement.members)
}
}
return false
}
// Whether a registry factory exists only to explain that this runtime
// cannot serve the backend: it throws, or hands back a rejected promise,
// without constructing anything.
function refuses(node: ts.Node): boolean {
let found = false
const scan = (child: ts.Node): void => {
if (ts.isThrowStatement(child)) found = true
if (
ts.isPropertyAccessExpression(child) &&
child.name.text === 'reject' &&
ts.isIdentifier(child.expression) &&
child.expression.text === 'Promise'
) {
found = true
}
ts.forEachChild(child, scan)
}
scan(node)
return found
}
/**
* Registry name to the class its factory constructs.
*
* Read from `registry.ts` rather than guessed from directory names: the
* S3-compatible entries and the HuggingFace variants each map several
* names onto classes whose directories do not match, and a guess that
* lands on the wrong class would report capabilities for a backend the
* user never mounts.
*
* Args:
* registryFile: absolute path to the variant's `resource/registry.ts`.
*/
export function registryClasses(registryFile: string): Map<string, string | null> {
const source = parse(registryFile)
const out = new Map<string, string | null>()
const visit = (node: ts.Node): void => {
if (
ts.isVariableDeclaration(node) &&
ts.isIdentifier(node.name) &&
node.name.text === 'REGISTRY' &&
node.initializer !== undefined &&
ts.isObjectLiteralExpression(node.initializer)
) {
for (const prop of node.initializer.properties) {
if (!ts.isPropertyAssignment(prop)) continue
const name = prop.name.getText(source).replace(/^['"]|['"]$/g, '')
const classes = new Set<string>()
const scan = (child: ts.Node): void => {
if (ts.isNewExpression(child) && ts.isIdentifier(child.expression)) {
classes.add(child.expression.text)
}
// `GitHubResource.create(...)` and `DatabricksVolumeResource
// .create(...)` are async static factories, so the class never
// appears under `new`.
if (
ts.isPropertyAccessExpression(child) &&
child.name.text === 'create' &&
ts.isIdentifier(child.expression) &&
child.expression.text.endsWith('Resource')
) {
classes.add(child.expression.text)
}
ts.forEachChild(child, scan)
}
scan(prop.initializer)
const resourceClasses = [...classes].filter((c) => c.endsWith('Resource'))
if (resourceClasses.length === 0 && refuses(prop.initializer)) {
// Registered so the name resolves and the error explains why,
// but there is no class to read capabilities from — the browser
// does this for lancedb (native addon) and email (raw TCP).
out.set(name, null)
continue
}
if (resourceClasses.length !== 1) {
throw new Error(
`registry entry ${name} constructs ${resourceClasses.length} resource classes ` +
`(${resourceClasses.join(', ') || 'none'}); the capability dump needs exactly one`,
)
}
out.set(name, resourceClasses[0] as string)
}
}
ts.forEachChild(node, visit)
}
visit(source)
if (out.size === 0) throw new Error(`no REGISTRY object literal found in ${registryFile}`)
return out
}
/**
* The wired `CommandIO` slots per backend command directory.
*
* The adapter's slot set is a hand-filled literal that nothing reads, so
* a backend can omit `du` or `find` and quietly fall back to the capped
* readdir walk while its twin pushes the work down. Dumping the key set
* makes that omission a spec diff.
*
* Args:
* packagesRoot: the `typescript/packages` directory.
* pkgs: package names to scan, in the variant's resolution order.
*/
export function commandIoFacts(
packagesRoot: string,
pkgs: readonly string[],
defaults: { maxGlobMatches: number; maxDuEntries: number },
): Record<string, CommandIoFacts> {
const out: Record<string, CommandIoFacts> = {}
for (const pkg of pkgs) {
const root = resolve(packagesRoot, pkg, 'src', 'commands', 'builtin')
if (!existsSync(root)) continue
for (const entry of readdirSync(root, { withFileTypes: true })) {
if (!entry.isDirectory()) continue
const file = resolve(root, entry.name, 'io.ts')
if (!existsSync(file)) continue
const source = parse(file)
let literal: ts.ObjectLiteralExpression | undefined
const visit = (node: ts.Node): void => {
if (
ts.isVariableDeclaration(node) &&
ts.isIdentifier(node.name) &&
node.name.text.endsWith('_IO') &&
node.initializer !== undefined &&
ts.isObjectLiteralExpression(node.initializer)
) {
if (literal !== undefined) {
throw new Error(`${file} declares more than one *_IO object literal`)
}
literal = node.initializer
}
ts.forEachChild(node, visit)
}
visit(source)
if (literal === undefined) continue
const slots: string[] = []
const values: Record<string, number | boolean | string> = {}
let readBytes: string | undefined
for (const prop of literal.properties) {
if (ts.isSpreadAssignment(prop)) {
throw new Error(
`${file} spreads into its *_IO literal; the slot dump cannot see through it`,
)
}
const name = prop.name?.getText(source)
if (name === undefined) continue
if (IO_VALUE_FIELDS.has(name)) {
let value = ts.isPropertyAssignment(prop) ? literalValue(prop.initializer) : true
if (
ts.isPropertyAssignment(prop) &&
ts.isIdentifier(prop.initializer) &&
typeof value === 'string'
) {
value = resolveIdentifier(source, prop.initializer.text) ?? value
}
values[name] = value
continue
}
if (
name === 'readBytes' &&
ts.isPropertyAssignment(prop) &&
ts.isIdentifier(prop.initializer)
) {
readBytes = prop.initializer.text
}
slots.push(snake(name))
}
// A reader that already takes a window but no `readRange` slot is
// the silent case: the ops factory reads the whole object and
// slices, which is correct, quiet, and throws the pushdown away.
// Python asserts the same rule from its own signatures
// (tests/commands/test_read_range_optin.py); without this the two
// sides can only diverge, never be caught.
if (
!slots.includes('read_range') &&
readBytes !== undefined &&
takesWindow(source, readBytes)
) {
throw new Error(
`${file}: readBytes takes a byte window but no readRange slot is wired, ` +
`so every ranged read downloads the whole object and slices — ` +
`add \`readRange: rangeOf(${readBytes})\``,
)
}
out[entry.name] = {
slots: slots.sort(),
local: values.local === undefined ? true : values.local === true,
max_glob_matches: numeric(values.maxGlobMatches, defaults.maxGlobMatches),
max_du_entries: numeric(values.maxDuEntries, defaults.maxDuEntries),
}
}
}
return out
}
function numeric(value: number | boolean | string | undefined, fallback: number): number | null {
if (value === undefined) return fallback
return typeof value === 'number' ? value : null
}