feat(spec): gate resource capabilities and CommandIO slots (#609 item 4)
Registry membership only ever said a backend could be *built*. What it
does once mounted — how stale a listing may be, whether reads are cached,
whether a `du` is pushed down to the API — was a second hand-maintained
surface with no gate at all, which is how python served up-to-ten-minute
stale listings of a live postgres schema while typescript pinned 0.
`spec/*/resources.json` now carries two more tables, diffed by
`check_spec_parity.py` with the same one-fact-per-exemption rule the
command checks use:
- `capabilities`: per registry name, `index_ttl` / `caches_reads` /
`supports_snapshot` / `sizes_always_known`, plus whether the class
overrides `storage_id` and `statfs`.
- `command_io`: per backend command package, the wired `CommandIO` slots
plus `local` / `max_glob_matches` / `max_du_entries`.
Python reads both off its classes and dataclasses. TypeScript reads them
from the source declarations (`scripts/resource_facts.ts`), because the
twins are instance fields and observing them at runtime would mean
constructing the resource — and construction is not inert:
`buildResource('github', {})` issues an HTTP request and `postgres` opens
a connection. A value the extractor cannot read as a literal is dumped as
`<expr:…>` rather than guessed.
The gate found 15 live divergences on its first run. Fixed here:
- `readRange` was wired on `disk` alone in typescript while python pushed
the window down on twelve backends, so `head -c 100` on a large object
downloaded the whole thing and sliced. Wired on the five whose read
already takes an `{offset, size}` window (s3, gridfs, nextcloud, hf,
databricks_volume) via a new `rangeOf` adapter, and `gen-specs` now
refuses to emit when a reader takes a window with no slot to hand it —
the twin of python's `test_read_range_optin.py`, which is why python
was fully wired and typescript was not. The check keys on `offset` *and*
`size`: postgres pairs `offset` with `limit` to mean SQL rows.
- python github ignored `SCOPE_ERROR` and refused globs at 10001 matches
instead of 5001; the constant had no importers.
The remaining ten are documented in `parity_exceptions.json`, each naming
one resource and one key: box/dropbox `du`+`find` and the seven readers
with no window argument (T1-I), github `supports_snapshot` (snapshot
redesign), lancedb `caches_reads` (computed per URI), notion/hf `find`
(slot on one side, bespoke command on the other), ssh `append` (wired in
typescript, read by no builder on either side).
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from functools import partial
|
||||
|
||||
from mirage.commands.builtin.generic_bind import CommandIO
|
||||
from mirage.commands.builtin.utils.wrap import stream_from_bytes
|
||||
from mirage.core.github.constants import SCOPE_ERROR
|
||||
from mirage.core.github.read import read as _read
|
||||
from mirage.core.github.readdir import readdir as _readdir
|
||||
from mirage.core.github.stat import stat as _stat
|
||||
@@ -33,6 +34,11 @@ IO = CommandIO(
|
||||
stat=_stat,
|
||||
is_mounted=lambda a: True,
|
||||
local=False,
|
||||
# A glob over a large repo walks the whole tree, so the refusal point
|
||||
# is lower than the 10000 default every other backend takes. The
|
||||
# typescript twin has always passed SCOPE_ERROR here; python left the
|
||||
# constant with no importers and refused only at 10001.
|
||||
max_glob_matches=SCOPE_ERROR,
|
||||
)
|
||||
|
||||
resolve_glob = IO.resolve_glob
|
||||
|
||||
@@ -43,6 +43,38 @@ def test_registry_matches_the_committed_spec_manifest():
|
||||
"rerun scripts/gen_specs.py")
|
||||
|
||||
|
||||
def test_capabilities_match_the_committed_spec_manifest():
|
||||
"""Every registry class's capability values must match the dump.
|
||||
|
||||
Membership alone says a backend can be built, not how it behaves once
|
||||
mounted. These four values decide how stale a listing may be, whether
|
||||
reads are cached, whether a snapshot records a fingerprint and whether
|
||||
a size is knowable without fetching, and each was an independent hand
|
||||
edit until the dump started carrying them.
|
||||
"""
|
||||
manifest = json.loads(SPEC_RESOURCES.read_text())["capabilities"]
|
||||
live = {
|
||||
name: {
|
||||
"index_ttl": cls.index_ttl,
|
||||
"caches_reads": cls.caches_reads,
|
||||
"supports_snapshot": cls.SUPPORTS_SNAPSHOT,
|
||||
"sizes_always_known": cls.SIZES_ALWAYS_KNOWN,
|
||||
}
|
||||
for name, cls in ((n, registry.resolve_class(e.resource_path))
|
||||
for n, e in REGISTRY.items())
|
||||
}
|
||||
dumped = {
|
||||
name: {
|
||||
k: v
|
||||
for k, v in entry.items() if k in live[name]
|
||||
}
|
||||
for name, entry in manifest.items()
|
||||
}
|
||||
assert live == dumped, ("resource capabilities drifted from "
|
||||
"spec/python/resources.json; rerun "
|
||||
"scripts/gen_specs.py")
|
||||
|
||||
|
||||
def test_build_ram_returns_ram_resource():
|
||||
from mirage.resource.ram import RAMResource
|
||||
p = build_resource("ram")
|
||||
|
||||
+179
-17
@@ -57,7 +57,158 @@ def meta_fields(py_meta: dict[str, Any], ts_meta: dict[str, Any]) -> list[str]:
|
||||
return sorted((set(py_meta) | set(ts_meta)) - {"by_resource"})
|
||||
|
||||
|
||||
def check_resources(language_only: set[str], expansions: dict[str, list[str]],
|
||||
def load_resource_trees() -> dict[str, dict[str, Any]]:
|
||||
"""The three ``resources.json`` payloads, or a SystemExit naming the
|
||||
generator that has not been run."""
|
||||
trees = {
|
||||
"python": SPEC / "python" / "resources.json",
|
||||
"node": SPEC / "typescript" / "node" / "resources.json",
|
||||
"browser": SPEC / "typescript" / "browser" / "resources.json",
|
||||
}
|
||||
loaded: dict[str, dict[str, Any]] = {}
|
||||
for tree, path in trees.items():
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing {path}\nrun scripts/gen_specs.py and "
|
||||
"typescript/scripts/gen-specs.ts first")
|
||||
loaded[tree] = json.loads(path.read_text())
|
||||
return loaded
|
||||
|
||||
|
||||
def merge_variants(loaded: dict[str, dict[str, Any]], key: str,
|
||||
language_only: set[str]) -> dict[str, Any]:
|
||||
"""One typescript view of ``key``, node's entry winning over browser's.
|
||||
|
||||
A backend registered in both runtimes must describe itself the same
|
||||
way, and ``compare_variants`` already fails when the two disagree
|
||||
about a command. Where only one runtime carries a real entry — the
|
||||
browser registers ``lancedb`` and ``email`` solely to explain that it
|
||||
cannot serve them, so their capabilities dump as null — the runtime
|
||||
that can actually mount the backend is the one worth comparing
|
||||
against python.
|
||||
|
||||
Args:
|
||||
loaded (dict[str, dict[str, Any]]): the three resource trees.
|
||||
key (str): the payload key to merge, e.g. ``"capabilities"``.
|
||||
language_only (set[str]): names with no counterpart to compare.
|
||||
"""
|
||||
out: dict[str, Any] = {}
|
||||
for tree in ("node", "browser"):
|
||||
for name, entry in loaded[tree].get(key, {}).items():
|
||||
if name in language_only:
|
||||
continue
|
||||
if out.get(name) is None:
|
||||
out[name] = entry
|
||||
return out
|
||||
|
||||
|
||||
def check_capabilities(loaded: dict[str, dict[str, Any]],
|
||||
expansions: dict[str,
|
||||
list[str]], language_only: set[str],
|
||||
allowed: dict[str,
|
||||
dict[str,
|
||||
str]], used: set[str]) -> list[str]:
|
||||
"""Per-resource behavior values: TTLs, caching, snapshot support.
|
||||
|
||||
Registry membership says a backend can be built; these say what it
|
||||
does once mounted, and they are just as hand-maintained. Python kept
|
||||
the 600 s ``index_ttl`` default for postgres and mongodb where
|
||||
typescript pins 0, so a python mount could serve a ten-minute-stale
|
||||
listing of a live schema while its typescript twin was exact.
|
||||
|
||||
Args:
|
||||
loaded (dict[str, dict[str, Any]]): the three resource trees.
|
||||
expansions (dict[str, list[str]]): python alias table.
|
||||
language_only (set[str]): names present in one runtime only.
|
||||
allowed (dict[str, dict[str, str]]): per-resource keys whose
|
||||
divergence is documented, each mapped to its reason.
|
||||
used (set[str]): collects the exemptions that fired.
|
||||
"""
|
||||
py: dict[str, Any] = {}
|
||||
for name, entry in loaded["python"].get("capabilities", {}).items():
|
||||
for alias in expansions.get(name, [name]):
|
||||
py[alias] = entry
|
||||
ts = merge_variants(loaded, "capabilities", language_only)
|
||||
failures = _membership(py, ts, language_only, "capabilities")
|
||||
for name in sorted(set(py) & set(ts)):
|
||||
a, b = py[name], ts[name]
|
||||
if b is None:
|
||||
failures.append(f"capabilities[{name}]: python builds it, no "
|
||||
f"typescript runtime does")
|
||||
continue
|
||||
exempt = allowed.get(name, {})
|
||||
for key in sorted(set(a) | set(b)):
|
||||
if a.get(key) == b.get(key):
|
||||
continue
|
||||
if key in exempt:
|
||||
used.add(f"{name}:{key}")
|
||||
continue
|
||||
failures.append(f"capabilities[{name}].{key}: "
|
||||
f"python={a.get(key)!r} typescript={b.get(key)!r}")
|
||||
return failures
|
||||
|
||||
|
||||
def check_command_io(loaded: dict[str, dict[str, Any]], aliases: dict[str,
|
||||
str],
|
||||
language_only: set[str], allowed: dict[str, dict[str,
|
||||
str]],
|
||||
used: set[str]) -> list[str]:
|
||||
"""The wired ``CommandIO`` slots per backend.
|
||||
|
||||
The adapter's slot set is a hand-filled literal that nothing else
|
||||
reads, so a backend can omit ``du`` or ``find`` and quietly fall back
|
||||
to the capped readdir walk — a partial total and an exit 1 past the
|
||||
cap — while its twin pushes the same query down to the API.
|
||||
|
||||
Args:
|
||||
loaded (dict[str, dict[str, Any]]): the three resource trees.
|
||||
aliases (dict[str, str]): python command-package name to the
|
||||
typescript one where the directories differ.
|
||||
language_only (set[str]): backends present in one runtime only.
|
||||
allowed (dict[str, dict[str, str]]): per-backend keys whose
|
||||
divergence is documented, each mapped to its reason.
|
||||
used (set[str]): collects the exemptions that fired.
|
||||
"""
|
||||
py = {
|
||||
aliases.get(name, name): entry
|
||||
for name, entry in loaded["python"].get("command_io", {}).items()
|
||||
}
|
||||
ts = merge_variants(loaded, "command_io", language_only)
|
||||
failures = _membership(py, ts, language_only, "command_io")
|
||||
for name in sorted(set(py) & set(ts)):
|
||||
a, b = py[name], ts[name]
|
||||
exempt = allowed.get(name, {})
|
||||
for key in sorted(set(a) | set(b)):
|
||||
if a.get(key) == b.get(key):
|
||||
continue
|
||||
if key in exempt:
|
||||
used.add(f"{name}:{key}")
|
||||
continue
|
||||
if key == "slots":
|
||||
only_py = sorted(set(a["slots"]) - set(b["slots"]))
|
||||
only_ts = sorted(set(b["slots"]) - set(a["slots"]))
|
||||
failures.append(f"command_io[{name}].slots: "
|
||||
f"python-only={only_py} "
|
||||
f"typescript-only={only_ts}")
|
||||
continue
|
||||
failures.append(f"command_io[{name}].{key}: "
|
||||
f"python={a.get(key)!r} typescript={b.get(key)!r}")
|
||||
return failures
|
||||
|
||||
|
||||
def _membership(py: dict[str, Any], ts: dict[str, Any],
|
||||
language_only: set[str], label: str) -> list[str]:
|
||||
only_py = sorted(set(py) - set(ts) - language_only)
|
||||
only_ts = sorted(set(ts) - set(py) - language_only)
|
||||
failures: list[str] = []
|
||||
if only_py:
|
||||
failures.append(f"{label} only in python: {only_py}")
|
||||
if only_ts:
|
||||
failures.append(f"{label} only in typescript: {only_ts}")
|
||||
return failures
|
||||
|
||||
|
||||
def check_resources(loaded: dict[str, dict[str, Any]], language_only: set[str],
|
||||
expansions: dict[str, list[str]],
|
||||
unconstructible: dict[str, dict[str, str]]) -> list[str]:
|
||||
"""Registry membership, the surface the command specs cannot see.
|
||||
|
||||
@@ -69,6 +220,7 @@ def check_resources(language_only: set[str], expansions: dict[str, list[str]],
|
||||
tables.
|
||||
|
||||
Args:
|
||||
loaded (dict[str, dict[str, Any]]): the three resource trees.
|
||||
language_only (set[str]): resources that exist in one runtime only.
|
||||
expansions (dict[str, list[str]]): python alias table, so one
|
||||
python name can stand for several typescript ones.
|
||||
@@ -76,20 +228,6 @@ def check_resources(language_only: set[str], expansions: dict[str, list[str]],
|
||||
register commands on purpose without a registry factory.
|
||||
"""
|
||||
failures: list[str] = []
|
||||
trees = {
|
||||
"python": SPEC / "python" / "resources.json",
|
||||
"node": SPEC / "typescript" / "node" / "resources.json",
|
||||
"browser": SPEC / "typescript" / "browser" / "resources.json",
|
||||
}
|
||||
loaded: dict[str, dict[str, list[str]]] = {}
|
||||
for tree, path in trees.items():
|
||||
if not path.is_file():
|
||||
return [
|
||||
f"missing {path}\nrun scripts/gen_specs.py and "
|
||||
"typescript/scripts/gen-specs.ts first"
|
||||
]
|
||||
loaded[tree] = json.loads(path.read_text())
|
||||
|
||||
for tree, payload in loaded.items():
|
||||
registry = set(payload["registry"])
|
||||
allowed = unconstructible.get(tree, {})
|
||||
@@ -287,12 +425,18 @@ def main() -> int:
|
||||
dict[str,
|
||||
str]] = exceptions["unconstructible_resources"]
|
||||
allowed: dict[str, Any] = exceptions["commands"]
|
||||
capability_exempt: dict[str,
|
||||
dict[str,
|
||||
str]] = exceptions["resource_capabilities"]
|
||||
io_exempt: dict[str, dict[str, str]] = exceptions["command_io"]
|
||||
io_aliases: dict[str, str] = exceptions["command_io_aliases"]["python"]
|
||||
|
||||
py_specs = load_dir(PYTHON)
|
||||
ts_variants = [load_dir(p) for p in TYPESCRIPT]
|
||||
|
||||
failures: list[str] = []
|
||||
used: set[str] = set()
|
||||
used_facts: set[str] = set()
|
||||
|
||||
# Python has no runtime split, so its command set must equal the union
|
||||
# of the two typescript variants; comparing against node alone would
|
||||
@@ -307,9 +451,27 @@ def main() -> int:
|
||||
if only_ts:
|
||||
failures.append(f"commands only in typescript: {only_ts}")
|
||||
|
||||
trees = load_resource_trees()
|
||||
failures.extend(compare_variants(ts_variants))
|
||||
failures.extend(check_resources(language_only, expansions,
|
||||
unconstructible))
|
||||
failures.extend(
|
||||
check_resources(trees, language_only, expansions, unconstructible))
|
||||
failures.extend(
|
||||
check_capabilities(trees, expansions, language_only, capability_exempt,
|
||||
used_facts))
|
||||
failures.extend(
|
||||
check_command_io(trees, io_aliases, language_only, io_exempt,
|
||||
used_facts))
|
||||
declared = {
|
||||
f"{name}:{key}"
|
||||
for table in (capability_exempt, io_exempt)
|
||||
for name, keys in table.items()
|
||||
for key in keys
|
||||
}
|
||||
stale_facts = sorted(declared - used_facts)
|
||||
if stale_facts:
|
||||
failures.append(f"stale resource-fact exemptions in "
|
||||
f"{EXCEPTIONS.name}, the divergence they cover is "
|
||||
f"gone: {stale_facts}")
|
||||
|
||||
for name in sorted(set(py_specs) & set(ts_variants[0])):
|
||||
py, ts = py_specs[name], ts_variants[0][name]
|
||||
|
||||
+64
-1
@@ -22,15 +22,24 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import mirage.commands.builtin
|
||||
from mirage.commands.builtin.generic_bind.adapter import CommandIO
|
||||
from mirage.commands.config import RegisteredCommand
|
||||
from mirage.commands.spec import SPECS
|
||||
from mirage.commands.spec.types import CommandSpec, Operand, Option
|
||||
from mirage.resource.registry import REGISTRY
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.registry import REGISTRY, resolve_class
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OUT = Path(__file__).resolve().parent.parent / "spec" / "python" / "general"
|
||||
|
||||
BUILTIN = Path(mirage.commands.builtin.__file__).resolve(
|
||||
).parent # type: ignore[arg-type]
|
||||
|
||||
# Slots holding a configuration value rather than an operation. Everything
|
||||
# else on the adapter is a wired operation, reported by name.
|
||||
IO_VALUE_FIELDS = frozenset({"local", "max_glob_matches", "max_du_entries"})
|
||||
|
||||
|
||||
def _walk_pkg(pkg: Any) -> list[str]:
|
||||
"""Import every builtin command module, reporting the ones that failed.
|
||||
@@ -178,6 +187,58 @@ def _emit_one(name: str, spec: Any, rcs: list[RegisteredCommand]) -> None:
|
||||
json.dumps(payload, indent=2, sort_keys=True, default=_default) + "\n")
|
||||
|
||||
|
||||
def _capabilities() -> dict[str, dict[str, Any]]:
|
||||
"""Per-resource behavior values, read off the class, never an instance.
|
||||
|
||||
Registry membership only says a backend can be built. How it behaves
|
||||
once mounted is a second hand-maintained surface that drifted just as
|
||||
quietly: python kept the 600 s ``index_ttl`` default for postgres and
|
||||
mongodb where typescript pins 0, so an ``ls`` of a live schema could
|
||||
be ten minutes stale. ``storage_id`` and ``statfs`` are reported as
|
||||
"does this class override the base" rather than by value, because the
|
||||
base answers are per-instance identity and UNKNOWN.
|
||||
"""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for name in sorted(REGISTRY):
|
||||
cls = resolve_class(REGISTRY[name].resource_path)
|
||||
out[name] = {
|
||||
"index_ttl": cls.index_ttl,
|
||||
"caches_reads": cls.caches_reads,
|
||||
"supports_snapshot": cls.SUPPORTS_SNAPSHOT,
|
||||
"sizes_always_known": cls.SIZES_ALWAYS_KNOWN,
|
||||
"storage_id": cls.storage_id is not BaseResource.storage_id,
|
||||
"statfs": cls.statfs is not BaseResource.statfs,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def _command_io() -> dict[str, dict[str, Any]]:
|
||||
"""The wired ``CommandIO`` slots per backend command package.
|
||||
|
||||
The adapter's slot set is a hand-filled literal that no gate reads, so
|
||||
a backend can omit ``du`` or ``find`` and quietly fall back to the
|
||||
capped readdir walk while its twin pushes the work down to the API.
|
||||
Dumping the key set turns that omission into a spec diff.
|
||||
"""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(BUILTIN.glob("*/io.py")):
|
||||
backend = path.parent.name
|
||||
mod = importlib.import_module(f"mirage.commands.builtin.{backend}.io")
|
||||
io = getattr(mod, "IO", None)
|
||||
if not isinstance(io, CommandIO):
|
||||
continue
|
||||
slots = sorted(f.name for f in fields(CommandIO)
|
||||
if f.name not in IO_VALUE_FIELDS
|
||||
and getattr(io, f.name) is not None)
|
||||
out[backend] = {
|
||||
"slots": slots,
|
||||
"local": io.local,
|
||||
"max_glob_matches": io.max_glob_matches,
|
||||
"max_du_entries": io.max_du_entries,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def _emit_resources(registry: dict[str, list[RegisteredCommand]]) -> None:
|
||||
"""Dump the two resource-name sets the parity gate compares.
|
||||
|
||||
@@ -201,6 +262,8 @@ def _emit_resources(registry: dict[str, list[RegisteredCommand]]) -> None:
|
||||
payload = {
|
||||
"registry": sorted(REGISTRY),
|
||||
"command_resources": sorted(command_resources),
|
||||
"capabilities": _capabilities(),
|
||||
"command_io": _command_io(),
|
||||
}
|
||||
path = OUT.parent / "resources.json"
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
@@ -68,6 +68,33 @@ exemption. `python/tests/resource/test_registry.py` and
|
||||
re-copying the name list, so neither can pin an omission the way the old
|
||||
hand-written set pinned SharePoint's.
|
||||
|
||||
Two more tables record what a backend *does*, not just that it exists.
|
||||
Membership never said whether a mount serves ten-minute-stale listings or
|
||||
pushes a `du` down to the API, and both facts were hand-maintained on each
|
||||
side.
|
||||
|
||||
`capabilities` carries, per registry name, `index_ttl`, `caches_reads`,
|
||||
`supports_snapshot` and `sizes_always_known`, plus whether the class overrides
|
||||
`storage_id` and `statfs` (booleans, since the base answers are per-instance
|
||||
identity and UNKNOWN). Python reads them off the class. TypeScript reads them
|
||||
from the class *declarations* (`scripts/resource_facts.ts`): the twins are
|
||||
instance fields, so observing them at runtime would mean constructing the
|
||||
resource, and construction is not inert — `buildResource('github', {})` issues
|
||||
an HTTP request and `postgres` opens a connection. A value the extractor
|
||||
cannot read as a literal is dumped verbatim as `<expr:…>` rather than guessed,
|
||||
so it surfaces as a real mismatch instead of a plausible default. A browser
|
||||
factory that exists only to explain that the runtime cannot serve the backend
|
||||
(`lancedb`, `email`) dumps `null`, and the node entry is the one compared
|
||||
against Python.
|
||||
|
||||
`command_io` carries, per backend command package, the wired `CommandIO` slot
|
||||
names plus `local`, `max_glob_matches` and `max_du_entries`. The adapter's slot
|
||||
set is a hand-filled literal nothing else reads, so a backend could omit `du`
|
||||
or `find` and fall back to the capped readdir walk — a partial total and an
|
||||
exit 1 past the cap — while its twin pushed the same query down to the API.
|
||||
Where the two languages name the package differently, `command_io_aliases`
|
||||
maps Python's name onto TypeScript's.
|
||||
|
||||
Divergences that are structural rather than bugs live in
|
||||
`parity_exceptions.json`:
|
||||
|
||||
@@ -80,6 +107,8 @@ Divergences that are structural rather than bugs live in
|
||||
- `commands` — per-command exemptions. `fields` mutes a whole top-level field;
|
||||
`by_resource` names one resource and one metadata key, so an exemption
|
||||
cannot quietly hide a second divergence on the same command.
|
||||
- `resource_capabilities` and `command_io` — one resource and one key per
|
||||
entry, same rule: an exemption covers the fact it names and nothing else.
|
||||
|
||||
The checker fails on a *stale* exception, so an entry cannot outlive the
|
||||
divergence it documents. An exemption counts as used only when it actually
|
||||
|
||||
+63
-17
@@ -1,31 +1,77 @@
|
||||
{
|
||||
"resource_expansions": {
|
||||
"python": {
|
||||
"hf_buckets": ["hf_buckets", "hf_datasets", "hf_models", "hf_spaces"]
|
||||
"command_io": {
|
||||
"box": {
|
||||
"slots": "typescript pushes du and find down to the Box API while python takes the capped readdir walk (core/box/du/ is written but unwired); python range-reads while typescript's core read has no window. Both halves of T1-I."
|
||||
},
|
||||
"dropbox": {
|
||||
"slots": "same as box: du/find are pushed down in typescript only, and the range read is python only."
|
||||
},
|
||||
"gdrive": {
|
||||
"slots": "python range-reads; the typescript core read has no window argument."
|
||||
},
|
||||
"hf": {
|
||||
"slots": "find is a slot in python and a bespoke command in typescript (see notion) \u2014 the same query, expressed at a different layer."
|
||||
},
|
||||
"notion": {
|
||||
"slots": "find is a slot in python and a bespoke command in typescript (notion/find.ts). Both pass stat=None, so the -mtime behavior matches; only the layer differs."
|
||||
},
|
||||
"onedrive": {
|
||||
"slots": "python range-reads; typescript has no core/onedrive/read.ts to add a window to."
|
||||
},
|
||||
"sharepoint": {
|
||||
"slots": "python range-reads; typescript has no core/sharepoint/read.ts to add a window to."
|
||||
},
|
||||
"ssh": {
|
||||
"slots": "python range-reads over SFTP; typescript's read is whole-file because of the ssh2 256 KiB packet cap. append is wired in typescript and read by no builder on either side \u2014 tee is read-modify-write everywhere \u2014 so the slot is pending a decision to wire it or drop it."
|
||||
}
|
||||
},
|
||||
"language_only_resources": {
|
||||
"opfs": "Browser-only OPFS backend. Python has no browser runtime, so there is nothing to mirror."
|
||||
},
|
||||
"unconstructible_resources": {
|
||||
"command_io_aliases": {
|
||||
"python": {
|
||||
"history": "The /.bash_history view mount is created by the workspace itself and is never named in YAML or a snapshot, so it registers commands without a build_resource entry."
|
||||
},
|
||||
"node": {
|
||||
"history": "See python: the history view mount is workspace-internal."
|
||||
},
|
||||
"browser": {
|
||||
"history": "See python: the history view mount is workspace-internal.",
|
||||
"databricks_volume": "Registers commands in the browser bundle but has no browser registry factory. Both it and jaeger are pure HTTP, so the omission looks accidental rather than a runtime limit — confirm the intended browser scope before wiring them up.",
|
||||
"jaeger": "Registers commands in the browser bundle but has no browser registry factory. See databricks_volume."
|
||||
"hf_buckets": "hf"
|
||||
}
|
||||
},
|
||||
"commands": {
|
||||
"grep": {
|
||||
"by_resource": {
|
||||
"github": ["has_provision"]
|
||||
"github": [
|
||||
"has_provision"
|
||||
]
|
||||
},
|
||||
"reason": "TypeScript has no github grep provision. Python's is a bespoke cost model that walks the index to sum matched bytes and read ops, and it reaches IndexCacheStore internals, so porting it needs its own change rather than riding along with the parity gate."
|
||||
}
|
||||
},
|
||||
"language_only_resources": {
|
||||
"opfs": "Browser-only OPFS backend. Python has no browser runtime, so there is nothing to mirror."
|
||||
},
|
||||
"resource_capabilities": {
|
||||
"github": {
|
||||
"supports_snapshot": "TypeScript records the blob SHA as a snapshot fingerprint; python stamps the same SHA in core/github/stat.py but does not declare SUPPORTS_SNAPSHOT, so no drift check fires. Left to the snapshot redesign (#721) rather than flipped here, because turning it on changes what a python snapshot records."
|
||||
},
|
||||
"lancedb": {
|
||||
"caches_reads": "TypeScript computes it per mount from the URI scheme (remote tables cache, local ones do not); python has no such branch and never caches. A value, not a table, so there is nothing to single-source."
|
||||
}
|
||||
},
|
||||
"resource_expansions": {
|
||||
"python": {
|
||||
"hf_buckets": [
|
||||
"hf_buckets",
|
||||
"hf_datasets",
|
||||
"hf_models",
|
||||
"hf_spaces"
|
||||
]
|
||||
}
|
||||
},
|
||||
"unconstructible_resources": {
|
||||
"browser": {
|
||||
"databricks_volume": "Registers commands in the browser bundle but has no browser registry factory. Both it and jaeger are pure HTTP, so the omission looks accidental rather than a runtime limit \u2014 confirm the intended browser scope before wiring them up.",
|
||||
"history": "See python: the history view mount is workspace-internal.",
|
||||
"jaeger": "Registers commands in the browser bundle but has no browser registry factory. See databricks_volume."
|
||||
},
|
||||
"node": {
|
||||
"history": "See python: the history view mount is workspace-internal."
|
||||
},
|
||||
"python": {
|
||||
"history": "The /.bash_history view mount is created by the workspace itself and is never named in YAML or a snapshot, so it registers commands without a build_resource entry."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,803 @@
|
||||
{
|
||||
"capabilities": {
|
||||
"aliyun": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"backblaze": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"box": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"ceph": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"chroma": {
|
||||
"caches_reads": false,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"dify": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"digitalocean": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"discord": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"dropbox": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"email": null,
|
||||
"gcs": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"gdocs": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"gdrive": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"github": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"github_ci": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"gmail": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"gsheets": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"gslides": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"lancedb": null,
|
||||
"langfuse": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"linear": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"mem0": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"minio": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"mongodb": {
|
||||
"caches_reads": false,
|
||||
"index_ttl": 0,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"notion": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"oci": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"onedrive": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"opfs": {
|
||||
"caches_reads": false,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"postgres": {
|
||||
"caches_reads": false,
|
||||
"index_ttl": 0,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"qdrant": {
|
||||
"caches_reads": false,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"qingstor": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"r2": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"ram": {
|
||||
"caches_reads": false,
|
||||
"index_ttl": 0,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"s3": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"scaleway": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"seaweedfs": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"sharepoint": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 86400,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"slack": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": true,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"supabase": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"tencent": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
},
|
||||
"trello": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": false
|
||||
},
|
||||
"wasabi": {
|
||||
"caches_reads": true,
|
||||
"index_ttl": 600,
|
||||
"sizes_always_known": false,
|
||||
"statfs": false,
|
||||
"storage_id": false,
|
||||
"supports_snapshot": true
|
||||
}
|
||||
},
|
||||
"command_io": {
|
||||
"box": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"dir_copy",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"chroma": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"databricks_volume": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"exists",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_range",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"dify": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"discord": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"dropbox": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"gdocs": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"gdrive": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"dir_copy",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"github": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 5000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"github_ci": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"gmail": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"gsheets": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"gslides": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"history": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"jaeger": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"lancedb": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"langfuse": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"linear": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"mem0": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"mongodb": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"notion": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"onedrive": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"dir_copy",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"opfs": {
|
||||
"local": true,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 50000,
|
||||
"slots": [
|
||||
"append",
|
||||
"copy",
|
||||
"create",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"postgres": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"qdrant": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"ram": {
|
||||
"local": true,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 50000,
|
||||
"slots": [
|
||||
"append",
|
||||
"copy",
|
||||
"create",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"set_attrs",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"s3": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 5000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_range",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"sharepoint": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"copy",
|
||||
"create",
|
||||
"dir_copy",
|
||||
"du",
|
||||
"exists",
|
||||
"find",
|
||||
"is_mounted",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"slack": {
|
||||
"local": false,
|
||||
"max_du_entries": 500,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
},
|
||||
"trello": {
|
||||
"local": false,
|
||||
"max_du_entries": 10000,
|
||||
"max_glob_matches": 10000,
|
||||
"slots": [
|
||||
"is_mounted",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"stat"
|
||||
]
|
||||
}
|
||||
},
|
||||
"command_resources": [
|
||||
"box",
|
||||
"chroma",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -26,11 +26,12 @@ import { stat as dbxStat } from '../../../core/databricks_volume/stat.ts'
|
||||
import { readStream as dbxStream } from '../../../core/databricks_volume/stream.ts'
|
||||
import { unlink as dbxUnlink } from '../../../core/databricks_volume/unlink.ts'
|
||||
import { writeBytes as dbxWrite } from '../../../core/databricks_volume/write.ts'
|
||||
import type { CommandIO } from '../generic_bind/index.ts'
|
||||
import { type CommandIO, rangeOf } from '../generic_bind/index.ts'
|
||||
|
||||
export const DATABRICKS_VOLUME_IO: CommandIO<DatabricksVolumeAccessor> = {
|
||||
readdir: dbxReaddir,
|
||||
readBytes: dbxRead,
|
||||
readRange: rangeOf(dbxRead),
|
||||
readStream: dbxStream,
|
||||
stat: dbxStat,
|
||||
isMounted: () => true,
|
||||
|
||||
@@ -153,6 +153,32 @@ export function resolveGlobOf<A extends Accessor = Accessor>(ops: CommandIO<A>):
|
||||
return makeResolveGlob(ops.readdir, ops.maxGlobMatches)
|
||||
}
|
||||
|
||||
/**
|
||||
* A `readRange` slot built from a backend read that already takes a byte
|
||||
* window as its options argument.
|
||||
*
|
||||
* Without the slot the ops factory reads the whole object and slices, so
|
||||
* `head -c 100` on a 2 GiB S3 key downloads 2 GiB. Python has pushed the
|
||||
* window down on every one of these backends since the slot existed by
|
||||
* pointing `read_range` at its own `read_bytes`; this is the same move,
|
||||
* spelled for a read whose window arrives in an options object.
|
||||
*
|
||||
* Args:
|
||||
* read: the backend's whole-file read, whose fourth argument is an
|
||||
* `{offset?, size?}` window.
|
||||
*/
|
||||
export function rangeOf<A extends Accessor = Accessor>(
|
||||
read: (
|
||||
accessor: A,
|
||||
path: PathSpec,
|
||||
index: IndexCacheStore | undefined,
|
||||
options: { offset?: number; size?: number },
|
||||
) => Promise<Uint8Array>,
|
||||
): NonNullable<CommandIO<A>['readRange']> {
|
||||
return (accessor, path, index, offset, size) =>
|
||||
read(accessor, path, index, size === null ? { offset } : { offset, size })
|
||||
}
|
||||
|
||||
// Whether a path that failed with ENOENT is an implicit directory. Keyed
|
||||
// backends (RAM/Redis/S3) have no directory entries: stat/read of a prefix
|
||||
// that only exists through deeper keys raises ENOENT. The operand's own
|
||||
|
||||
@@ -19,6 +19,7 @@ export {
|
||||
type DuOps,
|
||||
makeResolveGlob,
|
||||
overlaidStat,
|
||||
rangeOf,
|
||||
resolveGlobOf,
|
||||
} from './adapter.ts'
|
||||
export { type MakeGenericCommandsOptions, makeGenericCommands } from './factory.ts'
|
||||
|
||||
@@ -30,11 +30,12 @@ import { stream as s3Stream } from '../../../core/s3/stream.ts'
|
||||
import { truncate as s3Truncate } from '../../../core/s3/truncate.ts'
|
||||
import { unlink as s3Unlink } from '../../../core/s3/unlink.ts'
|
||||
import { write as s3Write } from '../../../core/s3/write.ts'
|
||||
import type { CommandIO } from '../generic_bind/index.ts'
|
||||
import { type CommandIO, rangeOf } from '../generic_bind/index.ts'
|
||||
|
||||
export const S3_IO: CommandIO<S3Accessor> = {
|
||||
readdir: s3Readdir,
|
||||
readBytes: s3Read,
|
||||
readRange: rangeOf(s3Read),
|
||||
readStream: s3Stream,
|
||||
stat: s3Stat,
|
||||
isMounted: () => true,
|
||||
|
||||
@@ -237,6 +237,7 @@ export {
|
||||
metadataProvision,
|
||||
overlaidStat,
|
||||
pureProvision,
|
||||
rangeOf,
|
||||
resolveGlobOf,
|
||||
withDefaultProvisions,
|
||||
writeMetadataProvision,
|
||||
@@ -294,7 +295,13 @@ export {
|
||||
export { walkFind } from './core/generic/find.ts'
|
||||
export { statGeneric } from './commands/builtin/generic/stat.ts'
|
||||
export { diffGeneric } from './commands/builtin/generic/diff.ts'
|
||||
export { duGeneric, parseDepth, parseDuFlags, runDu } from './commands/builtin/generic/du.ts'
|
||||
export {
|
||||
DEFAULT_MAX_DU_ENTRIES,
|
||||
duGeneric,
|
||||
parseDepth,
|
||||
parseDuFlags,
|
||||
runDu,
|
||||
} from './commands/builtin/generic/du.ts'
|
||||
export { treeGeneric } from './commands/builtin/generic/tree.ts'
|
||||
export { lsGeneric } from './commands/builtin/generic/ls.ts'
|
||||
export { fileGeneric } from './commands/builtin/generic/file.ts'
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { CommandIO } from '@struktoai/mirage-core'
|
||||
import { type CommandIO, rangeOf } from '@struktoai/mirage-core'
|
||||
import type { GridFSAccessor } from '../../../accessor/gridfs.ts'
|
||||
import { SCOPE_ERROR } from '../../../core/gridfs/constants.ts'
|
||||
import { copy as gridfsCopy } from '../../../core/gridfs/copy.ts'
|
||||
@@ -35,6 +35,7 @@ import { write as gridfsWrite } from '../../../core/gridfs/write.ts'
|
||||
export const GRIDFS_IO: CommandIO<GridFSAccessor> = {
|
||||
readdir: gridfsReaddir,
|
||||
readBytes: gridfsRead,
|
||||
readRange: rangeOf(gridfsRead),
|
||||
readStream: gridfsStream,
|
||||
stat: gridfsStat,
|
||||
isMounted: () => true,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { CommandIO } from '@struktoai/mirage-core'
|
||||
import { type CommandIO, rangeOf } from '@struktoai/mirage-core'
|
||||
import type { HfAccessor } from '../../../accessor/hf.ts'
|
||||
import { SCOPE_ERROR } from '../../../core/hf/constants.ts'
|
||||
import { create as hfCreate } from '../../../core/hf/create.ts'
|
||||
@@ -30,6 +30,7 @@ import { write as hfWrite } from '../../../core/hf/write.ts'
|
||||
export const HF_IO: CommandIO<HfAccessor> = {
|
||||
readdir: hfReaddir,
|
||||
readBytes: hfRead,
|
||||
readRange: rangeOf(hfRead),
|
||||
readStream: hfStream,
|
||||
stat: hfStat,
|
||||
du: { size: hfDu, entries: hfDuAll },
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { CommandIO } from '@struktoai/mirage-core'
|
||||
import { type CommandIO, rangeOf } from '@struktoai/mirage-core'
|
||||
import type { NextcloudAccessor } from '../../../accessor/nextcloud.ts'
|
||||
import { SCOPE_ERROR } from '../../../core/nextcloud/constants.ts'
|
||||
import { copy } from '../../../core/nextcloud/copy.ts'
|
||||
@@ -25,6 +25,7 @@ export const NEXTCLOUD_IO: CommandIO<NextcloudAccessor> = {
|
||||
maxGlobMatches: SCOPE_ERROR,
|
||||
readdir,
|
||||
readBytes: read,
|
||||
readRange: rangeOf(read),
|
||||
readStream: stream,
|
||||
stat,
|
||||
isMounted: () => true,
|
||||
|
||||
@@ -22,6 +22,15 @@ import * as Node from '@struktoai/mirage-node'
|
||||
|
||||
import type { CommandSpec, Operand, Option, RegisteredCommand } from '@struktoai/mirage-core'
|
||||
|
||||
import {
|
||||
type Capabilities,
|
||||
type CommandIoFacts,
|
||||
capabilitiesOf,
|
||||
collectClasses,
|
||||
commandIoFacts,
|
||||
registryClasses,
|
||||
} from './resource_facts.ts'
|
||||
|
||||
const { CommandSpec: SpecClass, Operand: OperandClass, Option: OptionClass } = Core
|
||||
|
||||
const __dirname = resolve(fileURLToPath(import.meta.url), '..')
|
||||
@@ -238,10 +247,19 @@ function sortedStringify(value: unknown): string {
|
||||
// command. A name in the second but not the first registers commands yet
|
||||
// cannot be mounted by name, which is how chroma/dify/lancedb/qdrant stayed
|
||||
// unconstructible in typescript while appearing in every command's `_meta`.
|
||||
//
|
||||
// `capabilities` and `command_io` carry the values behind those names.
|
||||
// Registry membership only says a backend can be built; how it behaves is
|
||||
// a second hand-maintained surface that drifted just as quietly — Python
|
||||
// served ten-minute-stale listings of a live postgres schema because its
|
||||
// `index_ttl` kept the 600 s default where typescript pinned 0, and box's
|
||||
// `du` slot is wired on one side and absent on the other.
|
||||
function emitResources(
|
||||
name: string,
|
||||
knownResources: string[],
|
||||
registry: Record<string, RegisteredCommand[]>,
|
||||
capabilities: Record<string, Capabilities | null>,
|
||||
commandIo: Record<string, CommandIoFacts>,
|
||||
): void {
|
||||
const commandResources = new Set<string>()
|
||||
for (const rcs of Object.values(registry)) {
|
||||
@@ -250,12 +268,30 @@ function emitResources(
|
||||
const payload = {
|
||||
registry: [...knownResources].sort(),
|
||||
command_resources: [...commandResources].sort(),
|
||||
capabilities,
|
||||
command_io: commandIo,
|
||||
}
|
||||
const path = resolve(SPEC_ROOT, name, 'resources.json')
|
||||
writeFileSync(path, sortedStringify(payload) + '\n')
|
||||
console.log(`emitted ${payload.registry.length} registry names to ${path}`)
|
||||
}
|
||||
|
||||
// Every registry name's capability values, read from the class the entry
|
||||
// constructs. A name whose class cannot be resolved is a hard error: a
|
||||
// missing row would read as "no divergence here" in the parity gate.
|
||||
function capabilitiesFor(
|
||||
pkgs: readonly string[],
|
||||
variantPkg: string,
|
||||
): Record<string, Capabilities | null> {
|
||||
const classes = collectClasses(PACKAGES, pkgs)
|
||||
const names = registryClasses(resolve(PACKAGES, variantPkg, 'src', 'resource', 'registry.ts'))
|
||||
const out: Record<string, Capabilities | null> = {}
|
||||
for (const [resource, className] of [...names].sort(([a], [b]) => a.localeCompare(b))) {
|
||||
out[resource] = className === null ? null : capabilitiesOf(className, classes)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function emitVariant(
|
||||
name: string,
|
||||
pkgs: readonly string[],
|
||||
@@ -274,7 +310,16 @@ function emitVariant(
|
||||
writeFileSync(resolve(outDir, `${cmd}.json`), sortedStringify(payload) + '\n')
|
||||
}
|
||||
console.log(`emitted ${cmdNames.length} specs to ${outDir}`)
|
||||
emitResources(name, knownResources, registry)
|
||||
emitResources(
|
||||
name,
|
||||
knownResources,
|
||||
registry,
|
||||
capabilitiesFor(pkgs, pkgs[pkgs.length - 1] as string),
|
||||
commandIoFacts(PACKAGES, pkgs, {
|
||||
maxGlobMatches: Core.DEFAULT_MAX_GLOB_MATCHES,
|
||||
maxDuEntries: Core.DEFAULT_MAX_DU_ENTRIES,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { existsSync, readdirSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import ts from 'typescript'
|
||||
|
||||
// Capability values and CommandIO slots are read from the source rather
|
||||
// than from a live object on purpose. Python can introspect its resource
|
||||
// classes because the values are class attributes, but the typescript
|
||||
// twins are instance fields, so the only way to observe them at runtime
|
||||
// is to construct the resource — and construction is not inert here:
|
||||
// `buildResource('github', {})` issues an HTTP request and `postgres`
|
||||
// opens a connection. A generator that reaches the network produces a
|
||||
// different spec depending on who runs it, so the values come from the
|
||||
// declarations instead.
|
||||
|
||||
const CAPABILITY_FIELDS = [
|
||||
'indexTtl',
|
||||
'cachesReads',
|
||||
'supportsSnapshot',
|
||||
'sizesAlwaysKnown',
|
||||
] as const
|
||||
|
||||
// Slots that carry a configuration value rather than an operation. They
|
||||
// are reported as values; every other key of the literal is a wired slot.
|
||||
const IO_VALUE_FIELDS = new Set(['local', 'maxGlobMatches', 'maxDuEntries'])
|
||||
|
||||
const BASE_CLASS = 'BaseResource'
|
||||
|
||||
export interface Capabilities {
|
||||
index_ttl: number | string
|
||||
caches_reads: boolean | string
|
||||
supports_snapshot: boolean | string
|
||||
sizes_always_known: boolean | string
|
||||
storage_id: boolean
|
||||
statfs: boolean
|
||||
}
|
||||
|
||||
export interface CommandIoFacts {
|
||||
slots: string[]
|
||||
local: boolean
|
||||
max_glob_matches: number | null
|
||||
max_du_entries: number | null
|
||||
}
|
||||
|
||||
interface ClassInfo {
|
||||
decl: ts.ClassDeclaration
|
||||
source: ts.SourceFile
|
||||
parent: string | undefined
|
||||
}
|
||||
|
||||
function snake(name: string): string {
|
||||
return name.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)
|
||||
}
|
||||
|
||||
function parse(file: string): ts.SourceFile {
|
||||
return ts.createSourceFile(file, ts.sys.readFile(file) ?? '', ts.ScriptTarget.ESNext, true)
|
||||
}
|
||||
|
||||
function sourceFiles(dir: string): string[] {
|
||||
if (!existsSync(dir)) return []
|
||||
const out: string[] = []
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const path = resolve(dir, entry.name)
|
||||
if (entry.isDirectory()) out.push(...sourceFiles(path))
|
||||
else if (entry.name.endsWith('.ts') && !entry.name.endsWith('.test.ts')) out.push(path)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The literal a capability field is initialized with. Anything computed
|
||||
// is reported verbatim as `<expr:Kind>` so the parity gate shows a real
|
||||
// mismatch instead of a plausible-looking default: a value this cannot
|
||||
// read is a value it must not guess.
|
||||
function literalValue(node: ts.Expression | undefined): number | boolean | string {
|
||||
if (node === undefined) return '<declared, no initializer>'
|
||||
if (ts.isNumericLiteral(node)) return Number(node.text.replaceAll('_', ''))
|
||||
if (node.kind === ts.SyntaxKind.TrueKeyword) return true
|
||||
if (node.kind === ts.SyntaxKind.FalseKeyword) return false
|
||||
if (ts.isPrefixUnaryExpression(node) && ts.isNumericLiteral(node.operand)) {
|
||||
const value = Number(node.operand.text.replaceAll('_', ''))
|
||||
return node.operator === ts.SyntaxKind.MinusToken ? -value : value
|
||||
}
|
||||
return `<expr:${ts.SyntaxKind[node.kind]}>`
|
||||
}
|
||||
|
||||
function heritageName(decl: ts.ClassDeclaration): string | undefined {
|
||||
for (const clause of decl.heritageClauses ?? []) {
|
||||
if (clause.token !== ts.SyntaxKind.ExtendsKeyword) continue
|
||||
const expr = clause.types[0]?.expression
|
||||
if (expr !== undefined && ts.isIdentifier(expr)) return expr.text
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Every resource class a variant can reach, keyed by class name.
|
||||
*
|
||||
* Duplicate names across the scanned packages would make the extends walk
|
||||
* ambiguous, so they are refused rather than resolved by import order.
|
||||
*/
|
||||
export function collectClasses(
|
||||
packagesRoot: string,
|
||||
pkgs: readonly string[],
|
||||
): Map<string, ClassInfo> {
|
||||
const out = new Map<string, ClassInfo>()
|
||||
for (const pkg of pkgs) {
|
||||
for (const file of sourceFiles(resolve(packagesRoot, pkg, 'src', 'resource'))) {
|
||||
const source = parse(file)
|
||||
ts.forEachChild(source, (node) => {
|
||||
if (!ts.isClassDeclaration(node) || node.name === undefined) return
|
||||
const name = node.name.text
|
||||
const seen = out.get(name)
|
||||
if (seen !== undefined) {
|
||||
throw new Error(
|
||||
`two resource classes named ${name}: ${seen.source.fileName} and ${file}; ` +
|
||||
`the capability walk cannot tell which one a registry entry means`,
|
||||
)
|
||||
}
|
||||
out.set(name, { decl: node, source, parent: heritageName(node) })
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function chain(className: string, classes: Map<string, ClassInfo>): ClassInfo[] {
|
||||
const out: ClassInfo[] = []
|
||||
const seen = new Set<string>()
|
||||
let name: string | undefined = className
|
||||
while (name !== undefined && !seen.has(name)) {
|
||||
seen.add(name)
|
||||
const info: ClassInfo | undefined = classes.get(name)
|
||||
if (info === undefined) break
|
||||
out.push(info)
|
||||
name = info.parent
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function declaresMethod(info: ClassInfo, name: string): boolean {
|
||||
return info.decl.members.some(
|
||||
(m) =>
|
||||
(ts.isMethodDeclaration(m) || ts.isPropertyDeclaration(m)) &&
|
||||
m.name.getText(info.source) === name,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* One class's capability values, resolved up its extends chain.
|
||||
*
|
||||
* The three boolean capabilities are optional members of the `Resource`
|
||||
* interface with no `BaseResource` declaration, and every reader coerces
|
||||
* with `=== true` (`resource/base.ts`), so a class that declares none of
|
||||
* them is false — not undefined. `indexTtl` does have a `BaseResource`
|
||||
* default and is picked up by the same walk.
|
||||
*
|
||||
* Args:
|
||||
* className: the class the registry constructs.
|
||||
* classes: every reachable resource class, from `collectClasses`.
|
||||
*/
|
||||
export function capabilitiesOf(className: string, classes: Map<string, ClassInfo>): Capabilities {
|
||||
const ancestry = chain(className, classes)
|
||||
if (ancestry.length === 0)
|
||||
throw new Error(`no source declaration for resource class ${className}`)
|
||||
const values: Record<string, number | boolean | string> = {}
|
||||
for (const info of ancestry) {
|
||||
for (const member of info.decl.members) {
|
||||
if (!ts.isPropertyDeclaration(member)) continue
|
||||
const name = member.name.getText(info.source)
|
||||
if (!(CAPABILITY_FIELDS as readonly string[]).includes(name)) continue
|
||||
if (name in values) continue
|
||||
values[name] = literalValue(member.initializer)
|
||||
}
|
||||
}
|
||||
const overrides = ancestry.filter((info) => info.decl.name?.text !== BASE_CLASS)
|
||||
return {
|
||||
index_ttl: values.indexTtl ?? 600,
|
||||
caches_reads: values.cachesReads ?? false,
|
||||
supports_snapshot: values.supportsSnapshot ?? false,
|
||||
sizes_always_known: values.sizesAlwaysKnown ?? false,
|
||||
storage_id: overrides.some((info) => declaresMethod(info, 'storageId')),
|
||||
statfs: overrides.some((info) => declaresMethod(info, 'statfs')),
|
||||
}
|
||||
}
|
||||
|
||||
// The value of a named constant a slot was set to, followed one import
|
||||
// hop. `maxGlobMatches: SCOPE_ERROR` is the whole reason this exists:
|
||||
// reporting the name instead of 5000 would make the two languages differ
|
||||
// on a value they agree about.
|
||||
function resolveIdentifier(
|
||||
source: ts.SourceFile,
|
||||
name: string,
|
||||
): number | boolean | string | undefined {
|
||||
for (const statement of source.statements) {
|
||||
if (ts.isVariableStatement(statement)) {
|
||||
for (const decl of statement.declarationList.declarations) {
|
||||
if (ts.isIdentifier(decl.name) && decl.name.text === name)
|
||||
return literalValue(decl.initializer)
|
||||
}
|
||||
}
|
||||
if (!ts.isImportDeclaration(statement)) continue
|
||||
const bindings = statement.importClause?.namedBindings
|
||||
if (bindings === undefined || !ts.isNamedImports(bindings)) continue
|
||||
if (!bindings.elements.some((el) => el.name.text === name)) continue
|
||||
const specifier = (statement.moduleSpecifier as ts.StringLiteral).text
|
||||
if (!specifier.startsWith('.')) continue
|
||||
const target = resolve(source.fileName, '..', specifier)
|
||||
if (!existsSync(target)) continue
|
||||
for (const inner of parse(target).statements) {
|
||||
if (!ts.isVariableStatement(inner)) continue
|
||||
for (const decl of inner.declarationList.declarations) {
|
||||
if (ts.isIdentifier(decl.name) && decl.name.text === name)
|
||||
return literalValue(decl.initializer)
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
// The file a local name was imported from, and the name it has there.
|
||||
// `import { read as s3Read }` binds `s3Read` locally to an exported
|
||||
// `read`, so both halves are needed to find the declaration.
|
||||
function importedFrom(
|
||||
source: ts.SourceFile,
|
||||
local: string,
|
||||
): { file: string; exported: string } | undefined {
|
||||
for (const statement of source.statements) {
|
||||
if (!ts.isImportDeclaration(statement)) continue
|
||||
const bindings = statement.importClause?.namedBindings
|
||||
if (bindings === undefined || !ts.isNamedImports(bindings)) continue
|
||||
for (const element of bindings.elements) {
|
||||
if (element.name.text !== local) continue
|
||||
const specifier = (statement.moduleSpecifier as ts.StringLiteral).text
|
||||
if (!specifier.startsWith('.')) return undefined
|
||||
const file = resolve(source.fileName, '..', specifier)
|
||||
if (!existsSync(file)) return undefined
|
||||
return { file, exported: (element.propertyName ?? element.name).text }
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Whether a backend's whole-file read declares a fourth parameter, i.e.
|
||||
// the `{offset, size}` window the `readRange` slot exists to hand it.
|
||||
// Parameter count rather than arity, because optional and defaulted
|
||||
// parameters do not show up in `Function.length` — `read(a, b, c?, opts =
|
||||
// {})` reports 2 at runtime, so nothing observable at runtime can answer
|
||||
// this question.
|
||||
function takesWindow(source: ts.SourceFile, local: string): boolean {
|
||||
const origin = importedFrom(source, local)
|
||||
if (origin === undefined) return false
|
||||
const declared = parse(origin.file)
|
||||
for (const statement of declared.statements) {
|
||||
if (!ts.isFunctionDeclaration(statement) || statement.name === undefined) continue
|
||||
if (statement.name.text !== origin.exported) continue
|
||||
const options = statement.parameters[3]?.type
|
||||
if (options === undefined) return false
|
||||
// The fourth parameter is not automatically a byte window — linear's
|
||||
// is a `ReadFilter` of query terms — so the type has to declare an
|
||||
// `offset` before this counts as a range the slot could carry.
|
||||
if (ts.isTypeLiteralNode(options)) return declaresOffset(options.members)
|
||||
if (!ts.isTypeReferenceNode(options) || !ts.isIdentifier(options.typeName)) return false
|
||||
return declaresOffsetNamed(declared, options.typeName.text)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A byte window is `offset` *and* `size`, the pair python's own opt-in
|
||||
// test keys on. `offset` alone is not enough: postgres pairs it with
|
||||
// `limit` to mean a SQL row range, which no byte slot can carry.
|
||||
function declaresOffset(members: ts.NodeArray<ts.TypeElement>): boolean {
|
||||
const names = new Set(members.filter((m) => m.name !== undefined).map((m) => m.name?.getText()))
|
||||
return names.has('offset') && names.has('size')
|
||||
}
|
||||
|
||||
function declaresOffsetNamed(source: ts.SourceFile, name: string): boolean {
|
||||
for (const statement of source.statements) {
|
||||
if (ts.isInterfaceDeclaration(statement) && statement.name.text === name) {
|
||||
return declaresOffset(statement.members)
|
||||
}
|
||||
}
|
||||
const origin = importedFrom(source, name)
|
||||
if (origin === undefined) return false
|
||||
for (const statement of parse(origin.file).statements) {
|
||||
if (ts.isInterfaceDeclaration(statement) && statement.name.text === origin.exported) {
|
||||
return declaresOffset(statement.members)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Whether a registry factory exists only to explain that this runtime
|
||||
// cannot serve the backend: it throws, or hands back a rejected promise,
|
||||
// without constructing anything.
|
||||
function refuses(node: ts.Node): boolean {
|
||||
let found = false
|
||||
const scan = (child: ts.Node): void => {
|
||||
if (ts.isThrowStatement(child)) found = true
|
||||
if (
|
||||
ts.isPropertyAccessExpression(child) &&
|
||||
child.name.text === 'reject' &&
|
||||
ts.isIdentifier(child.expression) &&
|
||||
child.expression.text === 'Promise'
|
||||
) {
|
||||
found = true
|
||||
}
|
||||
ts.forEachChild(child, scan)
|
||||
}
|
||||
scan(node)
|
||||
return found
|
||||
}
|
||||
|
||||
/**
|
||||
* Registry name to the class its factory constructs.
|
||||
*
|
||||
* Read from `registry.ts` rather than guessed from directory names: the
|
||||
* S3-compatible entries and the HuggingFace variants each map several
|
||||
* names onto classes whose directories do not match, and a guess that
|
||||
* lands on the wrong class would report capabilities for a backend the
|
||||
* user never mounts.
|
||||
*
|
||||
* Args:
|
||||
* registryFile: absolute path to the variant's `resource/registry.ts`.
|
||||
*/
|
||||
export function registryClasses(registryFile: string): Map<string, string | null> {
|
||||
const source = parse(registryFile)
|
||||
const out = new Map<string, string | null>()
|
||||
const visit = (node: ts.Node): void => {
|
||||
if (
|
||||
ts.isVariableDeclaration(node) &&
|
||||
ts.isIdentifier(node.name) &&
|
||||
node.name.text === 'REGISTRY' &&
|
||||
node.initializer !== undefined &&
|
||||
ts.isObjectLiteralExpression(node.initializer)
|
||||
) {
|
||||
for (const prop of node.initializer.properties) {
|
||||
if (!ts.isPropertyAssignment(prop)) continue
|
||||
const name = prop.name.getText(source).replace(/^['"]|['"]$/g, '')
|
||||
const classes = new Set<string>()
|
||||
const scan = (child: ts.Node): void => {
|
||||
if (ts.isNewExpression(child) && ts.isIdentifier(child.expression)) {
|
||||
classes.add(child.expression.text)
|
||||
}
|
||||
// `GitHubResource.create(...)` and `DatabricksVolumeResource
|
||||
// .create(...)` are async static factories, so the class never
|
||||
// appears under `new`.
|
||||
if (
|
||||
ts.isPropertyAccessExpression(child) &&
|
||||
child.name.text === 'create' &&
|
||||
ts.isIdentifier(child.expression) &&
|
||||
child.expression.text.endsWith('Resource')
|
||||
) {
|
||||
classes.add(child.expression.text)
|
||||
}
|
||||
ts.forEachChild(child, scan)
|
||||
}
|
||||
scan(prop.initializer)
|
||||
const resourceClasses = [...classes].filter((c) => c.endsWith('Resource'))
|
||||
if (resourceClasses.length === 0 && refuses(prop.initializer)) {
|
||||
// Registered so the name resolves and the error explains why,
|
||||
// but there is no class to read capabilities from — the browser
|
||||
// does this for lancedb (native addon) and email (raw TCP).
|
||||
out.set(name, null)
|
||||
continue
|
||||
}
|
||||
if (resourceClasses.length !== 1) {
|
||||
throw new Error(
|
||||
`registry entry ${name} constructs ${resourceClasses.length} resource classes ` +
|
||||
`(${resourceClasses.join(', ') || 'none'}); the capability dump needs exactly one`,
|
||||
)
|
||||
}
|
||||
out.set(name, resourceClasses[0] as string)
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit)
|
||||
}
|
||||
visit(source)
|
||||
if (out.size === 0) throw new Error(`no REGISTRY object literal found in ${registryFile}`)
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* The wired `CommandIO` slots per backend command directory.
|
||||
*
|
||||
* The adapter's slot set is a hand-filled literal that nothing reads, so
|
||||
* a backend can omit `du` or `find` and quietly fall back to the capped
|
||||
* readdir walk while its twin pushes the work down. Dumping the key set
|
||||
* makes that omission a spec diff.
|
||||
*
|
||||
* Args:
|
||||
* packagesRoot: the `typescript/packages` directory.
|
||||
* pkgs: package names to scan, in the variant's resolution order.
|
||||
*/
|
||||
export function commandIoFacts(
|
||||
packagesRoot: string,
|
||||
pkgs: readonly string[],
|
||||
defaults: { maxGlobMatches: number; maxDuEntries: number },
|
||||
): Record<string, CommandIoFacts> {
|
||||
const out: Record<string, CommandIoFacts> = {}
|
||||
for (const pkg of pkgs) {
|
||||
const root = resolve(packagesRoot, pkg, 'src', 'commands', 'builtin')
|
||||
if (!existsSync(root)) continue
|
||||
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const file = resolve(root, entry.name, 'io.ts')
|
||||
if (!existsSync(file)) continue
|
||||
const source = parse(file)
|
||||
let literal: ts.ObjectLiteralExpression | undefined
|
||||
const visit = (node: ts.Node): void => {
|
||||
if (
|
||||
ts.isVariableDeclaration(node) &&
|
||||
ts.isIdentifier(node.name) &&
|
||||
node.name.text.endsWith('_IO') &&
|
||||
node.initializer !== undefined &&
|
||||
ts.isObjectLiteralExpression(node.initializer)
|
||||
) {
|
||||
if (literal !== undefined) {
|
||||
throw new Error(`${file} declares more than one *_IO object literal`)
|
||||
}
|
||||
literal = node.initializer
|
||||
}
|
||||
ts.forEachChild(node, visit)
|
||||
}
|
||||
visit(source)
|
||||
if (literal === undefined) continue
|
||||
const slots: string[] = []
|
||||
const values: Record<string, number | boolean | string> = {}
|
||||
let readBytes: string | undefined
|
||||
for (const prop of literal.properties) {
|
||||
if (ts.isSpreadAssignment(prop)) {
|
||||
throw new Error(
|
||||
`${file} spreads into its *_IO literal; the slot dump cannot see through it`,
|
||||
)
|
||||
}
|
||||
const name = prop.name?.getText(source)
|
||||
if (name === undefined) continue
|
||||
if (IO_VALUE_FIELDS.has(name)) {
|
||||
let value = ts.isPropertyAssignment(prop) ? literalValue(prop.initializer) : true
|
||||
if (
|
||||
ts.isPropertyAssignment(prop) &&
|
||||
ts.isIdentifier(prop.initializer) &&
|
||||
typeof value === 'string'
|
||||
) {
|
||||
value = resolveIdentifier(source, prop.initializer.text) ?? value
|
||||
}
|
||||
values[name] = value
|
||||
continue
|
||||
}
|
||||
if (
|
||||
name === 'readBytes' &&
|
||||
ts.isPropertyAssignment(prop) &&
|
||||
ts.isIdentifier(prop.initializer)
|
||||
) {
|
||||
readBytes = prop.initializer.text
|
||||
}
|
||||
slots.push(snake(name))
|
||||
}
|
||||
// A reader that already takes a window but no `readRange` slot is
|
||||
// the silent case: the ops factory reads the whole object and
|
||||
// slices, which is correct, quiet, and throws the pushdown away.
|
||||
// Python asserts the same rule from its own signatures
|
||||
// (tests/commands/test_read_range_optin.py); without this the two
|
||||
// sides can only diverge, never be caught.
|
||||
if (
|
||||
!slots.includes('read_range') &&
|
||||
readBytes !== undefined &&
|
||||
takesWindow(source, readBytes)
|
||||
) {
|
||||
throw new Error(
|
||||
`${file}: readBytes takes a byte window but no readRange slot is wired, ` +
|
||||
`so every ranged read downloads the whole object and slices — ` +
|
||||
`add \`readRange: rangeOf(${readBytes})\``,
|
||||
)
|
||||
}
|
||||
out[entry.name] = {
|
||||
slots: slots.sort(),
|
||||
local: values.local === undefined ? true : values.local === true,
|
||||
max_glob_matches: numeric(values.maxGlobMatches, defaults.maxGlobMatches),
|
||||
max_du_entries: numeric(values.maxDuEntries, defaults.maxDuEntries),
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function numeric(value: number | boolean | string | undefined, fallback: number): number | null {
|
||||
if (value === undefined) return fallback
|
||||
return typeof value === 'number' ? value : null
|
||||
}
|
||||
Reference in New Issue
Block a user