Merge pull request #820 from strukto-ai/fix/build-resource-sync
fix: make build_resource sync again, hydrate github lazily
This commit is contained in:
@@ -175,7 +175,7 @@ grep -r "mirage" /slack /gmail /github
|
||||
from mirage.resource.slack import SlackConfig, SlackResource
|
||||
|
||||
slack = SlackResource(SlackConfig(token="xoxb-..."))
|
||||
github = await GitHubResource.build(
|
||||
github = GitHubResource(
|
||||
GitHubConfig(token="github_pat_..."),
|
||||
owner="strukto-ai",
|
||||
repo="mirage",
|
||||
|
||||
@@ -21,7 +21,7 @@ from mirage.core.github.config import GhConfig, GitHubConfig
|
||||
from mirage.resource.github import GitHubResource
|
||||
|
||||
token = os.environ["GITHUB_TOKEN"]
|
||||
repo = await GitHubResource.build(
|
||||
repo = GitHubResource(
|
||||
config=GitHubConfig(token=token), owner="acme", repo="tools", ref="main")
|
||||
|
||||
ws = Workspace({"/repo": repo})
|
||||
|
||||
@@ -17,7 +17,7 @@ from mirage import MountMode, Workspace
|
||||
from mirage.resource.github import GitHubConfig, GitHubResource
|
||||
|
||||
config = GitHubConfig(token=os.environ["GITHUB_TOKEN"])
|
||||
resource = await GitHubResource.build(
|
||||
resource = GitHubResource(
|
||||
config=config, owner="my-org", repo="my-repo", ref="main")
|
||||
ws = Workspace({"/github": resource}, mode=MountMode.READ)
|
||||
```
|
||||
@@ -44,9 +44,8 @@ the path - those are specified at mount time.
|
||||
|
||||
## Tree Fetching
|
||||
|
||||
`GitHubResource.build` fetches the full recursive tree, which is why it is
|
||||
a coroutine — the constructor itself takes that tree and touches no
|
||||
network. For repos with
|
||||
The full recursive tree is fetched on the first read, not at mount time, so
|
||||
constructing the resource never blocks on the network. For repos with
|
||||
|
||||
> 100K entries, it falls back to per-directory fetching.
|
||||
|
||||
@@ -70,7 +69,7 @@ load_dotenv(".env.development")
|
||||
|
||||
async def main():
|
||||
config = GitHubConfig(token=os.environ["GITHUB_TOKEN"])
|
||||
resource = await GitHubResource.build(
|
||||
resource = GitHubResource(
|
||||
config=config, owner="my-org", repo="my-repo", ref="main")
|
||||
ws = Workspace({"/github": resource}, mode=MountMode.READ)
|
||||
|
||||
|
||||
@@ -18,15 +18,14 @@ from mirage import Workspace, MountMode
|
||||
from mirage.resource.github import GitHubConfig, GitHubResource
|
||||
|
||||
config = GitHubConfig(token=os.environ["GITHUB_TOKEN"])
|
||||
resource = await GitHubResource.build(
|
||||
resource = GitHubResource(
|
||||
config=config, owner="my-org", repo="my-repo", ref="main")
|
||||
ws = Workspace({"/github": resource}, mode=MountMode.READ)
|
||||
```
|
||||
|
||||
`build` is a coroutine because it fetches the repo's git tree up front, so
|
||||
call it from async code. Every resource inherits a `build`; only the ones
|
||||
that need I/O to start override it, so `await Resource.build(...)` is
|
||||
uniform.
|
||||
Constructing the mount names the repository and contacts nothing, so it is
|
||||
an ordinary call from sync or async code. The git tree and the default
|
||||
branch are fetched on the first read that needs them.
|
||||
|
||||
## Config Reference
|
||||
|
||||
|
||||
@@ -29,8 +29,10 @@ await ws.execute('cat /repo/README.md')
|
||||
```
|
||||
|
||||
The constructor is private: `create` fetches the repo's git tree before it
|
||||
builds the resource, so it has to be awaited. Python's equivalent is
|
||||
`await GitHubResource.build(...)`.
|
||||
builds the resource, so it has to be awaited. Python deliberately differs
|
||||
here: `GitHubResource(...)` is an ordinary call that contacts nothing, and
|
||||
hydrates the tree on first read, which is what lets its `build_resource`
|
||||
stay synchronous.
|
||||
|
||||
## Browser
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ async def main() -> None:
|
||||
token=os.environ["SLACK_BOT_TOKEN"],
|
||||
search_token=os.environ.get("SLACK_USER_TOKEN"),
|
||||
))
|
||||
github = await GitHubResource.build(
|
||||
github = GitHubResource(
|
||||
config=GitHubConfig(token=os.environ["GITHUB_TOKEN"]),
|
||||
owner=GITHUB_OWNER,
|
||||
repo=GITHUB_REPO,
|
||||
|
||||
@@ -34,7 +34,7 @@ async def _timed(ws, cmd):
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
resource = await GitHubResource.build(
|
||||
resource = GitHubResource(
|
||||
config=config,
|
||||
owner="strukto-ai",
|
||||
repo="mirage",
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
@@ -24,16 +23,14 @@ load_dotenv(".env.development")
|
||||
|
||||
config = GitHubConfig(token=os.environ["GITHUB_TOKEN"])
|
||||
|
||||
# Only the repo fetch is async; the rest of this example is deliberately
|
||||
# synchronous, because that is the point of a FUSE mount. There is no
|
||||
# outer loop here to conflict with.
|
||||
resource = asyncio.run(
|
||||
GitHubResource.build(
|
||||
config=config,
|
||||
owner="strukto-ai",
|
||||
repo="mirage",
|
||||
ref="main",
|
||||
))
|
||||
# Nothing here is async: the mount names the repo and fetches its tree on
|
||||
# the first read, which is the point of a FUSE mount.
|
||||
resource = GitHubResource(
|
||||
config=config,
|
||||
owner="strukto-ai",
|
||||
repo="mirage",
|
||||
ref="main",
|
||||
)
|
||||
|
||||
with Workspace({
|
||||
"/github/":
|
||||
|
||||
@@ -27,7 +27,7 @@ config = GitHubConfig(token=os.environ["GITHUB_TOKEN"])
|
||||
|
||||
|
||||
async def main():
|
||||
resource = await GitHubResource.build(
|
||||
resource = GitHubResource(
|
||||
config=config,
|
||||
owner="strukto-ai",
|
||||
repo="mirage",
|
||||
|
||||
@@ -37,11 +37,9 @@ gdrive_config = GoogleDriveConfig(
|
||||
)
|
||||
github_config = GitHubConfig(token=os.environ["GITHUB_TOKEN"])
|
||||
|
||||
# GitHub fetches the repo tree while it builds, so its resource is the
|
||||
# one mount here that has to be awaited. Each aiohttp session opens and
|
||||
# closes inside that call, so bootstrapping on its own loop is safe.
|
||||
github_resource = asyncio.run(
|
||||
GitHubResource.build(github_config, owner="strukto", repo="mirage"))
|
||||
# GitHub fetches the repo tree on first read, so building the mount is an
|
||||
# ordinary call like every other one here.
|
||||
github_resource = GitHubResource(github_config, owner="strukto", repo="mirage")
|
||||
|
||||
ws = Workspace(
|
||||
{
|
||||
|
||||
+2
-2
@@ -112,7 +112,7 @@ async def yaml_controls_root() -> None:
|
||||
}
|
||||
}
|
||||
}})
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
check("yaml: '/' mount present in resources", "/"
|
||||
in kwargs["resources"])
|
||||
ws = Workspace(**kwargs)
|
||||
@@ -124,7 +124,7 @@ async def yaml_controls_root() -> None:
|
||||
os.path.exists(os.path.join(tmp, "y.txt")))
|
||||
await ws.close()
|
||||
|
||||
ws = Workspace(**await load_config({
|
||||
ws = Workspace(**load_config({
|
||||
"mounts": {
|
||||
"/data": {
|
||||
"resource": "ram"
|
||||
|
||||
@@ -1055,9 +1055,9 @@ class GitHubService:
|
||||
shared with the typescript host. It used to be out of process by
|
||||
necessity — GitHubResource fetched the repo tree with a blocking
|
||||
urlopen from its constructor, which would starve an aiohttp fake on
|
||||
the runner's loop. That constraint is gone now that the fetch is
|
||||
awaited in `GitHubResource.build`; sharing one fake across both hosts
|
||||
is why it stays external.
|
||||
the runner's loop. That constraint is gone now that the constructor
|
||||
touches no network and the tree hydrates on first read; sharing one
|
||||
fake across both hosts is why it stays external.
|
||||
|
||||
Args:
|
||||
url (str): GITHUB_URL origin the fake is listening on.
|
||||
@@ -1082,7 +1082,7 @@ class GitHubService:
|
||||
|
||||
async def resource(self, mount: dict) -> GitHubResource:
|
||||
owner, _, repo = mount["repo"].partition("/")
|
||||
return await GitHubResource.build(
|
||||
return GitHubResource(
|
||||
GitHubConfig(token="ghp-integ",
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
|
||||
@@ -459,7 +459,7 @@ async def build_github(spec: dict) -> Pair | None:
|
||||
repo=GITHUB_REPO,
|
||||
ref=GITHUB_REF,
|
||||
base_url=state.base)
|
||||
resource = await GitHubResource.build(config)
|
||||
resource = GitHubResource(config)
|
||||
ws = Workspace({spec["mount"]: resource}, mode=MountMode.WRITE)
|
||||
return ws, GitHubWriter(config, GITHUB_OWNER, GITHUB_REPO, GITHUB_REF)
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
from mirage.accessor.base import Accessor
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
|
||||
@@ -23,14 +25,23 @@ class GitHubAccessor(Accessor):
|
||||
owner,
|
||||
repo,
|
||||
ref,
|
||||
default_branch,
|
||||
default_branch: str | None = None,
|
||||
tree: dict[str, TreeEntry] | None = None,
|
||||
truncated=False):
|
||||
self.config = config
|
||||
self.owner = owner
|
||||
self.repo = repo
|
||||
self.ref = ref
|
||||
self.default_branch = default_branch
|
||||
# None until hydrated: the mount is constructed without touching
|
||||
# the network, so the repo's default branch is fetched on the
|
||||
# first read that needs it (`ensure_default_branch`).
|
||||
self.default_branch: str | None = default_branch
|
||||
# Guard the two lazy fetches so concurrent first reads make one
|
||||
# request each rather than one per caller. Constructed outside a
|
||||
# running loop on purpose; asyncio.Lock has not bound to a loop
|
||||
# at construction since 3.10.
|
||||
self.tree_lock = asyncio.Lock()
|
||||
self.branch_lock = asyncio.Lock()
|
||||
# The recursive git tree, keyed repo-relative with no leading
|
||||
# slash, which is this mount's whole listing. find, du and grep's
|
||||
# scope counter read it straight, the way TypeScript's always
|
||||
@@ -38,4 +49,11 @@ class GitHubAccessor(Accessor):
|
||||
# index whose keys are the mount's business. Reseated by every
|
||||
# refill, so it is as fresh as the last one.
|
||||
self.tree: dict[str, TreeEntry] = tree if tree is not None else {}
|
||||
# Whether that tree is an answer or just the empty default, which
|
||||
# is not the same question as whether it holds anything: an empty
|
||||
# repository, or one holding only excluded gitlinks, hydrates to
|
||||
# {}. Reading emptiness as "not hydrated yet" made every
|
||||
# direct-tree command refetch such a repo forever, twice per call
|
||||
# once an index was wired.
|
||||
self.tree_loaded: bool = tree is not None
|
||||
self.truncated = truncated
|
||||
|
||||
@@ -22,6 +22,7 @@ from mirage.commands.builtin.github.io import IO, resolve_glob
|
||||
from mirage.commands.config import CommandOpts
|
||||
from mirage.commands.registry import command
|
||||
from mirage.commands.spec import SPECS
|
||||
from mirage.core.github.tree import ensure_tree
|
||||
from mirage.io.types import ByteSource, IOResult
|
||||
from mirage.provision.types import ProvisionResult
|
||||
from mirage.types import PathSpec
|
||||
@@ -76,6 +77,9 @@ async def _stat(accessor: GitHubAccessor, index: IndexCacheStore,
|
||||
@command("du", resource="github", spec=SPECS["du"], provision=du_provision)
|
||||
async def du(accessor: GitHubAccessor, paths: list[PathSpec], texts: list[str],
|
||||
opts: CommandOpts) -> tuple[ByteSource | None, IOResult]:
|
||||
# `_subtree` reads accessor.tree directly rather than the index, so
|
||||
# the tree has to be hydrated first; the mount is built without it.
|
||||
await ensure_tree(accessor, opts.index, opts.mount_prefix)
|
||||
return await du_generic(paths, list(texts), opts,
|
||||
partial(_resolve, accessor, opts.index),
|
||||
partial(_stat, accessor, opts.index),
|
||||
|
||||
@@ -23,6 +23,7 @@ from mirage.commands.registry import command
|
||||
from mirage.commands.spec import SPECS
|
||||
from mirage.core.github.find import find as find_core
|
||||
from mirage.core.github.stat import stat as stat_core
|
||||
from mirage.core.github.tree import ensure_tree
|
||||
from mirage.io.types import ByteSource, IOResult
|
||||
from mirage.provision.types import ProvisionResult
|
||||
from mirage.types import PathSpec
|
||||
@@ -47,6 +48,9 @@ async def find(
|
||||
texts: list[str],
|
||||
opts: CommandOpts,
|
||||
) -> tuple[ByteSource | None, IOResult]:
|
||||
# find walks accessor.tree directly rather than the index, so the
|
||||
# tree has to be hydrated first; the mount is built without it.
|
||||
await ensure_tree(accessor, opts.index, opts.mount_prefix)
|
||||
paths = await resolve_glob(accessor, paths, opts.index)
|
||||
return await find_generic(paths,
|
||||
texts,
|
||||
|
||||
@@ -18,10 +18,13 @@ from mirage.commands.builtin.github.io import resolve_glob
|
||||
from mirage.commands.builtin.grep_helper import (is_literal_pattern,
|
||||
search_query)
|
||||
from mirage.core.github.constants import SCOPE_WARN
|
||||
from mirage.core.github.repo import ensure_default_branch
|
||||
from mirage.core.github.scope import (count_scope_files, scope_relative_key,
|
||||
should_use_search)
|
||||
from mirage.core.github.search import narrow_paths
|
||||
from mirage.core.github.tree import ensure_tree
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
|
||||
|
||||
async def narrow_scope(
|
||||
@@ -67,17 +70,26 @@ async def narrow_scope(
|
||||
search actually narrowed the set.
|
||||
"""
|
||||
key = scope_relative_key(paths[0])
|
||||
# Both facts below are hydrated on first use, not at construction:
|
||||
# the scope count reads the git tree, and the push-down is only
|
||||
# offered on the default branch.
|
||||
await ensure_tree(
|
||||
accessor, index,
|
||||
mount_prefix_of(paths[0].virtual, paths[0].resource_path))
|
||||
file_count = count_scope_files(accessor.tree, key)
|
||||
query = search_query(pattern,
|
||||
fixed_string) if pattern is not None else None
|
||||
literal = (pattern is not None
|
||||
and is_literal_pattern(pattern, fixed_string))
|
||||
# The scope size moved ahead of should_use_search: it is free, and
|
||||
# resolving the default branch is the one term here that can cost a
|
||||
# request.
|
||||
use_search = (query is not None and whole_word and literal
|
||||
and should_use_search(
|
||||
and file_count > SCOPE_WARN and should_use_search(
|
||||
recursive=recursive,
|
||||
on_default_branch=(accessor.ref
|
||||
== accessor.default_branch),
|
||||
) and file_count > SCOPE_WARN)
|
||||
on_default_branch=(accessor.ref == await
|
||||
ensure_default_branch(accessor)),
|
||||
))
|
||||
if use_search:
|
||||
assert query is not None
|
||||
narrowed = await narrow_paths(accessor.config, accessor.owner,
|
||||
|
||||
@@ -547,13 +547,15 @@ class WorkspaceConfig(BaseModel):
|
||||
def _v_cons(cls, v):
|
||||
return _coerce_consistency(v)
|
||||
|
||||
async def to_workspace_kwargs(self) -> dict[str, Any]:
|
||||
def to_workspace_kwargs(self) -> dict[str, Any]:
|
||||
"""Produce kwargs ready to splat into ``Workspace(**kwargs)``.
|
||||
|
||||
Async because building a mount's resource can be: a backend
|
||||
whose setup needs I/O does it in ``BaseResource.build``. Only
|
||||
the resources are awaited — ``Workspace(**kwargs)`` itself stays
|
||||
synchronous. Mirrors the TypeScript ``configToWorkspaceArgs``.
|
||||
Synchronous, and must stay that way: this is the YAML door, and
|
||||
:func:`mirage.resource.registry.build_resource` behind it is the
|
||||
one every embedder calls. A backend needing I/O hydrates lazily
|
||||
instead of moving that cost into construction; see
|
||||
``build_resource`` for the full rule. Deliberately diverges from
|
||||
the TypeScript ``configToWorkspaceArgs``, which is async.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: resource instances, cache config, and
|
||||
@@ -562,7 +564,7 @@ class WorkspaceConfig(BaseModel):
|
||||
"""
|
||||
resources: dict[str, Mount] = {}
|
||||
for prefix, block in self.mounts.items():
|
||||
prov = await build_resource(block.resource, block.config)
|
||||
prov = build_resource(block.resource, block.config)
|
||||
mode = block.mode if block.mode is not None else self.mode
|
||||
resources[prefix] = Mount(
|
||||
resource=prov,
|
||||
|
||||
@@ -37,6 +37,29 @@ async def fetch_default_branch(config: GitHubConfig, owner: str,
|
||||
return data["default_branch"]
|
||||
|
||||
|
||||
async def ensure_default_branch(accessor) -> str:
|
||||
"""Fetch the repo's default branch once, on the first read needing it.
|
||||
|
||||
The mount names a repository without contacting it, so this is the
|
||||
hydration point for the one caller that compares against the default
|
||||
branch (grep's code-search push-down, which GitHub only serves
|
||||
there).
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): the mount's accessor.
|
||||
|
||||
Returns:
|
||||
str: the repository's default branch.
|
||||
"""
|
||||
if accessor.default_branch is not None:
|
||||
return accessor.default_branch
|
||||
async with accessor.branch_lock:
|
||||
if accessor.default_branch is None:
|
||||
accessor.default_branch = await fetch_default_branch(
|
||||
accessor.config, accessor.owner, accessor.repo)
|
||||
return accessor.default_branch
|
||||
|
||||
|
||||
def parse_repo(spec: str) -> RepoRef:
|
||||
"""Split gh's `[HOST/]OWNER/REPO`.
|
||||
|
||||
|
||||
@@ -196,6 +196,7 @@ async def refill_index(accessor, index: IndexCacheStore, prefix: str) -> bool:
|
||||
accessor.repo, accessor.ref)
|
||||
accessor.truncated = truncated
|
||||
accessor.tree = tree
|
||||
accessor.tree_loaded = True
|
||||
seed_index(accessor, index, prefix)
|
||||
return True
|
||||
|
||||
@@ -247,3 +248,47 @@ async def ensure_live_index(accessor, index: IndexCacheStore,
|
||||
if accessor.truncated:
|
||||
return False
|
||||
return await refill_index(accessor, index, prefix)
|
||||
|
||||
|
||||
async def ensure_tree(accessor,
|
||||
index: IndexCacheStore = NULL_INDEX,
|
||||
prefix: str = "") -> None:
|
||||
"""Fetch the recursive tree if this mount has not got one yet.
|
||||
|
||||
The mount is constructed without touching the network, so readers
|
||||
that consult ``accessor.tree`` directly rather than through the
|
||||
index -- find, du and grep's scope counter -- have to hydrate it
|
||||
first. Readers that go through the index do not call this:
|
||||
:func:`ensure_live_index` already refetches for them.
|
||||
|
||||
Prefers that same refill when an index is wired, so a first `find`
|
||||
seeds the index for the `ls` after it instead of fetching a tree
|
||||
only this call can see. Falls back to a bare fetch when there is no
|
||||
index, which is the only case the old build-time fetch was really
|
||||
covering.
|
||||
|
||||
Hydration is tracked by ``tree_loaded``, never by whether the tree
|
||||
holds anything: an empty repository hydrates to ``{}``, and reading
|
||||
that as "not hydrated" refetched it on every call, twice per call
|
||||
once an index was wired (the refill seeds an empty root, then the
|
||||
fallback runs because the tree still looks empty).
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): the mount's accessor.
|
||||
index (IndexCacheStore): the mount's index, when it has one.
|
||||
prefix (str): the mount prefix the index keys are built against.
|
||||
"""
|
||||
if accessor.tree_loaded:
|
||||
return
|
||||
async with accessor.tree_lock:
|
||||
if accessor.tree_loaded:
|
||||
return
|
||||
if index is not NULL_INDEX:
|
||||
await ensure_live_index(accessor, index, prefix)
|
||||
if accessor.tree_loaded:
|
||||
return
|
||||
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, accessor.ref)
|
||||
accessor.truncated = truncated
|
||||
accessor.tree = tree
|
||||
accessor.tree_loaded = True
|
||||
|
||||
@@ -71,6 +71,7 @@ class GitHubWalk:
|
||||
# that reported a CREATE was followed by a find that could not see
|
||||
# the file.
|
||||
accessor.tree = tree
|
||||
accessor.tree_loaded = True
|
||||
stem = root.mount_path.strip("/")
|
||||
base = (stem + "/") if stem else ""
|
||||
for entry in tree.values():
|
||||
|
||||
@@ -141,6 +141,10 @@ async def refresh_access_token(config: GoogleConfig, ) -> tuple[str, int]:
|
||||
Returns:
|
||||
tuple[str, int]: (access_token, expires_in_seconds)
|
||||
"""
|
||||
if config.client_id is None or config.refresh_token is None:
|
||||
raise ValueError(
|
||||
"refresh_access_token needs client_id and refresh_token; this "
|
||||
"config authenticates with a pre-minted access_token")
|
||||
data = {
|
||||
"client_id": config.client_id,
|
||||
"refresh_token": reveal_secret(config.refresh_token),
|
||||
@@ -166,6 +170,15 @@ class TokenManager:
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def get_token(self) -> str:
|
||||
# A supplied token short-circuits the grant entirely, and is read
|
||||
# every call rather than cached: a provider callable is the
|
||||
# caller's own cache, and caching its answer here would outlive
|
||||
# the refresh it just performed. Mirrors _resolve_token in
|
||||
# core/msgraph/_client.py.
|
||||
supplied = self.config.access_token
|
||||
if supplied is not None:
|
||||
return reveal_secret(
|
||||
supplied() if callable(supplied) else supplied)
|
||||
async with self._lock:
|
||||
if self._access_token and time.time() < self._expires_at:
|
||||
return self._access_token
|
||||
|
||||
@@ -12,16 +12,56 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pydantic import BaseModel, SecretStr
|
||||
from collections.abc import Callable
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, SecretStr, model_validator
|
||||
|
||||
|
||||
class GoogleConfig(BaseModel):
|
||||
client_id: str
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
# Two ways to authenticate, the same two MsGraphConfig offers.
|
||||
#
|
||||
# A pre-minted token, either a fixed SecretStr or a provider called
|
||||
# on every request. The provider is how a caller that already owns
|
||||
# the OAuth dance (a service account, a host application's token
|
||||
# source) plugs in: it caches and refreshes on its own, so mirage
|
||||
# holds no long-lived credential and never contacts the token
|
||||
# endpoint. Without it the only way in was `refresh_token`, which a
|
||||
# service account cannot produce; a consumer worked around that by
|
||||
# monkeypatching `refresh_access_token`.
|
||||
access_token: SecretStr | Callable[[], str | SecretStr] | None = None
|
||||
# Or the refresh-token grant, where mirage mints and renews the
|
||||
# access token itself through TokenManager.
|
||||
client_id: str | None = None
|
||||
client_secret: SecretStr | None = None
|
||||
refresh_token: SecretStr
|
||||
refresh_token: SecretStr | None = None
|
||||
# Single-host override for every Google API (drive/docs/sheets/slides)
|
||||
# plus the OAuth token endpoint; used to point backends at a fake server.
|
||||
api_base: str | None = None
|
||||
# Drive-only: scope the mount to this folder ID instead of the Drive
|
||||
# root, the s3 key_prefix analog. Other Google backends ignore it.
|
||||
folder_id: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _one_credential(self) -> "GoogleConfig":
|
||||
"""Refuse a config that names neither way to authenticate.
|
||||
|
||||
Both fields became optional so either grant can stand alone, so
|
||||
this is what keeps a mount from being built with no credential
|
||||
at all and failing on the first read instead.
|
||||
|
||||
Returns:
|
||||
GoogleConfig: the validated config.
|
||||
|
||||
Raises:
|
||||
ValueError: neither an access_token nor a client_id plus
|
||||
refresh_token pair was supplied.
|
||||
"""
|
||||
if self.access_token is not None:
|
||||
return self
|
||||
if self.client_id is not None and self.refresh_token is not None:
|
||||
return self
|
||||
raise ValueError(
|
||||
"GoogleConfig needs either access_token (a token or a provider "
|
||||
"callable) or both client_id and refresh_token")
|
||||
|
||||
@@ -79,35 +79,6 @@ class BaseResource:
|
||||
self._index: IndexCacheStore
|
||||
self.set_index(index)
|
||||
|
||||
@classmethod
|
||||
async def build(cls, *args: Any, **kwargs: Any) -> "BaseResource":
|
||||
"""Construct a resource, awaiting any setup it needs first.
|
||||
|
||||
The default just calls the constructor: most backends open
|
||||
nothing at build time, so there is nothing to await. A backend
|
||||
whose setup needs I/O overrides this and keeps ``__init__``
|
||||
free of network calls — a constructor cannot await, so doing
|
||||
the I/O there means doing it with a blocking client, which
|
||||
stalls whatever event loop the caller runs on.
|
||||
|
||||
Mirrors the TypeScript ``ResourceFactory``
|
||||
(``node/src/resource/registry.ts``), which is uniformly
|
||||
``(config) => Promise<Resource>`` for the same reason. Named
|
||||
``build`` rather than TypeScript's ``create`` because ``create``
|
||||
is already an op name (make an empty file, what ``touch``
|
||||
calls): ops are served by ``__getattr__``, which only runs when
|
||||
normal lookup fails, so a real ``create`` on the class would
|
||||
shadow every backend's create op.
|
||||
|
||||
Args:
|
||||
*args (Any): forwarded to the constructor.
|
||||
**kwargs (Any): forwarded to the constructor.
|
||||
|
||||
Returns:
|
||||
BaseResource: a fresh instance, ready to mount.
|
||||
"""
|
||||
return cls(*args, **kwargs)
|
||||
|
||||
def set_index(self, config: IndexConfig | None = None) -> None:
|
||||
cfg = (config if config is not None else IndexConfig(
|
||||
ttl=self.index_ttl))
|
||||
|
||||
@@ -17,8 +17,6 @@ from typing import Any
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.readdir import readdir
|
||||
from mirage.core.github.repo import fetch_default_branch
|
||||
from mirage.core.github.tree import fetch_tree
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.core.github.watch import build_delta_hook
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -48,31 +46,60 @@ class GitHubResource(BaseResource):
|
||||
def __init__(
|
||||
self,
|
||||
config: GitHubConfig,
|
||||
owner: str,
|
||||
repo: str,
|
||||
ref: str,
|
||||
default_branch: str,
|
||||
tree: dict[str, TreeEntry],
|
||||
owner: str | None = None,
|
||||
repo: str | None = None,
|
||||
ref: str | None = None,
|
||||
default_branch: str | None = None,
|
||||
tree: dict[str, TreeEntry] | None = None,
|
||||
truncated: bool = False,
|
||||
) -> None:
|
||||
"""Build the mount from a tree that has already been fetched.
|
||||
"""Name the repository. Fetch nothing.
|
||||
|
||||
Takes the repo metadata rather than fetching it, so that
|
||||
nothing here touches the network. Use :meth:`create` unless the
|
||||
tree is already in hand.
|
||||
**Do not fetch here, and do not add an async factory in front of
|
||||
this.** A constructor cannot await, so network in one means a
|
||||
blocking client, which stalls whatever event loop the caller is
|
||||
on; the daemon's ``load_workspace`` froze for two GitHub round
|
||||
trips that way. The alternative tried in 0.0.5 was to make
|
||||
:func:`mirage.resource.registry.build_resource` async, which
|
||||
broke every out-of-tree caller for the sake of this one backend.
|
||||
So the tree and the default branch hydrate on first use instead,
|
||||
through ``ensure_tree`` and ``ensure_default_branch``.
|
||||
|
||||
Hydrating lazily also removes a wasted round trip rather than
|
||||
adding one: nothing seeds the index at build time, so the first
|
||||
``readdir`` ran ``ensure_live_index`` and refetched the whole
|
||||
tree anyway, discarding the one fetched here.
|
||||
|
||||
``default_branch``, ``tree`` and ``truncated`` stay accepted so a
|
||||
caller holding the answers (a test, a snapshot restore) can skip
|
||||
the hydration; they are not fetched when omitted.
|
||||
|
||||
Args:
|
||||
config (GitHubConfig): token, base URL and defaults.
|
||||
owner (str): repository owner.
|
||||
repo (str): repository name.
|
||||
ref (str): branch, tag or commit the mount is pinned to.
|
||||
default_branch (str): the repo's default branch, for
|
||||
``is_default_branch``.
|
||||
tree (dict[str, TreeEntry]): the recursive git tree, keyed
|
||||
by repo-relative path.
|
||||
owner (str | None): repository owner; falls back to
|
||||
``config.owner``.
|
||||
repo (str | None): repository name; falls back to
|
||||
``config.repo``.
|
||||
ref (str | None): branch, tag or commit the mount is pinned
|
||||
to; falls back to ``config.ref``.
|
||||
default_branch (str | None): the repo's default branch, for
|
||||
``is_default_branch``. Fetched on first use when None.
|
||||
tree (dict[str, TreeEntry] | None): the recursive git tree,
|
||||
keyed by repo-relative path. Fetched on first use when
|
||||
None.
|
||||
truncated (bool): whether GitHub truncated that tree, in
|
||||
which case readdir falls back to per-directory fetches.
|
||||
|
||||
Raises:
|
||||
ValueError: neither the kwargs nor the config name a repo.
|
||||
"""
|
||||
owner = owner or config.owner
|
||||
repo = repo or config.repo
|
||||
ref = ref or config.ref
|
||||
if owner is None or repo is None:
|
||||
raise ValueError(
|
||||
"GitHubResource requires owner and repo, either as "
|
||||
"constructor kwargs or in GitHubConfig")
|
||||
self.accessor = GitHubAccessor(config,
|
||||
owner,
|
||||
repo,
|
||||
@@ -89,55 +116,6 @@ class GitHubResource(BaseResource):
|
||||
for fn in _github_vfs_ops:
|
||||
self.register_op(fn)
|
||||
|
||||
@classmethod
|
||||
async def build(
|
||||
cls,
|
||||
config: GitHubConfig,
|
||||
owner: str | None = None,
|
||||
repo: str | None = None,
|
||||
ref: str | None = None,
|
||||
) -> "GitHubResource":
|
||||
"""Fetch the repo's tree, then build the mount around it.
|
||||
|
||||
The two GitHub round trips this needs are why construction is
|
||||
async. They used to run in ``__init__`` over a blocking
|
||||
``urlopen``, which froze whatever event loop the caller was on —
|
||||
for the daemon that meant every other mount's in-flight I/O and
|
||||
the FUSE queue stalling for the length of a recursive-tree call.
|
||||
Mirrors the TypeScript ``GitHubResource.create``.
|
||||
|
||||
Args:
|
||||
config (GitHubConfig): token, base URL and defaults.
|
||||
owner (str | None): repository owner; falls back to
|
||||
``config.owner``.
|
||||
repo (str | None): repository name; falls back to
|
||||
``config.repo``.
|
||||
ref (str | None): branch, tag or commit; falls back to
|
||||
``config.ref``.
|
||||
|
||||
Returns:
|
||||
GitHubResource: a mount pinned to ``ref``.
|
||||
|
||||
Raises:
|
||||
ValueError: neither the kwargs nor the config name a repo.
|
||||
"""
|
||||
owner = owner or config.owner
|
||||
repo = repo or config.repo
|
||||
ref = ref or config.ref
|
||||
if owner is None or repo is None:
|
||||
raise ValueError(
|
||||
"GitHubResource requires owner and repo, either as "
|
||||
"build() kwargs or in GitHubConfig")
|
||||
default_branch = await fetch_default_branch(config, owner, repo)
|
||||
tree, truncated = await fetch_tree(config, owner, repo, ref)
|
||||
return cls(config,
|
||||
owner,
|
||||
repo,
|
||||
ref,
|
||||
default_branch,
|
||||
tree,
|
||||
truncated=truncated)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
@@ -145,7 +123,27 @@ class GitHubResource(BaseResource):
|
||||
return await _resolve_glob(self.accessor, paths, self._index)
|
||||
|
||||
@property
|
||||
def is_default_branch(self) -> bool:
|
||||
def is_default_branch(self) -> bool | None:
|
||||
"""Whether the mount is pinned to the repo's default branch.
|
||||
|
||||
``None`` means not known yet, not "no": the default branch is
|
||||
fetched on first use, and until something calls
|
||||
:func:`mirage.core.github.repo.ensure_default_branch` there is
|
||||
nothing to compare ``ref`` against. Answering ``False`` there
|
||||
would be a wrong answer rather than an absent one, and an
|
||||
ordinary read hydrates only the tree, so it could stay wrong for
|
||||
the life of the mount.
|
||||
|
||||
Await ``ensure_default_branch(resource.accessor)`` first when a
|
||||
definite answer is needed. Diverges from the TypeScript
|
||||
``GitHubAccessor.isDefaultBranch``, which is always a bool
|
||||
because construction there fetches the fact.
|
||||
|
||||
Returns:
|
||||
bool | None: the comparison, or None if not yet hydrated.
|
||||
"""
|
||||
if self.accessor.default_branch is None:
|
||||
return None
|
||||
return self.accessor.ref == self.accessor.default_branch
|
||||
|
||||
def get_state(self) -> dict[str, Any]:
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import importlib.metadata
|
||||
import inspect
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
@@ -259,35 +258,8 @@ def resolve_class(ref: str | type) -> type:
|
||||
return ref if isinstance(ref, type) else load_backend_class(ref)
|
||||
|
||||
|
||||
async def _instantiate(resource_cls: type, *args: Any,
|
||||
**kwargs: Any) -> "BaseResource":
|
||||
"""Build one resource, awaiting its factory when it has one.
|
||||
|
||||
:func:`register_resource` and the ``mirage.resources`` entry point
|
||||
both accept any class, not only :class:`BaseResource` subclasses, so
|
||||
the async ``build`` factory is not guaranteed to exist. A class
|
||||
without one is constructed directly — which is how it has always
|
||||
been built, and a plain constructor has nothing to await anyway.
|
||||
The coroutine check also keeps a third-party class that happens to
|
||||
carry a synchronous ``build`` attribute out of the await path.
|
||||
|
||||
Args:
|
||||
resource_cls (type): the class to instantiate.
|
||||
*args (Any): forwarded to ``build`` or the constructor.
|
||||
**kwargs (Any): forwarded to ``build`` or the constructor.
|
||||
|
||||
Returns:
|
||||
BaseResource: the new instance.
|
||||
"""
|
||||
factory = getattr(resource_cls, "build", None)
|
||||
if not inspect.iscoroutinefunction(factory):
|
||||
return resource_cls(*args, **kwargs)
|
||||
return await factory(*args, **kwargs)
|
||||
|
||||
|
||||
async def build_resource(name: str,
|
||||
config: dict[str, Any] | None = None
|
||||
) -> "BaseResource":
|
||||
def build_resource(name: str,
|
||||
config: dict[str, Any] | None = None) -> "BaseResource":
|
||||
"""Construct a resource instance by its registry name.
|
||||
|
||||
Resolves resource and config classes lazily via importlib, so
|
||||
@@ -296,10 +268,25 @@ async def build_resource(name: str,
|
||||
order: builtin ``REGISTRY``, then :func:`register_resource` names,
|
||||
then ``mirage.resources`` entry points from installed packages.
|
||||
|
||||
Async because construction is: backends whose setup needs I/O do it
|
||||
in :meth:`BaseResource.create`, which this awaits. The alternative
|
||||
is a blocking client inside ``__init__``, which stalls the caller's
|
||||
event loop. Mirrors the TypeScript ``buildResource``.
|
||||
**Synchronous on purpose. Do not make this async.** It is the door
|
||||
every caller who describes a mount as data comes through: the YAML
|
||||
loader (:meth:`mirage.config.WorkspaceConfig.to_workspace_kwargs`),
|
||||
the daemon's create/load routes, ``clone``, and every embedder
|
||||
reaching it through ``mirage.sdk``. 0.0.5 made it async to let one
|
||||
backend fetch over the network at build time; that broke every
|
||||
out-of-tree caller, and because nothing validated the return value
|
||||
the failure surfaced as ``'coroutine' object has no attribute
|
||||
'set_index'`` two frames away in ``install_mounts``. Reverted in
|
||||
0.0.6.
|
||||
|
||||
A backend whose setup needs I/O hydrates lazily on first use, the
|
||||
way ``github`` does through ``ensure_tree`` /
|
||||
``ensure_default_branch``, and never from ``__init__``, which cannot
|
||||
await and so would have to block the caller's event loop. This is a
|
||||
deliberate divergence from the TypeScript ``buildResource``, which
|
||||
stays ``Promise<Resource>`` because two of its backends
|
||||
(``github``, ``databricks_volume``) construct through
|
||||
``static async create``.
|
||||
|
||||
Args:
|
||||
name (str): registry key such as ``"s3"`` or ``"ram"``.
|
||||
@@ -327,6 +314,6 @@ async def build_resource(name: str,
|
||||
if config_ref is None:
|
||||
config_ref = getattr(resource_cls, "CONFIG_CLS", None)
|
||||
if config_ref is None:
|
||||
return await _instantiate(resource_cls, **cfg_dict)
|
||||
return resource_cls(**cfg_dict)
|
||||
config_cls = resolve_class(config_ref)
|
||||
return await _instantiate(resource_cls, config_cls(**cfg_dict))
|
||||
return resource_cls(config_cls(**cfg_dict))
|
||||
|
||||
@@ -27,11 +27,32 @@ def reveal_secret(value: Any) -> Any:
|
||||
|
||||
|
||||
def redacted_config_dump(config: BaseModel) -> dict[str, Any]:
|
||||
return _walk_config_dump(config, config.model_dump(mode="json"), True)
|
||||
return _walk_config_dump(config, _base_dump(config), True)
|
||||
|
||||
|
||||
def revealed_config_dump(config: BaseModel) -> dict[str, Any]:
|
||||
return _walk_config_dump(config, config.model_dump(mode="json"), False)
|
||||
return _walk_config_dump(config, _base_dump(config), False)
|
||||
|
||||
|
||||
def _base_dump(config: BaseModel) -> dict[str, Any]:
|
||||
"""Dump every field pydantic can serialize, secrets excluded.
|
||||
|
||||
A credential is not always a ``SecretStr``: ``MsGraphConfig`` and
|
||||
``GoogleConfig`` both accept a provider callable, so the holder of
|
||||
the OAuth dance can hand over a fresh token per request instead of a
|
||||
long-lived one. pydantic cannot serialize a function, so dumping the
|
||||
whole model raised ``PydanticSerializationError`` and took the whole
|
||||
snapshot with it. The walk writes every secret field back off the
|
||||
model anyway, so excluding them here loses nothing.
|
||||
|
||||
Args:
|
||||
config (BaseModel): the resource config being dumped.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: the JSON-mode dump, minus the secret fields.
|
||||
"""
|
||||
return config.model_dump(mode="json",
|
||||
exclude=set(secret_field_names(config)))
|
||||
|
||||
|
||||
def _walk_config_dump(config: BaseModel, data: dict[str, Any],
|
||||
@@ -42,13 +63,25 @@ def _walk_config_dump(config: BaseModel, data: dict[str, Any],
|
||||
# as a real credential and never demands a fresh override.
|
||||
secrets = set(secret_field_names(config))
|
||||
for name in type(config).model_fields:
|
||||
if name not in data:
|
||||
# A secret is absent from `data` by construction (_base_dump
|
||||
# excludes them) and is written back below off the model.
|
||||
if name not in data and name not in secrets:
|
||||
continue
|
||||
value = getattr(config, name)
|
||||
if name in secrets:
|
||||
if value is None:
|
||||
continue
|
||||
data[name] = REDACTED_SECRET if redact else reveal_secret(value)
|
||||
data[name] = None
|
||||
elif callable(value):
|
||||
# A provider callable has no serialized form, and
|
||||
# revealing it would mean calling it and freezing one
|
||||
# token into a snapshot that outlives it. Redacted in
|
||||
# both modes, which is already the contract that makes
|
||||
# `requires_resource_override` demand a fresh resource
|
||||
# at load.
|
||||
data[name] = REDACTED_SECRET
|
||||
else:
|
||||
data[name] = REDACTED_SECRET if redact else reveal_secret(
|
||||
value)
|
||||
elif isinstance(value, BaseModel):
|
||||
data[name] = _walk_config_dump(value, data[name], redact)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
|
||||
@@ -21,7 +21,7 @@ from mirage.workspace.snapshot import requires_resource_override, to_state_dict
|
||||
from mirage.workspace.snapshot.utils import norm_mount_prefix
|
||||
|
||||
|
||||
async def _build_override_resources(
|
||||
def _build_override_resources(
|
||||
override: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not override:
|
||||
return {}
|
||||
@@ -35,8 +35,7 @@ async def _build_override_resources(
|
||||
config = block.get("config") or {}
|
||||
if resource_name is None:
|
||||
continue
|
||||
out[norm_mount_prefix(prefix)] = await build_resource(
|
||||
resource_name, config)
|
||||
out[norm_mount_prefix(prefix)] = build_resource(resource_name, config)
|
||||
return out
|
||||
|
||||
|
||||
@@ -82,7 +81,7 @@ async def clone_workspace_with_override(src_ws: Workspace,
|
||||
Workspace: a new, independent workspace.
|
||||
"""
|
||||
state = await to_state_dict(src_ws)
|
||||
override_resources = await _build_override_resources(override)
|
||||
override_resources = _build_override_resources(override)
|
||||
existing = _existing_redacted_resources(src_ws,
|
||||
state,
|
||||
skip=set(override_resources))
|
||||
|
||||
@@ -44,7 +44,7 @@ async def create_workspace(req: CreateWorkspaceRequest,
|
||||
# Map runtime entries construct their instances here, so a bad
|
||||
# entry (a wasi build dir that does not exist, an unknown
|
||||
# option) fails the create like any other config mistake.
|
||||
kwargs = await req.config.to_workspace_kwargs()
|
||||
kwargs = req.config.to_workspace_kwargs()
|
||||
except (FileNotFoundError, ImportError, ValueError, TypeError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
# The registry id and the state-store scope must be the same identity,
|
||||
@@ -161,7 +161,7 @@ async def load_workspace(req: LoadWorkspaceRequest,
|
||||
if req.id is not None and req.id in registry:
|
||||
raise HTTPException(status_code=409,
|
||||
detail=f"workspace id already exists: {req.id!r}")
|
||||
resources = await _build_load_resources(req.override)
|
||||
resources = _build_load_resources(req.override)
|
||||
try:
|
||||
ws = await Workspace.load(str(safe_path), resources=resources)
|
||||
except FileNotFoundError:
|
||||
@@ -176,7 +176,7 @@ async def load_workspace(req: LoadWorkspaceRequest,
|
||||
return await make_detail(entry)
|
||||
|
||||
|
||||
async def _build_load_resources(
|
||||
def _build_load_resources(
|
||||
override: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not override or "mounts" not in override:
|
||||
return None
|
||||
@@ -188,6 +188,5 @@ async def _build_load_resources(
|
||||
config = block.get("config") or {}
|
||||
if resource_name is None:
|
||||
continue
|
||||
out[norm_mount_prefix(prefix)] = await build_resource(
|
||||
resource_name, config)
|
||||
out[norm_mount_prefix(prefix)] = build_resource(resource_name, config)
|
||||
return out or None
|
||||
|
||||
@@ -12,8 +12,11 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import inspect
|
||||
|
||||
from mirage.cache.index import IndexConfig
|
||||
from mirage.ops import Ops
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.history import HISTORY_PREFIX
|
||||
from mirage.resource.ram import RAMResource
|
||||
from mirage.types import KERNEL_BACKENDS, MountBackend, MountMode
|
||||
@@ -22,6 +25,37 @@ from mirage.workspace.mount.spec import Mount
|
||||
from mirage.workspace.workspace.types import MountSpec, ResourceMount
|
||||
|
||||
|
||||
def check_resource(prefix: str, resource: BaseResource) -> None:
|
||||
"""Refuse a mount value that is not a resource, naming the mount.
|
||||
|
||||
``MountSpec.resource`` is annotated ``BaseResource`` but the class is
|
||||
a plain dataclass, so nothing enforced it and the wrong value rode
|
||||
all the way to ``install_mounts``, where it surfaced as
|
||||
``'X' object has no attribute 'set_index'`` -- a method the caller
|
||||
never called, in a file they never touched, with no mount named.
|
||||
|
||||
The coroutine arm is spelled out because it is the mistake this
|
||||
function exists for: 0.0.5 briefly made ``build_resource`` async, so
|
||||
every caller written against 0.0.3/0.0.4 handed the mount table an
|
||||
un-awaited coroutine.
|
||||
|
||||
Args:
|
||||
prefix (str): the mount point the value was given for.
|
||||
resource (BaseResource): the value to check.
|
||||
|
||||
Raises:
|
||||
TypeError: ``resource`` is a coroutine or not a BaseResource.
|
||||
"""
|
||||
if inspect.iscoroutine(resource):
|
||||
raise TypeError(
|
||||
f"mount {prefix!r}: got a coroutine, not a resource. "
|
||||
"build_resource() is synchronous; if you wrote "
|
||||
"`await build_resource(...)` against 0.0.5, drop the await.")
|
||||
if not isinstance(resource, BaseResource):
|
||||
raise TypeError(f"mount {prefix!r}: expected a BaseResource, got "
|
||||
f"{type(resource).__name__}")
|
||||
|
||||
|
||||
def normalize_resources(resources: dict[str, ResourceMount],
|
||||
default_mode: MountMode) -> list[MountSpec]:
|
||||
"""Narrow every accepted ``resources`` spelling to one shape.
|
||||
@@ -32,7 +66,8 @@ def normalize_resources(resources: dict[str, ResourceMount],
|
||||
|
||||
Raises:
|
||||
TypeError: a tuple entry is not (resource, mode) or
|
||||
(resource, mode, command_limits).
|
||||
(resource, mode, command_limits), or an entry's resource is
|
||||
not a :class:`BaseResource`.
|
||||
"""
|
||||
specs: list[MountSpec] = []
|
||||
for prefix, value in resources.items():
|
||||
@@ -61,6 +96,8 @@ def normalize_resources(resources: dict[str, ResourceMount],
|
||||
else:
|
||||
specs.append(
|
||||
MountSpec(prefix=prefix, resource=value, mode=default_mode))
|
||||
for spec in specs:
|
||||
check_resource(spec.prefix, spec.resource)
|
||||
return specs
|
||||
|
||||
|
||||
|
||||
@@ -69,7 +69,11 @@ build-backend = "setuptools.build_meta"
|
||||
include = ["mirage*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
mirage = ["**/*.lua"]
|
||||
# py.typed is PEP 561 consent: without the marker a type checker refuses
|
||||
# to read an installed package's annotations and every mirage import
|
||||
# resolves to Any, so our own gated-at-zero mypy proves nothing to anyone
|
||||
# downstream. It caught the 0.0.5 build_resource break in one line.
|
||||
mirage = ["**/*.lua", "py.typed"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# --- python runtime (sandboxed python3 default) ---
|
||||
|
||||
@@ -26,7 +26,7 @@ REF = "main"
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def github_env(mock_github_api, github_config):
|
||||
resource = await GitHubResource.build(
|
||||
resource = GitHubResource(
|
||||
config=github_config,
|
||||
owner=OWNER,
|
||||
repo=REPO,
|
||||
|
||||
@@ -67,7 +67,7 @@ async def test_load_full_yaml_with_env_interpolation():
|
||||
assert cfg.kernel_mounts() == {
|
||||
"/": (MountBackend.FUSE, "/tmp/mirage-fuse-full")
|
||||
}
|
||||
assert "kernel_mounts" not in await cfg.to_workspace_kwargs()
|
||||
assert "kernel_mounts" not in cfg.to_workspace_kwargs()
|
||||
|
||||
|
||||
def test_missing_env_var_raises_with_full_list():
|
||||
@@ -85,7 +85,7 @@ def test_redis_cache_discriminated_union():
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_workspace_kwargs_yields_constructible_workspace():
|
||||
cfg = load_config(FIXTURES / "minimal.yaml")
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert "/" in kwargs["resources"]
|
||||
mount = kwargs["resources"]["/"]
|
||||
assert isinstance(mount.resource, RAMResource)
|
||||
@@ -97,7 +97,7 @@ async def test_to_workspace_kwargs_yields_constructible_workspace():
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_workspace_kwargs_emits_redis_cache_config():
|
||||
cfg = load_config(FIXTURES / "redis_cache.yaml")
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert isinstance(kwargs["cache"], RedisCacheConfig)
|
||||
assert kwargs["cache"].url == "redis://localhost:6379/3"
|
||||
|
||||
@@ -115,7 +115,7 @@ async def test_to_workspace_kwargs_emits_ram_cache_config():
|
||||
}
|
||||
},
|
||||
})
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert isinstance(kwargs["cache"], CacheConfig)
|
||||
assert not isinstance(kwargs["cache"], RedisCacheConfig)
|
||||
assert kwargs["cache"].limit == "128MB"
|
||||
@@ -137,7 +137,7 @@ async def test_store_redis_block_builds_redis_provider():
|
||||
})
|
||||
assert cfg.store is not None
|
||||
assert cfg.store.key_prefix == "test_store:"
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert isinstance(kwargs["store"], RedisWorkspaceStateStore)
|
||||
assert isinstance(kwargs["store"].namespace("ws1"), RedisNamespaceStore)
|
||||
|
||||
@@ -154,7 +154,7 @@ async def test_store_ram_block_builds_ram_provider():
|
||||
}
|
||||
},
|
||||
})
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert isinstance(kwargs["store"], RAMWorkspaceStateStore)
|
||||
assert kwargs["owns_store"] is True
|
||||
assert isinstance(kwargs["store"].namespace("ws1"), RAMNamespaceStore)
|
||||
@@ -175,7 +175,7 @@ async def test_store_disk_block_builds_disk_provider(tmp_path):
|
||||
})
|
||||
assert cfg.store is not None
|
||||
assert cfg.store.root == str(tmp_path)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert isinstance(kwargs["store"], DiskWorkspaceStateStore)
|
||||
assert kwargs["owns_store"] is True
|
||||
assert isinstance(kwargs["store"].namespace("ws1"), DiskNamespaceStore)
|
||||
@@ -198,7 +198,7 @@ async def test_store_disk_group_override(tmp_path):
|
||||
},
|
||||
})
|
||||
assert isinstance(cfg.store.workspace, DiskStoreBlock)
|
||||
store = (await cfg.to_workspace_kwargs())["store"]
|
||||
store = cfg.to_workspace_kwargs()["store"]
|
||||
assert isinstance(store, RAMWorkspaceStateStore)
|
||||
assert isinstance(store.sessions("ws1"), DiskSessionStore)
|
||||
|
||||
@@ -221,7 +221,7 @@ async def test_store_group_override_redirects_one_plane():
|
||||
},
|
||||
})
|
||||
assert isinstance(cfg.store.observer, RedisStoreBlock)
|
||||
store = (await cfg.to_workspace_kwargs())["store"]
|
||||
store = cfg.to_workspace_kwargs()["store"]
|
||||
assert isinstance(store, RAMWorkspaceStateStore)
|
||||
assert isinstance(store.namespace("ws1"), RAMNamespaceStore)
|
||||
assert type(store.observer("ws1")).__name__ == "RedisObserverStore"
|
||||
@@ -246,7 +246,7 @@ async def test_store_s3_workspace_group_builds_s3_provider():
|
||||
},
|
||||
})
|
||||
assert isinstance(cfg.store.workspace, S3StoreBlock)
|
||||
store = (await cfg.to_workspace_kwargs())["store"]
|
||||
store = cfg.to_workspace_kwargs()["store"]
|
||||
assert isinstance(store, RAMWorkspaceStateStore)
|
||||
assert isinstance(store.namespace("ws1"), RAMNamespaceStore)
|
||||
assert type(store.sessions("ws1")).__name__ == "S3SessionStore"
|
||||
@@ -262,7 +262,7 @@ async def test_workspace_id_passes_through():
|
||||
}
|
||||
},
|
||||
})
|
||||
assert (await cfg.to_workspace_kwargs())["workspace_id"] == "agent-ws-7"
|
||||
assert cfg.to_workspace_kwargs()["workspace_id"] == "agent-ws-7"
|
||||
|
||||
|
||||
def test_store_block_rejects_unknown_field():
|
||||
@@ -300,7 +300,7 @@ def test_unknown_mount_field_rejected():
|
||||
@pytest.mark.asyncio
|
||||
async def test_workspace_built_from_config_executes_command():
|
||||
cfg = load_config(FIXTURES / "minimal.yaml")
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
ws = Workspace(**kwargs)
|
||||
result = await ws.execute("echo hello")
|
||||
assert result.exit_code == 0
|
||||
@@ -338,7 +338,7 @@ async def test_resource_built_via_registry_has_correct_type():
|
||||
},
|
||||
},
|
||||
})
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
mount = kwargs["resources"]["/s3"]
|
||||
assert isinstance(mount.resource, S3Resource)
|
||||
assert mount.mode == MountMode.READ
|
||||
@@ -360,7 +360,7 @@ runtimes:
|
||||
- vfs
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert kwargs["policy"] == ScriptSource("'local'")
|
||||
entry = kwargs["runtimes"][0]
|
||||
assert entry.script == ScriptSource("ctx['command'] == 'python3'")
|
||||
@@ -377,7 +377,7 @@ mounts:
|
||||
policy: policy.js
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert kwargs["policy"] == ScriptSource("null", language="js")
|
||||
assert kwargs["policy"].language == "js"
|
||||
|
||||
@@ -397,7 +397,7 @@ guards:
|
||||
commands: [python3]
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert kwargs["guards"] == [
|
||||
GuardSpec(reason="production data is protected",
|
||||
commands=("rm", "mv"),
|
||||
@@ -426,7 +426,7 @@ async def test_clis_section_parses_and_maps_to_kwargs():
|
||||
},
|
||||
},
|
||||
})
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert kwargs["clis"] == {
|
||||
"sl": ("slack", {
|
||||
"token": "x"
|
||||
@@ -451,7 +451,7 @@ clis:
|
||||
page_size: 20
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
spec, config = kwargs["clis"]["pager"]
|
||||
assert spec.name == "pager"
|
||||
assert spec.script == ScriptSource("print('page')")
|
||||
@@ -472,7 +472,7 @@ clis:
|
||||
script: pager.mjs
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
spec, _ = kwargs["clis"]["pager"]
|
||||
# .mjs also stamps module: the path is gone once the source is
|
||||
# embedded, so the engine could not otherwise know to run it as an
|
||||
@@ -494,8 +494,7 @@ clis:
|
||||
pager:
|
||||
script: pager.js
|
||||
""")
|
||||
spec, _ = (await
|
||||
load_config(cfg_file).to_workspace_kwargs())["clis"]["pager"]
|
||||
spec, _ = load_config(cfg_file).to_workspace_kwargs()["clis"]["pager"]
|
||||
assert spec.script.language == "js"
|
||||
assert spec.script.module is False
|
||||
|
||||
@@ -521,7 +520,7 @@ clis:
|
||||
""")
|
||||
monkeypatch.chdir(tmp_path.parent)
|
||||
cfg = load_config(cfg_file)
|
||||
ref, _ = (await cfg.to_workspace_kwargs())["clis"]["tool"]
|
||||
ref, _ = cfg.to_workspace_kwargs()["clis"]["tool"]
|
||||
assert ref == f"{tmp_path / 'tool.py'}:TREE"
|
||||
|
||||
|
||||
@@ -539,7 +538,7 @@ clis:
|
||||
cli: mypkg.clis:TREE
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
ref, _ = (await cfg.to_workspace_kwargs())["clis"]["tool"]
|
||||
ref, _ = cfg.to_workspace_kwargs()["clis"]["tool"]
|
||||
assert ref == "mypkg.clis:TREE"
|
||||
|
||||
|
||||
@@ -555,7 +554,7 @@ clis:
|
||||
cli: slack
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
ref, _ = (await cfg.to_workspace_kwargs())["clis"]["sl"]
|
||||
ref, _ = cfg.to_workspace_kwargs()["clis"]["sl"]
|
||||
assert ref == "slack"
|
||||
|
||||
|
||||
@@ -572,7 +571,7 @@ clis:
|
||||
""")
|
||||
cfg = load_config(cfg_file)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await cfg.to_workspace_kwargs()
|
||||
cfg.to_workspace_kwargs()
|
||||
|
||||
|
||||
def test_clis_entry_takes_exactly_one_of_cli_or_script():
|
||||
@@ -660,7 +659,7 @@ async def test_console_redis_block_builds_factory():
|
||||
}
|
||||
},
|
||||
})
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
factory = kwargs["console_factory"]
|
||||
first = factory(1)
|
||||
second = factory(1)
|
||||
@@ -688,7 +687,7 @@ async def test_console_ram_block_emits_no_factory():
|
||||
}
|
||||
},
|
||||
})
|
||||
assert "console_factory" not in await cfg.to_workspace_kwargs()
|
||||
assert "console_factory" not in cfg.to_workspace_kwargs()
|
||||
|
||||
|
||||
def test_shared_rejection_fixture_is_refused():
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import pytest
|
||||
from pydantic import SecretStr, ValidationError
|
||||
|
||||
from mirage.core.google._client import TokenManager
|
||||
from mirage.core.google.config import GoogleConfig
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_supplied_token_skips_the_refresh_grant(monkeypatch):
|
||||
# A service account can mint an access token but has no refresh
|
||||
# token, so before this the only way in was to monkeypatch
|
||||
# refresh_access_token.
|
||||
async def _boom(config):
|
||||
raise AssertionError("refresh grant must not run")
|
||||
|
||||
monkeypatch.setattr("mirage.core.google._client.refresh_access_token",
|
||||
_boom)
|
||||
config = GoogleConfig(access_token=SecretStr("sa-token"))
|
||||
assert await TokenManager(config).get_token() == "sa-token"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_provider_is_called_every_request():
|
||||
# The provider owns the refresh, so caching its answer here would
|
||||
# outlive the rotation it just performed.
|
||||
tokens = iter(["tok-1", "tok-2", "tok-3"])
|
||||
manager = TokenManager(GoogleConfig(access_token=lambda: next(tokens)))
|
||||
assert [await manager.get_token()
|
||||
for _ in range(3)] == ["tok-1", "tok-2", "tok-3"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_provider_may_answer_with_a_secret():
|
||||
manager = TokenManager(
|
||||
GoogleConfig(access_token=lambda: SecretStr("wrapped")))
|
||||
assert await manager.get_token() == "wrapped"
|
||||
|
||||
|
||||
def test_the_refresh_grant_is_still_accepted():
|
||||
config = GoogleConfig(client_id="cid", refresh_token=SecretStr("rt"))
|
||||
assert config.access_token is None
|
||||
|
||||
|
||||
def test_naming_no_credential_is_refused():
|
||||
with pytest.raises(ValidationError, match="either access_token"):
|
||||
GoogleConfig(api_base="http://localhost:1")
|
||||
@@ -139,7 +139,7 @@ async def test_yaml_clis_section_installs_through_load_config():
|
||||
}
|
||||
},
|
||||
})
|
||||
ws = Workspace(**await cfg.to_workspace_kwargs())
|
||||
ws = Workspace(**cfg.to_workspace_kwargs())
|
||||
code, out, _ = await run(ws, "sl message send -t x hi")
|
||||
assert (code, out) == (0, b"sent[yaml] to=x: hi\n")
|
||||
await ws.close()
|
||||
@@ -177,7 +177,7 @@ async def test_yaml_cli_reference_form_installs(tmp_path):
|
||||
}
|
||||
},
|
||||
})
|
||||
ws = Workspace(**await cfg.to_workspace_kwargs())
|
||||
ws = Workspace(**cfg.to_workspace_kwargs())
|
||||
code, out, _ = await run(ws, "sl send")
|
||||
assert (code, out) == (0, b"sent[ref]\n")
|
||||
await ws.close()
|
||||
@@ -198,7 +198,7 @@ async def test_yaml_unknown_cli_key_fails_loud():
|
||||
},
|
||||
})
|
||||
with pytest.raises(ValueError, match="unknown cli 'nope'"):
|
||||
Workspace(**await cfg.to_workspace_kwargs())
|
||||
Workspace(**cfg.to_workspace_kwargs())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -374,7 +374,7 @@ async def test_yaml_script_entry_executes_end_to_end(tmp_path):
|
||||
}
|
||||
},
|
||||
})
|
||||
ws = Workspace(**await cfg.to_workspace_kwargs())
|
||||
ws = Workspace(**cfg.to_workspace_kwargs())
|
||||
code, out, _ = await run(ws, "pager report.txt")
|
||||
assert (code, out) == (0, b'yaml report.txt {"width": 80}\n')
|
||||
await ws.close()
|
||||
|
||||
Vendored
+5
-6
@@ -127,13 +127,12 @@ def mock_github_api(monkeypatch):
|
||||
results = [r for r in results if r.path.startswith(path_filter)]
|
||||
return results
|
||||
|
||||
monkeypatch.setattr("mirage.resource.github.github.fetch_default_branch",
|
||||
# Both are patched in the module that fetches, because the mount is
|
||||
# built without touching the network and hydrates on first use:
|
||||
# `ensure_tree` and `refill_index` call fetch_tree in tree.py, and
|
||||
# `ensure_default_branch` calls fetch_default_branch in repo.py.
|
||||
monkeypatch.setattr("mirage.core.github.repo.fetch_default_branch",
|
||||
_fetch_default_branch)
|
||||
monkeypatch.setattr("mirage.resource.github.github.fetch_tree",
|
||||
_fetch_tree)
|
||||
# refill_index reads the name in its own module, and an empty index is
|
||||
# a refill trigger now, so leaving this one real let a test reach the
|
||||
# live API.
|
||||
monkeypatch.setattr("mirage.core.github.tree.fetch_tree", _fetch_tree)
|
||||
monkeypatch.setattr("mirage.core.github.read.read_bytes", _read_bytes)
|
||||
monkeypatch.setattr("mirage.core.github.search.search_code", _search_code)
|
||||
|
||||
@@ -12,7 +12,7 @@ async def test_chroma_resource_is_registered():
|
||||
assert REGISTRY[
|
||||
"chroma"].config_path == "mirage.resource.chroma:ChromaConfig"
|
||||
|
||||
resource = await build_resource("chroma", {"collection_name": "docs"})
|
||||
resource = build_resource("chroma", {"collection_name": "docs"})
|
||||
|
||||
assert resource.name == ResourceName.CHROMA
|
||||
assert resource.caches_reads is False
|
||||
@@ -24,7 +24,7 @@ async def test_chroma_resource_is_registered():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chroma_resource_registers_expected_commands_and_ops():
|
||||
resource = await build_resource("chroma", {"collection_name": "docs"})
|
||||
resource = build_resource("chroma", {"collection_name": "docs"})
|
||||
|
||||
commands = {item.name for item in resource.commands()}
|
||||
ops = {item.name for item in resource.ops_list()}
|
||||
|
||||
@@ -13,7 +13,7 @@ async def test_dify_resource_is_registered_and_redacts_api_key():
|
||||
"dify"].resource_path == "mirage.resource.dify:DifyResource"
|
||||
assert REGISTRY["dify"].config_path == "mirage.resource.dify:DifyConfig"
|
||||
|
||||
resource = await build_resource(
|
||||
resource = build_resource(
|
||||
"dify",
|
||||
{
|
||||
"api_key": "dataset-secret",
|
||||
@@ -48,7 +48,7 @@ async def test_dify_resource_is_registered_and_redacts_api_key():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dify_resource_accepts_configured_slug_metadata_name():
|
||||
resource = await build_resource(
|
||||
resource = build_resource(
|
||||
"dify",
|
||||
{
|
||||
"api_key": "dataset-secret",
|
||||
@@ -84,7 +84,7 @@ def test_dify_config_rejects_non_positive_request_limits(field):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dify_resource_registers_expected_commands_and_ops():
|
||||
resource = await build_resource(
|
||||
resource = build_resource(
|
||||
"dify",
|
||||
{
|
||||
"api_key": "dataset-secret",
|
||||
@@ -103,7 +103,7 @@ async def test_dify_resource_registers_expected_commands_and_ops():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dify_resource_close_closes_shared_client():
|
||||
resource = await build_resource(
|
||||
resource = build_resource(
|
||||
"dify",
|
||||
{
|
||||
"api_key": "dataset-secret",
|
||||
|
||||
@@ -58,7 +58,7 @@ def test_state_does_not_leak_secrets():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_builds_dropbox():
|
||||
resource = await build_resource(
|
||||
resource = build_resource(
|
||||
"dropbox", {
|
||||
"client_id": "c",
|
||||
"client_secret": "s",
|
||||
|
||||
@@ -12,13 +12,16 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.cache.index import IndexConfig
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.repo import ensure_default_branch
|
||||
from mirage.core.github.stat import stat
|
||||
from mirage.core.github.tree import ensure_tree
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.resource.github.github import GitHubResource
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
@@ -31,133 +34,123 @@ REPO = "test-repo"
|
||||
def _offline(tree: dict,
|
||||
truncated: bool = False,
|
||||
default_branch: str = "main"):
|
||||
"""Patch every path that would reach the network.
|
||||
"""Patch both paths that would reach the network.
|
||||
|
||||
``mirage.core.github.tree.fetch_tree`` is the second one: a read fills
|
||||
the index by refetching, because the tree a mount was built with is
|
||||
only true at build time.
|
||||
Each is patched in the module that fetches, because the mount is
|
||||
built without touching either: the tree hydrates through
|
||||
``ensure_tree`` / ``refill_index`` in ``core.github.tree``, and the
|
||||
default branch through ``ensure_default_branch`` in
|
||||
``core.github.repo``.
|
||||
|
||||
Args:
|
||||
tree (dict): The recursive tree to answer with.
|
||||
truncated (bool): Whether to report it truncated.
|
||||
default_branch (str): Branch the repo endpoint reports.
|
||||
"""
|
||||
return (patch("mirage.resource.github.github.fetch_default_branch",
|
||||
return (patch("mirage.core.github.repo.fetch_default_branch",
|
||||
return_value=default_branch),
|
||||
patch("mirage.resource.github.github.fetch_tree",
|
||||
return_value=(tree, truncated)),
|
||||
patch("mirage.core.github.tree.fetch_tree",
|
||||
return_value=(tree, truncated)))
|
||||
|
||||
|
||||
async def _make_resource(ref: str = "main",
|
||||
default_branch: str = "main",
|
||||
tree: dict | None = None,
|
||||
truncated: bool = False) -> GitHubResource:
|
||||
if tree is None:
|
||||
tree = {}
|
||||
branch_p, build_p, core_p = _offline(tree, truncated, default_branch)
|
||||
with branch_p, build_p, core_p:
|
||||
return await GitHubResource.build(
|
||||
config=CONFIG,
|
||||
owner=OWNER,
|
||||
repo=REPO,
|
||||
ref=ref,
|
||||
)
|
||||
def _make_resource(ref: str = "main",
|
||||
default_branch: str | None = "main",
|
||||
tree: dict | None = None,
|
||||
truncated: bool = False) -> GitHubResource:
|
||||
return GitHubResource(
|
||||
config=CONFIG,
|
||||
owner=OWNER,
|
||||
repo=REPO,
|
||||
ref=ref,
|
||||
default_branch=default_branch,
|
||||
tree=tree,
|
||||
truncated=truncated,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_name() -> None:
|
||||
resource = await _make_resource()
|
||||
def test_name() -> None:
|
||||
resource = _make_resource()
|
||||
assert resource.name == ResourceName.GITHUB
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_caches_reads() -> None:
|
||||
resource = await _make_resource()
|
||||
def test_caches_reads() -> None:
|
||||
resource = _make_resource()
|
||||
assert resource.caches_reads is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bind_args() -> None:
|
||||
resource = await _make_resource()
|
||||
def test_bind_args() -> None:
|
||||
resource = _make_resource()
|
||||
assert resource.accessor.config is CONFIG
|
||||
assert resource.accessor.owner == OWNER
|
||||
assert resource.accessor.repo == REPO
|
||||
assert resource.accessor.ref == "main"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_owner_repo_ref_fall_back_to_config() -> None:
|
||||
def test_owner_repo_ref_fall_back_to_config() -> None:
|
||||
config = GitHubConfig(token="test-token",
|
||||
owner="cfg-owner",
|
||||
repo="cfg-repo",
|
||||
ref="cfg-ref")
|
||||
with patch("mirage.resource.github.github.fetch_default_branch",
|
||||
return_value="main"), \
|
||||
patch("mirage.resource.github.github.fetch_tree",
|
||||
return_value=({}, False)):
|
||||
resource = await GitHubResource.build(config=config)
|
||||
resource = GitHubResource(config=config)
|
||||
assert resource.accessor.owner == "cfg-owner"
|
||||
assert resource.accessor.repo == "cfg-repo"
|
||||
assert resource.accessor.ref == "cfg-ref"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kwargs_take_precedence_over_config() -> None:
|
||||
def test_kwargs_take_precedence_over_config() -> None:
|
||||
config = GitHubConfig(token="test-token",
|
||||
owner="cfg-owner",
|
||||
repo="cfg-repo",
|
||||
ref="cfg-ref")
|
||||
with patch("mirage.resource.github.github.fetch_default_branch",
|
||||
return_value="main"), \
|
||||
patch("mirage.resource.github.github.fetch_tree",
|
||||
return_value=({}, False)):
|
||||
resource = await GitHubResource.build(config=config,
|
||||
owner="kw-owner",
|
||||
repo="kw-repo",
|
||||
ref="kw-ref")
|
||||
resource = GitHubResource(config=config,
|
||||
owner="kw-owner",
|
||||
repo="kw-repo",
|
||||
ref="kw-ref")
|
||||
assert resource.accessor.owner == "kw-owner"
|
||||
assert resource.accessor.repo == "kw-repo"
|
||||
assert resource.accessor.ref == "kw-ref"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_owner_repo_raises() -> None:
|
||||
def test_missing_owner_repo_raises() -> None:
|
||||
with pytest.raises(ValueError, match="requires owner and repo"):
|
||||
await GitHubResource.build(config=GitHubConfig(token="test-token"))
|
||||
GitHubResource(config=GitHubConfig(token="test-token"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_owner_repo_refuses_before_any_fetch() -> None:
|
||||
# The guard runs first, so a misconfigured mount costs no API call.
|
||||
with patch("mirage.resource.github.github.fetch_default_branch") as branch:
|
||||
with pytest.raises(ValueError, match="requires owner and repo"):
|
||||
await GitHubResource.build(config=GitHubConfig(token="test-token"))
|
||||
branch.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_default_branch_true() -> None:
|
||||
resource = await _make_resource(ref="main", default_branch="main")
|
||||
def test_is_default_branch_true() -> None:
|
||||
resource = _make_resource(ref="main", default_branch="main")
|
||||
assert resource.is_default_branch is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_default_branch_false() -> None:
|
||||
resource = await _make_resource(ref="feature-branch",
|
||||
default_branch="main")
|
||||
def test_is_default_branch_false() -> None:
|
||||
resource = _make_resource(ref="feature-branch", default_branch="main")
|
||||
assert resource.is_default_branch is False
|
||||
|
||||
|
||||
def test_is_default_branch_is_unknown_before_hydration() -> None:
|
||||
# None, not False: the branch is fetched on first use, and a bare
|
||||
# read hydrates only the tree, so False here would be a wrong answer
|
||||
# that could survive the life of the mount.
|
||||
resource = _make_resource(ref="main", default_branch=None)
|
||||
assert resource.is_default_branch is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_default_branch_answers_once_hydrated() -> None:
|
||||
resource = _make_resource(ref="main", default_branch=None)
|
||||
with patch("mirage.core.github.repo.fetch_default_branch",
|
||||
return_value="main"):
|
||||
await ensure_default_branch(resource.accessor)
|
||||
assert resource.is_default_branch is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_returns_sha_fingerprint() -> None:
|
||||
tree = {
|
||||
"src/main.py":
|
||||
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
|
||||
}
|
||||
resource = await _make_resource(tree=tree)
|
||||
with _offline(tree)[2]:
|
||||
resource = _make_resource(tree=tree)
|
||||
with _offline(tree)[1]:
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"),
|
||||
resource.index)
|
||||
@@ -170,12 +163,12 @@ async def test_replacing_index_still_serves_the_tree() -> None:
|
||||
"src/main.py":
|
||||
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
|
||||
}
|
||||
resource = await _make_resource(tree=tree)
|
||||
resource = _make_resource(tree=tree)
|
||||
resource.set_index(IndexConfig())
|
||||
|
||||
# The fresh store is empty, which reads as not-live, so the next read
|
||||
# fills it by refetching rather than reporting the path gone.
|
||||
with _offline(tree)[2]:
|
||||
with _offline(tree)[1]:
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"),
|
||||
resource.index)
|
||||
@@ -184,46 +177,62 @@ async def test_replacing_index_still_serves_the_tree() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_raises_when_path_not_in_tree() -> None:
|
||||
resource = await _make_resource()
|
||||
with _offline({})[2], pytest.raises(FileNotFoundError):
|
||||
resource = _make_resource()
|
||||
with _offline({})[1], pytest.raises(FileNotFoundError):
|
||||
await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/nonexistent.py"), resource.index)
|
||||
|
||||
|
||||
def test_the_constructor_reaches_no_network() -> None:
|
||||
# The rule the lazy split exists to keep: naming a repository costs
|
||||
# nothing, so building a mount never blocks the caller's event loop
|
||||
# and build_resource can stay synchronous.
|
||||
branch, fetch = _offline({})
|
||||
with branch as branch_m, fetch as fetch_m:
|
||||
resource = GitHubResource(CONFIG, OWNER, REPO, "main")
|
||||
branch_m.assert_not_called()
|
||||
fetch_m.assert_not_called()
|
||||
assert resource.accessor.default_branch is None
|
||||
assert resource.accessor.tree == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.resource.github.github.fetch_tree")
|
||||
@patch("mirage.resource.github.github.fetch_default_branch")
|
||||
async def test_create_fetches_default_branch(mock_fetch_branch,
|
||||
mock_fetch_tree) -> None:
|
||||
mock_fetch_branch.return_value = "develop"
|
||||
mock_fetch_tree.return_value = ({}, False)
|
||||
resource = await GitHubResource.build(config=CONFIG,
|
||||
owner=OWNER,
|
||||
repo=REPO,
|
||||
ref="main")
|
||||
async def test_ensure_default_branch_fetches_once_and_caches() -> None:
|
||||
resource = _make_resource(default_branch=None)
|
||||
with patch("mirage.core.github.repo.fetch_default_branch",
|
||||
return_value="develop") as mock_branch:
|
||||
assert await ensure_default_branch(resource.accessor) == "develop"
|
||||
assert await ensure_default_branch(resource.accessor) == "develop"
|
||||
assert resource.accessor.default_branch == "develop"
|
||||
mock_fetch_branch.assert_awaited_once_with(CONFIG, OWNER, REPO)
|
||||
mock_branch.assert_awaited_once_with(CONFIG, OWNER, REPO)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_constructor_reaches_no_network() -> None:
|
||||
# The whole point of the build() split: __init__ takes the fetched
|
||||
# tree, so building one never blocks the caller's event loop.
|
||||
async def test_ensure_tree_fetches_once_and_caches() -> None:
|
||||
tree = {
|
||||
"src/main.py":
|
||||
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
|
||||
}
|
||||
branch_target = "mirage.resource.github.github.fetch_default_branch"
|
||||
with patch(branch_target) as branch, \
|
||||
patch("mirage.resource.github.github.fetch_tree") as fetch:
|
||||
resource = GitHubResource(CONFIG, OWNER, REPO, "main", "main", tree)
|
||||
branch.assert_not_called()
|
||||
fetch.assert_not_called()
|
||||
assert resource.accessor.default_branch == "main"
|
||||
# A read is where the network comes in: the index is keyed by mount
|
||||
# prefix, so filling it waits for a PathSpec and refetches then.
|
||||
with _offline(tree)[2]:
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"),
|
||||
resource.index)
|
||||
assert result.fingerprint == "abc123"
|
||||
resource = _make_resource()
|
||||
with patch("mirage.core.github.tree.fetch_tree",
|
||||
return_value=(tree, False)) as mock_tree:
|
||||
await ensure_tree(resource.accessor)
|
||||
await ensure_tree(resource.accessor)
|
||||
assert resource.accessor.tree == tree
|
||||
assert mock_tree.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_ensure_tree_fetches_once() -> None:
|
||||
# The lock is the whole reason a first `find` and a first `du` racing
|
||||
# on a cold mount cost one tree fetch rather than two.
|
||||
resource = _make_resource()
|
||||
with patch("mirage.core.github.tree.fetch_tree",
|
||||
return_value=({}, False)) as mock_tree:
|
||||
mock_tree.return_value = ({
|
||||
"a.py":
|
||||
TreeEntry(path="a.py", type="blob", sha="s", size=1),
|
||||
}, False)
|
||||
await asyncio.gather(*(ensure_tree(resource.accessor)
|
||||
for _ in range(8)))
|
||||
assert mock_tree.await_count == 1
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.cache.index.ram import RAMIndexCacheStore
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.readdir import readdir
|
||||
from mirage.core.github.tree import ensure_tree
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.resource.github.github import GitHubResource
|
||||
from mirage.types import PathSpec
|
||||
|
||||
CONFIG = GitHubConfig(token="ghp_test")
|
||||
TREE = {
|
||||
"src": TreeEntry(path="src", type="tree", sha="a", size=None),
|
||||
"src/main.py": TreeEntry(path="src/main.py", type="blob", sha="b",
|
||||
size=10),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tree_calls(monkeypatch):
|
||||
calls = []
|
||||
|
||||
async def _fetch_tree(config, owner, repo, ref):
|
||||
calls.append((owner, repo, ref))
|
||||
return dict(TREE), False
|
||||
|
||||
monkeypatch.setattr("mirage.core.github.tree.fetch_tree", _fetch_tree)
|
||||
return calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_readdir_costs_one_tree_fetch(tree_calls):
|
||||
# Building used to fetch the tree and nothing seeded the index with
|
||||
# it, so the first readdir refetched and threw the first away. Two
|
||||
# `git/trees` calls where one does; hydrating lazily removes one.
|
||||
resource = GitHubResource(CONFIG, "o", "r", "main")
|
||||
assert tree_calls == []
|
||||
|
||||
index = RAMIndexCacheStore()
|
||||
entries = await readdir(
|
||||
resource.accessor,
|
||||
PathSpec(resource_path="", virtual="/", directory="/"), index)
|
||||
assert sorted(entries) == ["/src"]
|
||||
assert len(tree_calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_empty_repo_hydrates_once(tree_calls, monkeypatch):
|
||||
# Hydration was tracked by whether the tree held anything, so an
|
||||
# empty repository read as "never hydrated" and refetched forever:
|
||||
# twice per call with an index wired, since the refill seeds an empty
|
||||
# root and then the fallback runs anyway.
|
||||
async def _empty(config, owner, repo, ref):
|
||||
tree_calls.append((owner, repo, ref))
|
||||
return {}, False
|
||||
|
||||
monkeypatch.setattr("mirage.core.github.tree.fetch_tree", _empty)
|
||||
resource = GitHubResource(CONFIG, "o", "r", "main")
|
||||
index = RAMIndexCacheStore()
|
||||
for _ in range(3):
|
||||
await ensure_tree(resource.accessor, index, "/gh")
|
||||
assert resource.accessor.tree == {}
|
||||
assert resource.accessor.tree_loaded is True
|
||||
assert len(tree_calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tree_passed_to_the_constructor_counts_as_hydrated(tree_calls):
|
||||
# A caller holding the answer (a test, a snapshot restore) must not
|
||||
# trigger a fetch on the first direct-tree command.
|
||||
resource = GitHubResource(CONFIG, "o", "r", "main", tree=dict(TREE))
|
||||
await ensure_tree(resource.accessor)
|
||||
assert tree_calls == []
|
||||
@@ -51,7 +51,7 @@ def test_resource_registers_commands():
|
||||
@pytest.mark.asyncio
|
||||
async def test_resource_in_registry():
|
||||
assert "lancedb" in REGISTRY
|
||||
res = await build_resource("lancedb", {"uri": "/tmp/db"})
|
||||
res = build_resource("lancedb", {"uri": "/tmp/db"})
|
||||
assert res.name == "lancedb"
|
||||
|
||||
|
||||
|
||||
@@ -44,8 +44,7 @@ async def test_resource_in_registry():
|
||||
from mirage.resource.registry import REGISTRY, build_resource
|
||||
|
||||
assert "postgres" in REGISTRY
|
||||
res = await build_resource("postgres",
|
||||
config={"dsn": "postgres://localhost/db"})
|
||||
res = build_resource("postgres", config={"dsn": "postgres://localhost/db"})
|
||||
assert res.name == "postgres"
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ def test_resource_registers_commands():
|
||||
@pytest.mark.asyncio
|
||||
async def test_resource_in_registry():
|
||||
assert "qdrant" in REGISTRY
|
||||
res = await build_resource("qdrant", {"collection": "docs"})
|
||||
res = build_resource("qdrant", {"collection": "docs"})
|
||||
assert res.name == "qdrant"
|
||||
|
||||
|
||||
|
||||
@@ -75,24 +75,21 @@ def test_capabilities_match_the_committed_spec_manifest():
|
||||
"scripts/gen_specs.py")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_ram_returns_ram_resource():
|
||||
def test_build_ram_returns_ram_resource():
|
||||
from mirage.resource.ram import RAMResource
|
||||
p = await build_resource("ram")
|
||||
p = build_resource("ram")
|
||||
assert isinstance(p, RAMResource)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_disk_takes_raw_kwargs(tmp_path):
|
||||
def test_build_disk_takes_raw_kwargs(tmp_path):
|
||||
from mirage.resource.disk import DiskResource
|
||||
p = await build_resource("disk", {"root": str(tmp_path)})
|
||||
p = build_resource("disk", {"root": str(tmp_path)})
|
||||
assert isinstance(p, DiskResource)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_s3_uses_config_class():
|
||||
def test_build_s3_uses_config_class():
|
||||
from mirage.resource.s3 import S3Resource
|
||||
p = await build_resource(
|
||||
p = build_resource(
|
||||
"s3", {
|
||||
"bucket": "b",
|
||||
"region": "us-east-1",
|
||||
@@ -104,10 +101,9 @@ async def test_build_s3_uses_config_class():
|
||||
assert p.config.region == "us-east-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_r2_uses_r2_config():
|
||||
def test_build_r2_uses_r2_config():
|
||||
from mirage.resource.r2 import R2Resource
|
||||
p = await build_resource(
|
||||
p = build_resource(
|
||||
"r2", {
|
||||
"bucket": "b",
|
||||
"account_id": "acct",
|
||||
@@ -119,20 +115,18 @@ async def test_build_r2_uses_r2_config():
|
||||
|
||||
@pytest.mark.skipif(not os.environ.get("REDIS_URL"),
|
||||
reason="REDIS_URL not set")
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_redis_takes_raw_kwargs():
|
||||
def test_build_redis_takes_raw_kwargs():
|
||||
from mirage.resource.redis import RedisResource
|
||||
p = await build_resource("redis", {
|
||||
p = build_resource("redis", {
|
||||
"url": os.environ["REDIS_URL"],
|
||||
"key_prefix": "test:",
|
||||
})
|
||||
assert isinstance(p, RedisResource)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_resource_raises_keyerror():
|
||||
def test_unknown_resource_raises_keyerror():
|
||||
with pytest.raises(KeyError, match="unknown resource 'nonsense'"):
|
||||
await build_resource("nonsense")
|
||||
build_resource("nonsense")
|
||||
|
||||
|
||||
def test_registry_module_import_is_free_of_resource_deps():
|
||||
@@ -143,9 +137,8 @@ def test_registry_module_import_is_free_of_resource_deps():
|
||||
importlib.import_module("mirage.resource.registry")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_hf_buckets_resource():
|
||||
r = await build_resource("hf_buckets", {"bucket": "o/b"})
|
||||
def test_build_hf_buckets_resource():
|
||||
r = build_resource("hf_buckets", {"bucket": "o/b"})
|
||||
assert isinstance(r, HfBucketsResource)
|
||||
|
||||
|
||||
@@ -181,26 +174,23 @@ def clean_registry(monkeypatch):
|
||||
monkeypatch.setattr(registry, "_entry_points_loaded", False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_resource_class_and_config(clean_registry):
|
||||
def test_register_resource_class_and_config(clean_registry):
|
||||
register_resource("fake_custom", FakeCustomResource, FakeCustomConfig)
|
||||
built = await build_resource("fake_custom", {"url": "http://x"})
|
||||
built = build_resource("fake_custom", {"url": "http://x"})
|
||||
assert isinstance(built, FakeCustomResource)
|
||||
assert built.config.url == "http://x"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_resource_kwargs_config(clean_registry):
|
||||
def test_register_resource_kwargs_config(clean_registry):
|
||||
register_resource("fake_kwargs", FakeKwargsResource)
|
||||
built = await build_resource("fake_kwargs", {"root": "/data"})
|
||||
built = build_resource("fake_kwargs", {"root": "/data"})
|
||||
assert isinstance(built, FakeKwargsResource)
|
||||
assert built.root == "/data"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_resource_config_cls_attribute(clean_registry):
|
||||
def test_register_resource_config_cls_attribute(clean_registry):
|
||||
register_resource("fake_attr", FakeConfigClsResource)
|
||||
built = await build_resource("fake_attr", {"url": "http://y"})
|
||||
built = build_resource("fake_attr", {"url": "http://y"})
|
||||
assert built.config.url == "http://y"
|
||||
|
||||
|
||||
@@ -209,11 +199,10 @@ def test_register_resource_rejects_builtin_shadow(clean_registry):
|
||||
register_resource("s3", FakeCustomResource)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_resource_spec_string(clean_registry):
|
||||
def test_register_resource_spec_string(clean_registry):
|
||||
register_resource("fake_spec",
|
||||
"tests.resource.test_registry:FakeKwargsResource")
|
||||
built = await build_resource("fake_spec", {"root": "/spec"})
|
||||
built = build_resource("fake_spec", {"root": "/spec"})
|
||||
assert built.root == "/spec"
|
||||
|
||||
|
||||
@@ -224,8 +213,7 @@ def test_known_resources_includes_custom(clean_registry):
|
||||
assert "s3" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_entry_point_discovery(clean_registry, monkeypatch):
|
||||
def test_entry_point_discovery(clean_registry, monkeypatch):
|
||||
import importlib.metadata
|
||||
|
||||
ep = importlib.metadata.EntryPoint(
|
||||
@@ -239,14 +227,12 @@ async def test_entry_point_discovery(clean_registry, monkeypatch):
|
||||
return [ep]
|
||||
|
||||
monkeypatch.setattr(importlib.metadata, "entry_points", fake_entry_points)
|
||||
built = await build_resource("fake_ep", {"root": "/ep"})
|
||||
built = build_resource("fake_ep", {"root": "/ep"})
|
||||
assert built.root == "/ep"
|
||||
assert "fake_ep" in known_resources()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_entry_point_does_not_shadow_registered(clean_registry,
|
||||
monkeypatch):
|
||||
def test_entry_point_does_not_shadow_registered(clean_registry, monkeypatch):
|
||||
import importlib.metadata
|
||||
|
||||
ep = importlib.metadata.EntryPoint(
|
||||
@@ -257,11 +243,10 @@ async def test_entry_point_does_not_shadow_registered(clean_registry,
|
||||
monkeypatch.setattr(importlib.metadata, "entry_points",
|
||||
lambda *, group: [ep])
|
||||
register_resource("fake_custom", FakeCustomResource, FakeCustomConfig)
|
||||
built = await build_resource("fake_custom", {"url": "http://z"})
|
||||
built = build_resource("fake_custom", {"url": "http://z"})
|
||||
assert isinstance(built, FakeCustomResource)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_resource_lists_known(clean_registry):
|
||||
def test_unknown_resource_lists_known(clean_registry):
|
||||
with pytest.raises(KeyError, match="unknown resource"):
|
||||
await build_resource("nope_not_real")
|
||||
build_resource("nope_not_real")
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pydantic import BaseModel, SecretStr
|
||||
from collections.abc import Callable
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, SecretStr
|
||||
|
||||
from mirage.resource.secrets import (REDACTED_SECRET, has_redacted_secret,
|
||||
redacted_config_dump,
|
||||
@@ -60,3 +62,35 @@ def test_top_level_secret_fields_still_redact():
|
||||
data = redacted_config_dump(Inner(token=SecretStr("s")))
|
||||
assert data == {"token": REDACTED_SECRET, "host": "h"}
|
||||
assert revealed_config_dump(Inner(token=SecretStr("s")))["token"] == "s"
|
||||
|
||||
|
||||
class ProviderConfig(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
access_token: SecretStr | Callable[[], str | SecretStr] | None = None
|
||||
host: str = "h"
|
||||
|
||||
|
||||
def test_a_provider_callable_redacts_instead_of_crashing():
|
||||
# msgraph and google both accept a token provider, and pydantic
|
||||
# cannot serialize a function: dumping the model raised
|
||||
# PydanticSerializationError and took the whole snapshot with it.
|
||||
data = redacted_config_dump(ProviderConfig(access_token=lambda: "tok"))
|
||||
assert data == {"access_token": REDACTED_SECRET, "host": "h"}
|
||||
|
||||
|
||||
def test_a_provider_callable_is_never_revealed():
|
||||
# Revealing one would mean calling it and freezing a token that
|
||||
# expires into a snapshot that does not.
|
||||
data = revealed_config_dump(ProviderConfig(access_token=lambda: "tok"))
|
||||
assert data["access_token"] == REDACTED_SECRET
|
||||
|
||||
|
||||
def test_a_provider_config_still_reports_a_redacted_secret():
|
||||
# This is what routes the mount down the fresh-resource path at load.
|
||||
assert has_redacted_secret(
|
||||
redacted_config_dump(ProviderConfig(access_token=lambda: "tok")))
|
||||
|
||||
|
||||
def test_an_absent_secret_stays_none():
|
||||
assert redacted_config_dump(ProviderConfig())["access_token"] is None
|
||||
|
||||
@@ -337,7 +337,7 @@ async def test_registry_resource_state_masks_credential(name, config, secret):
|
||||
from mirage.resource.registry import build_resource
|
||||
from mirage.resource.secrets import has_redacted_secret
|
||||
|
||||
p = await build_resource(name, config)
|
||||
p = build_resource(name, config)
|
||||
state = p.get_state()
|
||||
assert state["type"] == name
|
||||
assert state["config"] is not None
|
||||
|
||||
@@ -44,6 +44,6 @@ async def test_config_index_redis_block_builds_redis_config():
|
||||
mounts={"/m": MountBlock(resource="ram")},
|
||||
index=RedisIndexBlock(type="redis"),
|
||||
)
|
||||
kwargs = await cfg.to_workspace_kwargs()
|
||||
kwargs = cfg.to_workspace_kwargs()
|
||||
assert isinstance(kwargs["index"], RedisIndexConfig)
|
||||
assert kwargs["index"].key_prefix == "mirage:index:"
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.resource.ram import RAMResource
|
||||
@@ -87,3 +89,35 @@ def test_limits_are_copied_not_aliased():
|
||||
{"/a": (RAMResource(), MountMode.READ, source)}, MountMode.WRITE)
|
||||
source["wget"] = guard
|
||||
assert set(specs[0].command_limits) == {"curl"}
|
||||
|
||||
|
||||
def test_a_coroutine_is_refused_naming_the_await():
|
||||
# 0.0.5 made build_resource async, so every caller written against
|
||||
# 0.0.3/0.0.4 handed the mount table an un-awaited coroutine and got
|
||||
# `'coroutine' object has no attribute 'set_index'` from
|
||||
# install_mounts. The mount and the fix have to be in the message.
|
||||
coro = asyncio.sleep(0)
|
||||
try:
|
||||
with pytest.raises(TypeError) as excinfo:
|
||||
normalize_resources({"/gh": (coro, MountMode.READ)},
|
||||
MountMode.WRITE)
|
||||
finally:
|
||||
coro.close()
|
||||
message = str(excinfo.value)
|
||||
assert "'/gh'" in message
|
||||
assert "await" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["not-a-resource", 42, None])
|
||||
def test_a_non_resource_is_refused_naming_the_mount(value):
|
||||
with pytest.raises(TypeError, match=r"'/x'.*expected a BaseResource"):
|
||||
normalize_resources({"/x": value}, MountMode.WRITE)
|
||||
|
||||
|
||||
def test_the_guard_runs_before_any_mount_is_installed():
|
||||
# A bad second entry must not leave the first one half-installed.
|
||||
with pytest.raises(TypeError):
|
||||
normalize_resources({
|
||||
"/good": RAMResource(),
|
||||
"/bad": "nope",
|
||||
}, MountMode.WRITE)
|
||||
|
||||
Reference in New Issue
Block a user