Merge pull request #811 from strukto-ai/feat/watch-delta-hooks
feat(watch): ship delta_hook for ten more backends
This commit is contained in:
@@ -702,11 +702,56 @@ jobs:
|
||||
- name: Run declarative battery on nextcloud
|
||||
run: ./python/.venv/bin/python integ/runners/python/main.py --target nextcloud --strict
|
||||
|
||||
- name: Run watch battery on nextcloud
|
||||
# disk, ssh and dropbox need nothing (tempdir, in-process SFTP
|
||||
# server, in-process fake); s3 and gridfs need a real service each,
|
||||
# and skip when the env is absent rather than failing.
|
||||
- name: Start MinIO for the watch battery
|
||||
run: |
|
||||
docker run -d --name minio-watch -p 9000:9000 \
|
||||
-e MINIO_ROOT_USER=minio -e MINIO_ROOT_PASSWORD=minio123 \
|
||||
minio/minio:latest server /data
|
||||
for i in $(seq 1 30); do
|
||||
curl -sf http://localhost:9000/minio/health/live && break
|
||||
sleep 1
|
||||
done
|
||||
curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc
|
||||
chmod +x /tmp/mc
|
||||
/tmp/mc alias set local http://localhost:9000 minio minio123
|
||||
/tmp/mc mb local/mirage-watch
|
||||
|
||||
- name: Start MongoDB for the watch battery
|
||||
run: |
|
||||
docker run -d --name mongo-watch -p 27017:27017 mongo:8
|
||||
for i in $(seq 1 30); do
|
||||
docker exec mongo-watch mongosh --quiet \
|
||||
--eval 'db.runCommand({ping:1}).ok' >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# gdrive is the one watch target whose fake is TypeScript and
|
||||
# out-of-process; the other five in-process fakes need nothing.
|
||||
- name: Start the fake Google Workspace server
|
||||
run: |
|
||||
cd integ && nohup pnpm exec tsx server/gws_server.ts --port 19999 \
|
||||
> /tmp/gws.log 2>&1 &
|
||||
for i in $(seq 1 30); do
|
||||
grep -q "GWS_URL=" /tmp/gws.log && break
|
||||
sleep 1
|
||||
done
|
||||
cat /tmp/gws.log
|
||||
|
||||
- name: Run watch battery
|
||||
env:
|
||||
NEXTCLOUD_URL: http://localhost:8080/remote.php/dav/files/admin/
|
||||
NEXTCLOUD_USERNAME: admin
|
||||
NEXTCLOUD_PASSWORD: admin123
|
||||
S3_ENDPOINT: http://localhost:9000
|
||||
S3_BUCKET: mirage-watch
|
||||
S3_REGION: us-east-1
|
||||
AWS_ACCESS_KEY_ID: minio
|
||||
AWS_SECRET_ACCESS_KEY: minio123
|
||||
MONGODB_URI: mongodb://localhost:27017
|
||||
GWS_URL: http://127.0.0.1:19999
|
||||
run: ./python/.venv/bin/python integ/watch/run.py
|
||||
|
||||
integ-fuse:
|
||||
|
||||
@@ -106,6 +106,7 @@ jobs:
|
||||
run examples/python/ram/ram_python.py integ/truth/python/ram_python.txt
|
||||
run examples/python/disk/disk.py integ/truth/python/disk.txt
|
||||
run examples/python/disk/disk_vfs.py integ/truth/python/disk_vfs.txt
|
||||
run examples/python/disk/watch.py integ/truth/watch_delta.txt
|
||||
run examples/python/other/custom_command.py integ/truth/python/custom_command.txt
|
||||
run examples/python/filetype/filetype.py integ/truth/python/filetype.txt
|
||||
run examples/python/redis_resource/example_redis.py integ/truth/python/redis.txt
|
||||
|
||||
@@ -100,6 +100,7 @@ jobs:
|
||||
run ram/ram_vfs.ts integ/truth/typescript/ram_vfs.txt
|
||||
run disk/disk.ts integ/truth/typescript/disk.txt
|
||||
run disk/disk_vfs.ts integ/truth/typescript/disk_vfs.txt
|
||||
run disk/watch.ts integ/truth/watch_delta.txt
|
||||
run other/custom_command.ts integ/truth/typescript/custom_command.txt
|
||||
run filetype/filetype.ts integ/truth/typescript/filetype.txt
|
||||
run pyodide/basic.ts integ/truth/typescript/pyodide_basic.txt
|
||||
|
||||
@@ -18,17 +18,45 @@ Watching splits into two halves, and they have different support stories:
|
||||
would subscribe to; mapping its payload to a `FileEvent` is consumer code,
|
||||
and Mirage ships a sample where marked.
|
||||
|
||||
| Resource | Pull (`delta_hook`) | Push signal (provider-side) | Sample |
|
||||
| --- | --- | --- | --- |
|
||||
| Nextcloud | ✓ ETag walk diff | `webhook_listeners` app (Nextcloud 30+) | [example](https://github.com/strukto-ai/mirage/blob/main/examples/python/nextcloud/watch.py) + integ receiver |
|
||||
| S3 / S3-compatible | planned | S3 Event Notifications (SQS/EventBridge) | — |
|
||||
| Google Drive | planned | `changes.watch` push channels | — |
|
||||
| Dropbox | planned | Dropbox webhooks + cursor delta | — |
|
||||
| OneDrive / SharePoint | planned | Graph change notifications + delta API | — |
|
||||
| GitHub | planned | repository webhooks (push events) | — |
|
||||
| Slack | planned | Events API (`message`, `file_shared`, ...) | — |
|
||||
| Disk | planned | local inotify / FSEvents / watchdog | — |
|
||||
| RAM, Redis, others | — | (no external writers to observe) | — |
|
||||
| Resource | Pull (`delta_hook`) | Fingerprint | Push signal (provider-side) | Sample |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Nextcloud | ✓ recursive PROPFIND | WebDAV ETag | `webhook_listeners` app (Nextcloud 30+) | [example](https://github.com/strukto-ai/mirage/blob/main/examples/python/nextcloud/watch.py) + integ receiver |
|
||||
| S3 / S3-compatible | ✓ recursive `list_objects_v2` | object ETag | S3 Event Notifications (SQS/EventBridge) | — |
|
||||
| GitHub | ✓ recursive git tree | blob sha | repository webhooks (push events) | — |
|
||||
| Dropbox | ✓ recursive `list_folder` | `content_hash` | Dropbox webhooks + cursor delta | — |
|
||||
| GridFS | ✓ `fs.files` prefix query | revision ObjectId | MongoDB change streams | — |
|
||||
| Hugging Face | ✓ recursive tree listing | Hub ETag | — | — |
|
||||
| Disk | ✓ recursive directory walk | mtime\|size | local inotify / FSEvents / watchdog | [example](https://github.com/strukto-ai/mirage/blob/main/examples/python/disk/watch.py) |
|
||||
| SSH / SFTP | ✓ per-directory descent | mtime\|size | — | — |
|
||||
| Google Drive | ✓ per-folder descent | `modifiedTime` | `changes.watch` push channels | — |
|
||||
| OneDrive / SharePoint | ✓ per-folder descent | `cTag` | Graph change notifications + delta API | — |
|
||||
| Box | ✓ per-folder descent | content `sha1` | Box `/events` long poll | — |
|
||||
| Slack | planned | — | Events API (`message`, `file_shared`, ...) | — |
|
||||
| RAM, Redis, others | — | — | (no external writers to observe) | — |
|
||||
|
||||
The **Fingerprint** column is what decides UPDATE. A content-addressed one
|
||||
(GitHub's blob sha, S3's single-part ETag, Dropbox's `content_hash`, Box's
|
||||
`sha1`) reports
|
||||
nothing when a write stores identical bytes. `mtime|size` cannot: rewriting a
|
||||
file with its own contents moves the mtime, so disk and SSH report an UPDATE.
|
||||
That is the filesystem's own resolution, not a Mirage choice.
|
||||
|
||||
### Cost per pull
|
||||
|
||||
Not every hook costs the same, and the poll cadence should follow the shape:
|
||||
|
||||
- **One request per pull**, whatever the tree: GitHub (one recursive tree
|
||||
call), Nextcloud (one recursive PROPFIND), Hugging Face, and GridFS (one
|
||||
prefix query). S3 is one request per 1000 keys.
|
||||
- **One request per directory**: Google Drive, OneDrive/SharePoint and Box key
|
||||
their trees by opaque id and offer no whole-subtree listing, so the walk
|
||||
descends folder by folder. SFTP does too, for the same reason.
|
||||
|
||||
Where a provider offers a native cursor (Dropbox `list_folder/continue`, Graph
|
||||
`/delta`, Drive `changes.list`), that is a **faster** pull, not a more correct
|
||||
one, and it does not replace the walk: a server may invalidate a cursor at any
|
||||
time, and the only answer to that is a full listing. Those fast paths belong
|
||||
behind `pull()` with the walk as their reset.
|
||||
|
||||
## What "Delivery ✓" buys you without a hook
|
||||
|
||||
@@ -54,6 +82,17 @@ The guarantee is identical on every backend: caches for the changed path and
|
||||
its ancestor listings are invalidated before delivery, so reads after an event
|
||||
are fresh.
|
||||
|
||||
One thing every backend has to agree on for that to hold: the index is
|
||||
keyed by the mount-absolute path (`/m/data/x`), which is what
|
||||
`CacheManager` builds when it evicts. GitHub was the one exception, keying
|
||||
its by the repo-relative path because its index is a materialized git
|
||||
tree, and the mismatch was silent: evicting a key an index never held
|
||||
succeeds, so a GitHub mount kept serving pre-change bytes with nothing
|
||||
failing anywhere a caller could see. It now keys like the rest, and the
|
||||
repo-relative path logic that wanted the other spelling (`find`, `du`,
|
||||
grep's scope counter) reads the git tree on the accessor instead, which is
|
||||
where a repo-relative path belongs.
|
||||
|
||||
## Adding pull detection to a backend
|
||||
|
||||
`ListingDeltaHook` is generic; a backend earns the **Pull** column with one
|
||||
@@ -71,7 +110,32 @@ def delta_hook(self):
|
||||
return ListingDeltaHook(S3Walk(self.accessor))
|
||||
```
|
||||
|
||||
Two shared helpers cover the shapes that recur, so a new backend usually
|
||||
writes neither loop itself:
|
||||
|
||||
- **`synth_dirs`** (`mirage/watch/walk.py`) builds the directory rows a prefix
|
||||
store implies but does not store. An object store has no directories, so a
|
||||
walk that reported only keys would show a file appearing inside a directory
|
||||
that never appeared. S3 and GridFS use it, and it takes explicitly stored
|
||||
markers too, so an empty directory made by `mkdir` is still reported.
|
||||
- **`ReaddirWalk`** covers a backend with no recursive listing at all. It
|
||||
descends through the backend's own `readdir` and `stat`, exactly as `find`
|
||||
does, and gives each pull a **fresh private index**. That is what keeps the
|
||||
DeltaHook contract: the index is not Mirage's read cache, so the walk cannot
|
||||
compare the cache to itself, and it starts empty every pull. It has to exist
|
||||
at all because Drive, Box and Graph resolve a path's id through the index its
|
||||
parent's readdir populated; a null index makes every path below the root read
|
||||
as absent.
|
||||
|
||||
Backends with a native cursor API (Dropbox `list_folder/continue`, Graph
|
||||
delta) can skip the walk and implement `pull(root, checkpoint)` directly; the
|
||||
opaque checkpoint becomes the server cursor, and consumers cannot tell the
|
||||
difference.
|
||||
delta) can implement `pull(root, checkpoint)` directly and let the opaque
|
||||
checkpoint be the server cursor; consumers cannot tell the difference. Keep the
|
||||
walk as the reset path, though. A cursor is a promise the server keeps, and
|
||||
when it breaks (`path/reset`, `resyncRequired`) a full listing is the only
|
||||
answer.
|
||||
|
||||
A walk that knows it saw only part of the tree must raise
|
||||
`IncompleteWalkError` rather than return what it has. A snapshot diff reads
|
||||
every unlisted path as a DELETE, so a partial listing does not degrade into
|
||||
fewer events, it invents wrong ones. GitHub does this when the API truncates a
|
||||
large repository's tree.
|
||||
|
||||
+10
-3
@@ -111,8 +111,10 @@ is [`examples/python/nextcloud/watch.py`](https://github.com/strukto-ai/mirage/b
|
||||
|
||||
## Pull Mode
|
||||
|
||||
Backends that implement `delta_hook()` (Nextcloud today) answer one question:
|
||||
*what changed under this root since the last checkpoint?* A baseline pull
|
||||
Backends that implement `delta_hook()` answer one question:
|
||||
*what changed under this root since the last checkpoint?* Eleven resource
|
||||
families ship one today, listed in the [Watch Matrix](/python/watch-matrix)
|
||||
with the fingerprint each one compares on. A baseline pull
|
||||
(`checkpoint=None`) emits nothing; every later pull diffs against the
|
||||
checkpoint you hand back. The whole consumer poller is:
|
||||
|
||||
@@ -127,6 +129,11 @@ while True:
|
||||
await asyncio.sleep(30)
|
||||
```
|
||||
|
||||
A runnable version needing no credentials is
|
||||
[`examples/python/disk/watch.py`](https://github.com/strukto-ai/mirage/blob/main/examples/python/disk/watch.py):
|
||||
it writes to the mount's directory behind mirage's back, pulls the diff, and
|
||||
reads the changed file back through the workspace.
|
||||
|
||||
Pull is self-healing: it diffs current backend state against your checkpoint,
|
||||
so events missed while your service was down surface on the next pull. A
|
||||
common production shape is push-first with a pull at startup as recovery, or
|
||||
@@ -180,7 +187,7 @@ loops end cleanly) and the workspace returns to its idle state; the next
|
||||
`ws.notify`.
|
||||
- The TypeScript API has the same delivery, queue, and delta-hook design; see
|
||||
[TypeScript Watch](/typescript/watch).
|
||||
- Nextcloud is the first backend with a `delta_hook`; push mode works with any
|
||||
- Pull detection ships for eleven resource families; push mode works with any
|
||||
backend today since `ws.notify` accepts events from any detection you run.
|
||||
See the [Watch Matrix](/python/watch-matrix) for per-resource support.
|
||||
- Events are in-memory and at-most-once per subscriber; durable queues and
|
||||
|
||||
@@ -14,17 +14,45 @@ Watching has two independent halves:
|
||||
`deltaHook()` ships with Mirage. **Push signal** names the provider mechanism
|
||||
that your application can map to a `FileEvent`.
|
||||
|
||||
| Resource | Pull (`deltaHook`) | Push signal (provider-side) |
|
||||
| --------------------- | ------------------ | ------------------------------------------ |
|
||||
| Nextcloud | ✓ ETag walk diff | `webhook_listeners` app (Nextcloud 30+) |
|
||||
| S3 / S3-compatible | planned | S3 Event Notifications (SQS/EventBridge) |
|
||||
| Google Drive | planned | `changes.watch` push channels |
|
||||
| Dropbox | planned | Dropbox webhooks + cursor delta |
|
||||
| OneDrive / SharePoint | planned | Graph change notifications + delta API |
|
||||
| GitHub | planned | repository webhooks |
|
||||
| Slack | planned | Events API (`message`, `file_shared`, ...) |
|
||||
| Disk | planned | inotify / FSEvents / filesystem watcher |
|
||||
| RAM, Redis, others | — | no external writers to observe |
|
||||
| Resource | Pull (`deltaHook`) | Fingerprint | Push signal (provider-side) |
|
||||
| --------------------- | ----------------------------- | ----------------- | ------------------------------------------ |
|
||||
| Nextcloud | ✓ recursive PROPFIND | WebDAV ETag | `webhook_listeners` app (Nextcloud 30+) |
|
||||
| S3 / S3-compatible | ✓ recursive `ListObjectsV2` | object ETag | S3 Event Notifications (SQS/EventBridge) |
|
||||
| GitHub | ✓ recursive git tree | blob sha | repository webhooks |
|
||||
| Dropbox | ✓ recursive `list_folder` | `content_hash` | Dropbox webhooks + cursor delta |
|
||||
| GridFS | ✓ `fs.files` prefix query | revision ObjectId | MongoDB change streams |
|
||||
| Hugging Face | ✓ recursive tree listing | Hub ETag | — |
|
||||
| Disk | ✓ recursive directory walk | mtime\|size | inotify / FSEvents / filesystem watcher |
|
||||
| SSH / SFTP | ✓ per-directory descent | mtime\|size | — |
|
||||
| Google Drive | ✓ per-folder descent | `modifiedTime` | `changes.watch` push channels |
|
||||
| OneDrive / SharePoint | ✓ per-folder descent | `cTag` | Graph change notifications + delta API |
|
||||
| Box | ✓ per-folder descent | content `sha1` | Box `/events` long poll |
|
||||
| Slack | planned | — | Events API (`message`, `file_shared`, ...) |
|
||||
| RAM, Redis, others | — | — | no external writers to observe |
|
||||
|
||||
The **Fingerprint** column is what decides UPDATE. A content-addressed one
|
||||
(GitHub's blob sha, S3's single-part ETag, Dropbox's `content_hash`, Box's
|
||||
`sha1`) reports
|
||||
nothing when a write stores identical bytes. `mtime|size` cannot: rewriting a
|
||||
file with its own contents moves the mtime, so disk and SSH report an UPDATE.
|
||||
That is the filesystem's own resolution, not a Mirage choice.
|
||||
|
||||
### Cost Per Pull
|
||||
|
||||
Not every hook costs the same, and the poll cadence should follow the shape:
|
||||
|
||||
- **One request per pull**, whatever the tree: GitHub (one recursive tree
|
||||
call), Nextcloud (one recursive PROPFIND), Hugging Face, and GridFS (one
|
||||
prefix query). S3 is one request per 1000 keys.
|
||||
- **One request per directory**: Google Drive, OneDrive/SharePoint and Box key
|
||||
their trees by opaque id and offer no whole-subtree listing, so the walk
|
||||
descends folder by folder. SFTP does too, for the same reason.
|
||||
|
||||
Where a provider offers a native cursor (Dropbox `list_folder/continue`, Graph
|
||||
`/delta`, Drive `changes.list`), that is a **faster** pull, not a more correct
|
||||
one, and it does not replace the walk: a server may invalidate a cursor at any
|
||||
time, and the only answer to that is a full listing. Those fast paths belong
|
||||
behind `pull()` with the walk as their reset.
|
||||
|
||||
## Delivery Without a Hook
|
||||
|
||||
@@ -56,6 +84,17 @@ the iterator drops the event.
|
||||
The guarantee is the same on every backend: caches for the changed path and
|
||||
its ancestor listings are invalidated before delivery.
|
||||
|
||||
One thing every backend has to agree on for that to hold: the index is
|
||||
keyed by the mount-absolute path (`/m/data/x`), which is what
|
||||
`CacheManager` builds when it evicts. GitHub was the one exception, keying
|
||||
its by the repo-relative path because its index is a materialized git
|
||||
tree, and the mismatch was silent: evicting a key an index never held
|
||||
succeeds, so a GitHub mount kept serving pre-change bytes with nothing
|
||||
failing anywhere a caller could see. It now keys like the rest, and the
|
||||
repo-relative path logic that wanted the other spelling (`find`, `du`,
|
||||
grep's scope counter) reads the git tree on the accessor instead, which is
|
||||
where a repo-relative path belongs.
|
||||
|
||||
## Adding Pull Detection
|
||||
|
||||
`ListingDeltaHook` provides a generic checkpointed diff. A backend supplies an
|
||||
@@ -83,6 +122,31 @@ async function* walk(root: PathSpec): AsyncGenerator<WalkEntry> {
|
||||
const hook = new ListingDeltaHook(walk);
|
||||
```
|
||||
|
||||
Two shared helpers cover the shapes that recur, so a new backend usually
|
||||
writes neither loop itself:
|
||||
|
||||
- **`synthDirs`** builds the directory rows a prefix store implies but does not
|
||||
store. An object store has no directories, so a walk that reported only keys
|
||||
would show a file appearing inside a directory that never appeared. S3 and
|
||||
GridFS use it, and it takes explicitly stored markers too, so an empty
|
||||
directory made by `mkdir` is still reported.
|
||||
- **`ReaddirWalk`** covers a backend with no recursive listing at all. It
|
||||
descends through the backend's own `readdir` and `stat`, exactly as `find`
|
||||
does, and gives each pull a **fresh private index**. That is what keeps the
|
||||
DeltaHook contract: the index is not Mirage's read cache, so the walk cannot
|
||||
compare the cache to itself, and it starts empty every pull. It has to exist
|
||||
at all because Drive, Box and Graph resolve a path's id through the index its
|
||||
parent's readdir populated; a null index makes every path below the root read
|
||||
as absent.
|
||||
|
||||
Backends with a native cursor API can implement `pull(root, checkpoint)`
|
||||
directly. The opaque checkpoint can be a Dropbox cursor, a Microsoft Graph
|
||||
delta link, or any other serialized provider state.
|
||||
delta link, or any other serialized provider state. Keep the walk as the reset
|
||||
path, though. A cursor is a promise the server keeps, and when it breaks
|
||||
(`path/reset`, `resyncRequired`) a full listing is the only answer.
|
||||
|
||||
A walk that knows it saw only part of the tree must throw
|
||||
`IncompleteWalkError` rather than return what it has. A snapshot diff reads
|
||||
every unlisted path as a DELETE, so a partial listing does not degrade into
|
||||
fewer events, it invents wrong ones. GitHub does this when the API truncates a
|
||||
large repository's tree.
|
||||
|
||||
@@ -112,8 +112,10 @@ HTTP framework, authentication, retries, and deployment model.
|
||||
## Pull Mode
|
||||
|
||||
Backends that implement `deltaHook()` answer one question: what changed under
|
||||
this root since the last checkpoint? Nextcloud ships this capability today. A
|
||||
baseline pull (`checkpoint === null`) establishes state and emits no changes.
|
||||
this root since the last checkpoint? Eleven resource families ship one today,
|
||||
listed in the [Watch Matrix](/typescript/watch-matrix) with the fingerprint
|
||||
each one compares on. A baseline pull (`checkpoint === null`) establishes
|
||||
state and emits no changes.
|
||||
|
||||
```ts
|
||||
import { PathSpec } from "@struktoai/mirage-node";
|
||||
@@ -135,6 +137,11 @@ for (;;) {
|
||||
}
|
||||
```
|
||||
|
||||
A runnable version needing no credentials is
|
||||
[`examples/typescript/disk/watch.ts`](https://github.com/strukto-ai/mirage/blob/main/examples/typescript/disk/watch.ts):
|
||||
it writes to the mount's directory behind mirage's back, pulls the diff, and
|
||||
reads the changed file back through the workspace.
|
||||
|
||||
Pull is self-healing because current backend state is compared with the saved
|
||||
checkpoint. Events missed while your service was down appear on the next pull.
|
||||
A common production shape is push-first with a startup pull for recovery, or a
|
||||
@@ -178,7 +185,7 @@ watch loops cleanly without closing the workspace. The next `watch()` or
|
||||
the scope.
|
||||
- One watch may span mounts, including nested mounts. Each event is invalidated
|
||||
on the longest-prefix mount that owns its path.
|
||||
- Delivery works on every backend. Nextcloud is currently the first backend
|
||||
with a shipped `deltaHook()`; see the [Watch Matrix](/typescript/watch-matrix).
|
||||
- Delivery works on every backend. Pull detection ships for eleven resource
|
||||
families; see the [Watch Matrix](/typescript/watch-matrix).
|
||||
- Events are in-memory and at-most-once per subscriber unless you provide a
|
||||
durable queue implementation.
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from mirage import MountMode, Workspace
|
||||
from mirage.resource.disk import DiskResource
|
||||
from mirage.types import PathSpec
|
||||
|
||||
MOUNT = "/data"
|
||||
|
||||
|
||||
def seed(root: Path) -> None:
|
||||
"""Lay down the files the baseline pull will record.
|
||||
|
||||
Args:
|
||||
root (Path): Directory the mount is rooted at.
|
||||
"""
|
||||
(root / "reports").mkdir()
|
||||
(root / "reports" / "q1.txt").write_text("first quarter\n")
|
||||
(root / "reports" / "q2.txt").write_text("second quarter\n")
|
||||
|
||||
|
||||
def write_behind_mirage(root: Path) -> None:
|
||||
"""Change the directory the way an outside writer would.
|
||||
|
||||
Nothing here goes through the workspace: this stands in for the
|
||||
teammate, cron job or pipeline that mirage has to detect rather
|
||||
than be told about.
|
||||
|
||||
Args:
|
||||
root (Path): Directory the mount is rooted at.
|
||||
"""
|
||||
(root / "reports" / "q3.txt").write_text("third quarter\n")
|
||||
(root / "reports" / "q1.txt").write_text("first quarter, revised\n")
|
||||
(root / "reports" / "q2.txt").unlink()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
seed(tmp)
|
||||
|
||||
resource = DiskResource(root=str(tmp))
|
||||
ws = Workspace({MOUNT: resource}, mode=MountMode.READ)
|
||||
hook = resource.delta_hook()
|
||||
root = PathSpec.from_str_path(MOUNT, resource_path="")
|
||||
|
||||
# A baseline pull records state and reports nothing. Hand the
|
||||
# checkpoint back on the next call and it diffs against it.
|
||||
delta = await hook.pull(root, None)
|
||||
checkpoint = delta.checkpoint
|
||||
print(f"baseline: {len(delta.changes)} changes")
|
||||
|
||||
write_behind_mirage(tmp)
|
||||
|
||||
delta = await hook.pull(root, checkpoint)
|
||||
print(f"\npull: {len(delta.changes)} changes")
|
||||
for change in sorted(delta.changes, key=lambda c: c.path.virtual):
|
||||
print(f" {change.kind.value:6} {change.path.virtual}")
|
||||
await ws.notify(change)
|
||||
|
||||
# notify invalidates the caches for the changed path and its
|
||||
# ancestor listings before delivering, so a read after an event
|
||||
# can never serve pre-change bytes.
|
||||
result = await ws.execute(f"cat {MOUNT}/reports/q1.txt")
|
||||
print(f"\nread after notify: {(await result.stdout_str()).strip()!r}")
|
||||
result = await ws.execute(f"ls {MOUNT}/reports")
|
||||
print(f"listing: {' '.join((await result.stdout_str()).split())}")
|
||||
|
||||
await ws.close()
|
||||
shutil.rmtree(tmp)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,75 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { DiskResource, MountMode, PathSpec, Workspace } from '@struktoai/mirage-node'
|
||||
|
||||
const MOUNT = '/data'
|
||||
|
||||
// Lay down the files the baseline pull will record.
|
||||
function seed(root: string): void {
|
||||
mkdirSync(join(root, 'reports'))
|
||||
writeFileSync(join(root, 'reports', 'q1.txt'), 'first quarter\n')
|
||||
writeFileSync(join(root, 'reports', 'q2.txt'), 'second quarter\n')
|
||||
}
|
||||
|
||||
// Change the directory the way an outside writer would. Nothing here
|
||||
// goes through the workspace: this stands in for the teammate, cron job
|
||||
// or pipeline that mirage has to detect rather than be told about.
|
||||
function writeBehindMirage(root: string): void {
|
||||
writeFileSync(join(root, 'reports', 'q3.txt'), 'third quarter\n')
|
||||
writeFileSync(join(root, 'reports', 'q1.txt'), 'first quarter, revised\n')
|
||||
unlinkSync(join(root, 'reports', 'q2.txt'))
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'mirage-disk-watch-'))
|
||||
seed(tmp)
|
||||
|
||||
const resource = new DiskResource({ root: tmp })
|
||||
const ws = new Workspace({ [MOUNT]: resource }, { mode: MountMode.READ })
|
||||
const hook = resource.deltaHook()
|
||||
const root = PathSpec.fromStrPath(MOUNT, '')
|
||||
|
||||
// A baseline pull records state and reports nothing. Hand the
|
||||
// checkpoint back on the next call and it diffs against it.
|
||||
let delta = await hook.pull(root, null)
|
||||
const checkpoint = delta.checkpoint
|
||||
console.log(`baseline: ${delta.changes.length} changes`)
|
||||
|
||||
writeBehindMirage(tmp)
|
||||
|
||||
delta = await hook.pull(root, checkpoint)
|
||||
console.log(`\npull: ${delta.changes.length} changes`)
|
||||
const changes = [...delta.changes].sort((a, b) => a.path.virtual.localeCompare(b.path.virtual))
|
||||
for (const change of changes) {
|
||||
console.log(` ${change.kind.padEnd(6)} ${change.path.virtual}`)
|
||||
await ws.notify(change)
|
||||
}
|
||||
|
||||
// notify invalidates the caches for the changed path and its ancestor
|
||||
// listings before delivering, so a read after an event can never serve
|
||||
// pre-change bytes.
|
||||
let result = await ws.execute(`cat ${MOUNT}/reports/q1.txt`)
|
||||
console.log(`\nread after notify: '${result.stdoutText.trim()}'`)
|
||||
result = await ws.execute(`ls ${MOUNT}/reports`)
|
||||
console.log(`listing: ${result.stdoutText.split(/\s+/).filter(Boolean).join(' ')}`)
|
||||
|
||||
await ws.close()
|
||||
rmSync(tmp, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
await main()
|
||||
+59
-9
@@ -12,6 +12,7 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
createServer,
|
||||
type IncomingMessage,
|
||||
@@ -38,6 +39,8 @@ interface DropboxEntryJson {
|
||||
path_display: string
|
||||
size?: number
|
||||
server_modified?: string
|
||||
rev?: string
|
||||
content_hash?: string
|
||||
}
|
||||
|
||||
interface SearchMatchJson {
|
||||
@@ -96,6 +99,7 @@ class Account {
|
||||
readonly folders = new Set<string>()
|
||||
readonly files = new Map<string, StoredFile>()
|
||||
readonly searchCursors = new Map<string, { matches: SearchMatchJson[]; start: number; limit: number }>()
|
||||
readonly listCursors = new Map<string, DropboxEntryJson[]>()
|
||||
|
||||
addAncestors(path: string): void {
|
||||
const parts = path.split('/').slice(1, -1)
|
||||
@@ -109,6 +113,12 @@ class Account {
|
||||
entryFor(path: string): DropboxEntryJson | null {
|
||||
const stored = this.files.get(path)
|
||||
if (stored !== undefined) {
|
||||
// Real Dropbox derives content_hash from a SHA-256 over 4 MiB
|
||||
// block digests; the fake hashes the whole content instead, which
|
||||
// is opaque to a client that only ever compares it for equality,
|
||||
// and keeps the property that matters: identical bytes hash
|
||||
// identically.
|
||||
const digest = createHash('sha256').update(stored.data).digest('hex')
|
||||
return {
|
||||
'.tag': 'file',
|
||||
id: `id:${path}`,
|
||||
@@ -117,6 +127,8 @@ class Account {
|
||||
path_display: path,
|
||||
size: stored.data.length,
|
||||
server_modified: stored.modified,
|
||||
rev: digest.slice(0, 16),
|
||||
content_hash: digest,
|
||||
}
|
||||
}
|
||||
if (this.folders.has(path)) {
|
||||
@@ -131,18 +143,37 @@ class Account {
|
||||
return null
|
||||
}
|
||||
|
||||
listChildren(path: string): DropboxEntryJson[] | null {
|
||||
listChildren(path: string, recursive = false): DropboxEntryJson[] | null {
|
||||
if (path !== '' && !this.folders.has(path)) return null
|
||||
const prefix = `${path}/`
|
||||
const under = (candidate: string): boolean =>
|
||||
recursive ? candidate.startsWith(prefix) : candidate.slice(0, candidate.lastIndexOf('/')) === path
|
||||
const out: DropboxEntryJson[] = []
|
||||
for (const folder of this.folders) {
|
||||
if (folder.slice(0, folder.lastIndexOf('/')) !== path) continue
|
||||
out.push(this.entryFor(folder) as DropboxEntryJson)
|
||||
if (under(folder)) out.push(this.entryFor(folder) as DropboxEntryJson)
|
||||
}
|
||||
for (const file of this.files.keys()) {
|
||||
if (file.slice(0, file.lastIndexOf('/')) !== path) continue
|
||||
out.push(this.entryFor(file) as DropboxEntryJson)
|
||||
if (under(file)) out.push(this.entryFor(file) as DropboxEntryJson)
|
||||
}
|
||||
return out.sort((a, b) => (a.name < b.name ? -1 : 1))
|
||||
// A recursive listing is ordered parent-before-child, which is what
|
||||
// real Dropbox guarantees and what a consumer building a tree from
|
||||
// the stream relies on.
|
||||
const key = recursive ? 'path_display' : 'name'
|
||||
return out.sort((a, b) => ((a[key] ?? '') < (b[key] ?? '') ? -1 : 1))
|
||||
}
|
||||
|
||||
// Real list_folder always returns a cursor and pages on `limit`, so
|
||||
// the fake does too: a client that ignores `has_more` sees a short
|
||||
// listing, which is the bug the pagination loop exists to avoid.
|
||||
listPage(
|
||||
entries: DropboxEntryJson[],
|
||||
limit: number,
|
||||
): { entries: DropboxEntryJson[]; cursor: string; has_more: boolean } {
|
||||
const head = entries.slice(0, limit)
|
||||
const tail = entries.slice(limit)
|
||||
const token = `cursor-${this.listCursors.size}`
|
||||
this.listCursors.set(token, tail)
|
||||
return { entries: head, cursor: token, has_more: tail.length > 0 }
|
||||
}
|
||||
|
||||
// Removes a file, or a folder plus its subtree (delete_v2 semantics).
|
||||
@@ -223,13 +254,32 @@ function handle(
|
||||
return
|
||||
}
|
||||
if (url === '/2/files/list_folder') {
|
||||
const { path = '' } = JSON.parse(body.toString('utf8') || '{}') as { path?: string }
|
||||
const entries = account.listChildren(path)
|
||||
const {
|
||||
path = '',
|
||||
recursive = false,
|
||||
limit = 2000,
|
||||
} = JSON.parse(body.toString('utf8') || '{}') as {
|
||||
path?: string
|
||||
recursive?: boolean
|
||||
limit?: number
|
||||
}
|
||||
const entries = account.listChildren(path, recursive)
|
||||
if (entries === null) {
|
||||
jsonError(res, 'path/not_found/...')
|
||||
return
|
||||
}
|
||||
json(res, { entries, cursor: 'cursor-0', has_more: false })
|
||||
json(res, account.listPage(entries, limit))
|
||||
return
|
||||
}
|
||||
if (url === '/2/files/list_folder/continue') {
|
||||
const { cursor = '' } = JSON.parse(body.toString('utf8') || '{}') as { cursor?: string }
|
||||
const rest = account.listCursors.get(cursor)
|
||||
if (rest === undefined) {
|
||||
jsonError(res, 'reset/...')
|
||||
return
|
||||
}
|
||||
account.listCursors.delete(cursor)
|
||||
json(res, account.listPage(rest, 2000))
|
||||
return
|
||||
}
|
||||
if (url === '/2/files/get_metadata') {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
@@ -68,6 +69,7 @@ class FakeDropbox:
|
||||
self.folders: set[str] = set()
|
||||
self.files: dict[str, tuple[bytes, str]] = {}
|
||||
self.search_cursors: dict[str, tuple[list, int, int]] = {}
|
||||
self.list_cursors: dict[str, list[dict]] = {}
|
||||
self.endpoint = ""
|
||||
|
||||
def _add_ancestors(self, path: str) -> None:
|
||||
@@ -80,6 +82,12 @@ class FakeDropbox:
|
||||
def _entry_for(self, path: str) -> dict | None:
|
||||
stored = self.files.get(path)
|
||||
if stored is not None:
|
||||
# Real Dropbox derives content_hash from a SHA-256 over
|
||||
# 4 MiB block digests; the fake hashes the whole content
|
||||
# instead, which is opaque to a client that only ever
|
||||
# compares it for equality, and keeps the property that
|
||||
# matters: identical bytes hash identically.
|
||||
digest = hashlib.sha256(stored[0]).hexdigest()
|
||||
return {
|
||||
".tag": "file",
|
||||
"id": f"id:{path}",
|
||||
@@ -88,6 +96,8 @@ class FakeDropbox:
|
||||
"path_display": path,
|
||||
"size": len(stored[0]),
|
||||
"server_modified": stored[1],
|
||||
"rev": digest[:16],
|
||||
"content_hash": digest,
|
||||
}
|
||||
if path in self.folders:
|
||||
return {
|
||||
@@ -99,17 +109,30 @@ class FakeDropbox:
|
||||
}
|
||||
return None
|
||||
|
||||
def _list_children(self, path: str) -> list[dict] | None:
|
||||
def _list_children(self,
|
||||
path: str,
|
||||
recursive: bool = False) -> list[dict] | None:
|
||||
if path and path not in self.folders:
|
||||
return None
|
||||
prefix = f"{path}/"
|
||||
|
||||
def under(candidate: str) -> bool:
|
||||
if recursive:
|
||||
return candidate.startswith(prefix)
|
||||
return candidate.rsplit("/", 1)[0] == path
|
||||
|
||||
out: list[dict] = []
|
||||
for folder in self.folders:
|
||||
if folder.rsplit("/", 1)[0] == path:
|
||||
if under(folder):
|
||||
out.append(self._entry_for(folder))
|
||||
for file in self.files:
|
||||
if file.rsplit("/", 1)[0] == path:
|
||||
if under(file):
|
||||
out.append(self._entry_for(file))
|
||||
return sorted(out, key=lambda e: e["name"])
|
||||
# A recursive listing is ordered parent-before-child, which is
|
||||
# what real Dropbox guarantees and what a consumer building a
|
||||
# tree from the stream relies on.
|
||||
key = "path_display" if recursive else "name"
|
||||
return sorted(out, key=lambda e: e[key])
|
||||
|
||||
def _remove(self, path: str) -> bool:
|
||||
# Removes a file, or a folder plus its subtree (delete_v2).
|
||||
@@ -151,17 +174,46 @@ class FakeDropbox:
|
||||
"expires_in": 14400,
|
||||
})
|
||||
|
||||
def _list_page(self, entries: list[dict], limit: int) -> dict:
|
||||
"""Cut one page off a listing and park the rest under a cursor.
|
||||
|
||||
Real list_folder always returns a cursor and pages on ``limit``,
|
||||
so the fake does too: a client that ignores ``has_more`` sees a
|
||||
short listing, which is the bug the pagination loop exists to
|
||||
avoid.
|
||||
|
||||
Args:
|
||||
entries (list[dict]): full listing to page.
|
||||
limit (int): maximum entries per page.
|
||||
"""
|
||||
head, tail = entries[:limit], entries[limit:]
|
||||
token = f"cursor-{len(self.list_cursors)}"
|
||||
self.list_cursors[token] = tail
|
||||
return {
|
||||
"entries": head,
|
||||
"cursor": token,
|
||||
"has_more": bool(tail),
|
||||
}
|
||||
|
||||
async def handle_list_folder(self, request: web.Request) -> web.Response:
|
||||
body = await request.json()
|
||||
entries = self._list_children(body.get("path") or "")
|
||||
entries = self._list_children(body.get("path") or "",
|
||||
recursive=bool(body.get("recursive")))
|
||||
if entries is None:
|
||||
return web.json_response({"error_summary": "path/not_found/..."},
|
||||
status=409)
|
||||
return web.json_response({
|
||||
"entries": entries,
|
||||
"cursor": "cursor-0",
|
||||
"has_more": False,
|
||||
})
|
||||
return web.json_response(
|
||||
self._list_page(entries, int(body.get("limit") or 2000)))
|
||||
|
||||
async def handle_list_folder_continue(
|
||||
self, request: web.Request) -> web.Response:
|
||||
body = await request.json()
|
||||
token = body.get("cursor") or ""
|
||||
rest = self.list_cursors.pop(token, None)
|
||||
if rest is None:
|
||||
return web.json_response({"error_summary": "reset/..."},
|
||||
status=409)
|
||||
return web.json_response(self._list_page(rest, 2000))
|
||||
|
||||
async def handle_get_metadata(self, request: web.Request) -> web.Response:
|
||||
body = await request.json()
|
||||
@@ -348,6 +400,8 @@ async def start_fake_dropbox() -> tuple[FakeDropbox, web.AppRunner]:
|
||||
app = web.Application(client_max_size=8 * 1024 * 1024)
|
||||
app.router.add_post("/oauth2/token", fake.handle_token)
|
||||
app.router.add_post("/2/files/list_folder", fake.handle_list_folder)
|
||||
app.router.add_post("/2/files/list_folder/continue",
|
||||
fake.handle_list_folder_continue)
|
||||
app.router.add_post("/2/files/get_metadata", fake.handle_get_metadata)
|
||||
app.router.add_post("/2/files/download", fake.handle_download)
|
||||
app.router.add_post("/2/files/upload", fake.handle_upload)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Shared by both languages: a delta hook reports the same events from the
|
||||
# same directory whichever implementation pulled it, so one truth file
|
||||
# gates examples/python/disk/watch.py and examples/typescript/disk/watch.ts.
|
||||
baseline: 0 changes
|
||||
pull: 3 changes
|
||||
update /data/reports/q1.txt
|
||||
delete /data/reports/q2.txt
|
||||
create /data/reports/q3.txt
|
||||
read after notify: 'first quarter, revised'
|
||||
listing: q1.txt q3.txt
|
||||
@@ -0,0 +1,591 @@
|
||||
{
|
||||
"resources": [
|
||||
"disk",
|
||||
"ssh",
|
||||
"dropbox",
|
||||
"s3",
|
||||
"gridfs",
|
||||
"onedrive",
|
||||
"box",
|
||||
"hf_buckets",
|
||||
"gdrive",
|
||||
"github"
|
||||
],
|
||||
"mount": "/m",
|
||||
"watch_dir": "/m/data",
|
||||
"modes": [
|
||||
"pull"
|
||||
],
|
||||
"seed": [
|
||||
"keep.txt"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "create_file",
|
||||
"warm": [
|
||||
"ls /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/report.txt",
|
||||
"body": "alpha line\nbeta line\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/report.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/report.txt",
|
||||
"contains": "alpha line"
|
||||
},
|
||||
{
|
||||
"cmd": "head -n 1 /m/data/report.txt",
|
||||
"contains": "alpha line"
|
||||
},
|
||||
{
|
||||
"cmd": "ls /m/data",
|
||||
"contains": "report.txt"
|
||||
},
|
||||
{
|
||||
"cmd": "grep -rl beta /m/data",
|
||||
"contains": "report.txt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "update_etag_change",
|
||||
"warm": [
|
||||
"cat /m/data/report.txt",
|
||||
"grep -rl alpha /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/report.txt",
|
||||
"body": "gamma line\ndelta line\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "update",
|
||||
"path": "/m/data/report.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/report.txt",
|
||||
"contains": "gamma line"
|
||||
},
|
||||
{
|
||||
"cmd": "cat /m/data/report.txt",
|
||||
"absent": "alpha"
|
||||
},
|
||||
{
|
||||
"cmd": "grep -rl gamma /m/data",
|
||||
"contains": "report.txt"
|
||||
},
|
||||
{
|
||||
"cmd": "grep -r alpha /m/data",
|
||||
"absent": "report.txt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "create_nested",
|
||||
"warm": [
|
||||
"ls /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/sub/deep.txt",
|
||||
"body": "deep marker\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/sub/deep.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/sub/deep.txt",
|
||||
"contains": "deep marker"
|
||||
},
|
||||
{
|
||||
"cmd": "grep -rl 'deep marker' /m/data",
|
||||
"contains": "deep.txt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "delete_file",
|
||||
"warm": [
|
||||
"cat /m/data/report.txt",
|
||||
"ls /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "delete",
|
||||
"path": "data/report.txt"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "delete",
|
||||
"path": "/m/data/report.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "ls /m/data",
|
||||
"absent": "report.txt"
|
||||
},
|
||||
{
|
||||
"cmd": "grep -rl gamma /m/data",
|
||||
"absent": "report.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"scopes": [
|
||||
{
|
||||
"id": "folder",
|
||||
"watch": [
|
||||
"/m/data/sub"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "inside_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/sub/in.txt",
|
||||
"body": "inside scope\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/sub/in.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/sub/in.txt",
|
||||
"contains": "inside scope"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "outside_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/out.txt",
|
||||
"body": "outside scope\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/out.txt",
|
||||
"delivered": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "pattern",
|
||||
"watch": [
|
||||
"/m/data/*.txt"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "txt_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/notes.txt",
|
||||
"body": "txt matches\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/notes.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/notes.txt",
|
||||
"contains": "txt matches"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "md_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/notes.md",
|
||||
"body": "md does not match\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/notes.md",
|
||||
"delivered": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "nested_txt_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/sub/deep.txt",
|
||||
"body": "glob does not cross slash\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/sub/deep.txt",
|
||||
"delivered": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "shallow_glob",
|
||||
"watch": [
|
||||
"/m/data/*"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "top_level_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/top.txt",
|
||||
"body": "shallow scope\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/top.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/top.txt",
|
||||
"contains": "shallow scope"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "nested_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/sub/deep.txt",
|
||||
"body": "below shallow scope\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/sub/deep.txt",
|
||||
"delivered": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "dir_glob",
|
||||
"watch": [
|
||||
"/m/data/*/"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "inside_dir_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/sub/in.txt",
|
||||
"body": "inside a child dir\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/sub/in.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/sub/in.txt",
|
||||
"contains": "inside a child dir"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "top_level_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/top.txt",
|
||||
"body": "not inside a dir\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/top.txt",
|
||||
"delivered": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "mid_wildcard",
|
||||
"watch": [
|
||||
"/m/data/*/reports/"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "inside_reports_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/proj1/reports/spec.md",
|
||||
"body": "quarterly spec\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/proj1/reports/spec.md"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/proj1/reports/spec.md",
|
||||
"contains": "quarterly spec"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "project_sibling_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/proj1/other.txt",
|
||||
"body": "outside reports\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/proj1/other.txt",
|
||||
"delivered": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "multi_roots",
|
||||
"watch": [
|
||||
"/m/data/sub",
|
||||
"/m/data/*.yaml"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "literal_root_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/sub/in.txt",
|
||||
"body": "literal subtree\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/sub/in.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/sub/in.txt",
|
||||
"contains": "literal subtree"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "glob_root_delivered",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/app.yaml",
|
||||
"body": "yaml root\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/app.yaml"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/app.yaml",
|
||||
"contains": "yaml root"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "neither_root_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/note.md",
|
||||
"body": "matches no root\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/note.md",
|
||||
"delivered": false
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rename",
|
||||
"watch": [
|
||||
"/m/data"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "seed_orig",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/orig.txt",
|
||||
"body": "original content\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/orig.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/orig.txt",
|
||||
"contains": "original content"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rename_as_move",
|
||||
"modes": [
|
||||
"push"
|
||||
],
|
||||
"warm": [
|
||||
"cat /m/data/orig.txt",
|
||||
"ls /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "rename",
|
||||
"path": "data/orig.txt",
|
||||
"to": "data/renamed.txt"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "move",
|
||||
"path": "/m/data/renamed.txt",
|
||||
"previous": "/m/data/orig.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/renamed.txt",
|
||||
"contains": "original content"
|
||||
},
|
||||
{
|
||||
"cmd": "ls /m/data",
|
||||
"absent": "orig.txt"
|
||||
},
|
||||
{
|
||||
"cmd": "cat /m/data/orig.txt",
|
||||
"absent": "original content"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rename_as_delete_create",
|
||||
"modes": [
|
||||
"pull"
|
||||
],
|
||||
"warm": [
|
||||
"cat /m/data/orig.txt",
|
||||
"ls /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "rename",
|
||||
"path": "data/orig.txt",
|
||||
"to": "data/renamed.txt"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/renamed.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/renamed.txt",
|
||||
"contains": "original content"
|
||||
},
|
||||
{
|
||||
"cmd": "ls /m/data",
|
||||
"absent": "orig.txt"
|
||||
},
|
||||
{
|
||||
"cmd": "cat /m/data/orig.txt",
|
||||
"absent": "original content"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"skip_resources": [
|
||||
"hf_buckets"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "file",
|
||||
"watch": [
|
||||
"/m/data/keep.txt"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"id": "update_delivered",
|
||||
"warm": [
|
||||
"cat /m/data/keep.txt"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/keep.txt",
|
||||
"body": "fresh keep\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "update",
|
||||
"path": "/m/data/keep.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "cat /m/data/keep.txt",
|
||||
"contains": "fresh keep"
|
||||
},
|
||||
{
|
||||
"cmd": "cat /m/data/keep.txt",
|
||||
"absent": "seed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "sibling_skipped",
|
||||
"mutate": {
|
||||
"op": "write",
|
||||
"path": "data/other.txt",
|
||||
"body": "sibling file\n"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "create",
|
||||
"path": "/m/data/other.txt",
|
||||
"delivered": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "delete_delivered",
|
||||
"warm": [
|
||||
"ls /m/data"
|
||||
],
|
||||
"mutate": {
|
||||
"op": "delete",
|
||||
"path": "data/keep.txt"
|
||||
},
|
||||
"expect": {
|
||||
"kind": "delete",
|
||||
"path": "/m/data/keep.txt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "ls /m/data",
|
||||
"absent": "keep.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"overflow": {
|
||||
"max_pending": 2,
|
||||
"paths": [
|
||||
"data/f1.txt",
|
||||
"data/f2.txt",
|
||||
"data/f3.txt",
|
||||
"data/f4.txt",
|
||||
"data/f5.txt"
|
||||
],
|
||||
"checks": [
|
||||
{
|
||||
"cmd": "ls /m/data",
|
||||
"contains": "f5.txt"
|
||||
},
|
||||
{
|
||||
"cmd": "cat /m/data/f3.txt",
|
||||
"contains": "burst"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
import aiohttp
|
||||
from pydantic import SecretStr
|
||||
|
||||
from mirage import MountMode, Workspace
|
||||
from mirage.core.github._client import github_request
|
||||
from mirage.core.github.read import read_bytes
|
||||
from mirage.core.github.tree import fetch_tree
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.resource.box import BoxConfig, BoxResource
|
||||
from mirage.resource.disk import DiskResource
|
||||
from mirage.resource.dropbox import DropboxConfig, DropboxResource
|
||||
from mirage.resource.gdrive import GoogleDriveResource
|
||||
from mirage.resource.gdrive.config import GoogleDriveConfig
|
||||
from mirage.resource.github import GitHubConfig, GitHubResource
|
||||
from mirage.resource.gridfs import GridFSConfig, GridFSResource
|
||||
from mirage.resource.hf_buckets import HfBucketsConfig, HfBucketsResource
|
||||
from mirage.resource.onedrive import OneDriveConfig, OneDriveResource
|
||||
from mirage.resource.s3 import S3Config, S3Resource
|
||||
from mirage.resource.ssh import SSHConfig, SSHResource
|
||||
|
||||
SERVER_DIR = Path(__file__).resolve().parents[1] / "server"
|
||||
GITHUB_OWNER = "integ"
|
||||
GITHUB_REPO = "watch"
|
||||
GITHUB_REF = "main"
|
||||
|
||||
Pair = tuple[Workspace, "WorkspaceWriter | GitHubWriter"]
|
||||
ResourceFactory = Callable[[], Any]
|
||||
|
||||
|
||||
class WorkspaceWriter:
|
||||
"""External writer backed by a second workspace over one backend.
|
||||
|
||||
The batteries need a mutation the watched workspace did not make, so
|
||||
its caches are genuinely stale when the event lands. A second
|
||||
workspace over the same backend is exactly that: its own cache
|
||||
manager, its own index, the same bytes underneath. It also gives
|
||||
every backend one writer instead of one adapter per API, which is
|
||||
what lets the same cases run against all of them.
|
||||
|
||||
The surface mirrors the opendal operator the Nextcloud battery
|
||||
writes through, so ``_mutate`` and ``_seed`` need no branch.
|
||||
"""
|
||||
|
||||
def __init__(self, ws: Workspace, mount: str) -> None:
|
||||
"""Args:
|
||||
ws (Workspace): Writer workspace, distinct from the watched
|
||||
one.
|
||||
mount (str): Mount prefix both workspaces share.
|
||||
"""
|
||||
self._ws = ws
|
||||
self._mount = mount.rstrip("/")
|
||||
|
||||
def _virtual(self, path: str) -> str:
|
||||
return f"{self._mount}/{path.strip('/')}"
|
||||
|
||||
async def create_dir(self, path: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative directory, trailing slash
|
||||
optional.
|
||||
"""
|
||||
await self._ws.execute(f"mkdir -p {self._virtual(path)}")
|
||||
|
||||
async def write(self, path: str, data: bytes) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative file path.
|
||||
data (bytes): Content to store.
|
||||
"""
|
||||
key = path.strip("/")
|
||||
parent = key.rsplit("/", 1)[0]
|
||||
if parent != key:
|
||||
await self.create_dir(parent)
|
||||
await self._ws.ops.write(self._virtual(key), data)
|
||||
|
||||
async def delete(self, path: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative path; a directory goes with its
|
||||
subtree, matching opendal's delete.
|
||||
"""
|
||||
await self._ws.execute(f"rm -rf {self._virtual(path)}")
|
||||
|
||||
async def remove_all(self, path: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative subtree to empty.
|
||||
"""
|
||||
await self.delete(path)
|
||||
|
||||
async def rename(self, path: str, to: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative source.
|
||||
to (str): Mount-relative destination.
|
||||
"""
|
||||
await self._ws.ops.rename(self._virtual(path), self._virtual(to))
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._ws.close()
|
||||
|
||||
|
||||
class GitHubWriter:
|
||||
"""External writer over GitHub's own contents API.
|
||||
|
||||
Every other backend here is written through a second workspace,
|
||||
which needs the resource to have write ops. GitHub's has none, and
|
||||
should not: a mount is a read view of one ref, and a ref changes by
|
||||
being committed to. So this speaks what a committer speaks, ``PUT``
|
||||
and ``DELETE`` on ``/contents/{path}``, each carrying the blob sha
|
||||
it replaces, which is also the strongest fingerprint the differ
|
||||
ever compares.
|
||||
|
||||
Git stores no directory object, so ``create_dir`` is a no-op: a
|
||||
directory exists exactly as long as a path runs through it. That is
|
||||
also why ``remove_all`` enumerates the tree rather than deleting
|
||||
one marker, and why ``rename`` is a write plus a delete.
|
||||
"""
|
||||
|
||||
def __init__(self, config: GitHubConfig, owner: str, repo: str,
|
||||
ref: str) -> None:
|
||||
"""Args:
|
||||
config (GitHubConfig): Token and API base of the fake.
|
||||
owner (str): Repository owner.
|
||||
repo (str): Repository name.
|
||||
ref (str): Branch the mount is pinned to.
|
||||
"""
|
||||
self._config = config
|
||||
self._owner = owner
|
||||
self._repo = repo
|
||||
self._ref = ref
|
||||
|
||||
async def _tree(self) -> dict[str, TreeEntry]:
|
||||
"""The ref's recursive tree, which names every blob's sha."""
|
||||
tree, _truncated = await fetch_tree(self._config, self._owner,
|
||||
self._repo, self._ref)
|
||||
return tree
|
||||
|
||||
async def _commit(self, method: str, path: str, body: dict[str,
|
||||
str]) -> None:
|
||||
"""Send one contents-API call against the pinned branch.
|
||||
|
||||
Args:
|
||||
method (str): "PUT" or "DELETE".
|
||||
path (str): Repo-relative path.
|
||||
body (dict[str, str]): Call-specific fields.
|
||||
"""
|
||||
await github_request(
|
||||
self._config.token,
|
||||
method,
|
||||
f"/repos/{self._owner}/{self._repo}/contents/{path}", {
|
||||
"branch": self._ref,
|
||||
"message": f"integ watch {path}",
|
||||
**body,
|
||||
},
|
||||
base_url=self._config.base_url)
|
||||
|
||||
async def create_dir(self, path: str) -> None:
|
||||
"""No-op: git has no directory object to create.
|
||||
|
||||
Args:
|
||||
path (str): Mount-relative directory, ignored.
|
||||
"""
|
||||
|
||||
async def write(self, path: str, data: bytes) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative file path.
|
||||
data (bytes): Content to commit.
|
||||
"""
|
||||
key = path.strip("/")
|
||||
entry = (await self._tree()).get(key)
|
||||
body = {"content": base64.b64encode(data).decode("ascii")}
|
||||
if entry is not None:
|
||||
body["sha"] = entry.sha
|
||||
await self._commit("PUT", key, body)
|
||||
|
||||
async def delete(self, path: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative file path.
|
||||
"""
|
||||
entry = (await self._tree()).get(path.strip("/"))
|
||||
if entry is not None:
|
||||
await self._commit("DELETE", entry.path, {"sha": entry.sha})
|
||||
|
||||
async def remove_all(self, path: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative subtree to empty.
|
||||
"""
|
||||
stem = path.strip("/")
|
||||
base = f"{stem}/" if stem else ""
|
||||
for entry in (await self._tree()).values():
|
||||
if entry.type == "tree" or not entry.path.startswith(base):
|
||||
continue
|
||||
await self._commit("DELETE", entry.path, {"sha": entry.sha})
|
||||
|
||||
async def rename(self, path: str, to: str) -> None:
|
||||
"""Args:
|
||||
path (str): Mount-relative source.
|
||||
to (str): Mount-relative destination.
|
||||
"""
|
||||
entry = (await self._tree())[path.strip("/")]
|
||||
data = await read_bytes(self._config, self._owner, self._repo,
|
||||
entry.sha)
|
||||
await self.write(to, data)
|
||||
await self._commit("DELETE", entry.path, {"sha": entry.sha})
|
||||
|
||||
|
||||
def _load(path: Path, name: str) -> ModuleType:
|
||||
"""Import one integ server module by path.
|
||||
|
||||
Args:
|
||||
path (Path): Module file.
|
||||
name (str): Name to register it under.
|
||||
"""
|
||||
spec_obj = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec_obj)
|
||||
spec_obj.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _pair(spec: dict, make: ResourceFactory) -> Pair:
|
||||
"""Build the watched workspace and its external writer.
|
||||
|
||||
Each side gets its own resource instance over the same backend, so
|
||||
nothing is shared but the bytes.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
make (ResourceFactory): Builds one fresh resource.
|
||||
"""
|
||||
mount = spec["mount"]
|
||||
watched = Workspace({mount: make()}, mode=MountMode.WRITE)
|
||||
writer = Workspace({mount: make()}, mode=MountMode.WRITE)
|
||||
return watched, WorkspaceWriter(writer, mount)
|
||||
|
||||
|
||||
async def build_disk(spec: dict) -> Pair | None:
|
||||
"""Disk battery: a throwaway directory, no service at all.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
root = tempfile.mkdtemp(prefix="mirage-watch-disk-")
|
||||
return _pair(spec, lambda: DiskResource(root))
|
||||
|
||||
|
||||
async def build_ssh(spec: dict) -> Pair | None:
|
||||
"""SSH battery against the in-process asyncssh SFTP server.
|
||||
|
||||
A real server on a real socket, chrooted to a throwaway directory,
|
||||
so the per-directory SFTP descent is exercised end to end.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
module = _load(SERVER_DIR / "ssh_server.py", "integ_watch_ssh")
|
||||
root = tempfile.mkdtemp(prefix="mirage-watch-ssh-")
|
||||
server = await module.start_server(root)
|
||||
port = server.get_port()
|
||||
return _pair(
|
||||
spec, lambda: SSHResource(
|
||||
SSHConfig(host="127.0.0.1",
|
||||
port=port,
|
||||
username="integ",
|
||||
known_hosts=None,
|
||||
root="/")))
|
||||
|
||||
|
||||
async def build_dropbox(spec: dict) -> Pair | None:
|
||||
"""Dropbox battery against the in-process fake.
|
||||
|
||||
Exercises the recursive ``list_folder`` and its cursor pagination,
|
||||
and fingerprints on the fake's ``content_hash``.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
module = _load(SERVER_DIR / "dropbox_server.py", "integ_watch_dropbox")
|
||||
fake, _runner = await module.start_fake_dropbox()
|
||||
return _pair(
|
||||
spec, lambda: DropboxResource(
|
||||
DropboxConfig(client_id="integ-client",
|
||||
client_secret="integ-secret",
|
||||
refresh_token="integ-refresh",
|
||||
endpoint=fake.endpoint,
|
||||
root_path="/")))
|
||||
|
||||
|
||||
async def build_s3(spec: dict) -> Pair | None:
|
||||
"""S3 battery against whatever ``S3_ENDPOINT`` names (MinIO in CI).
|
||||
|
||||
Each run takes its own key prefix so repeat runs cannot see each
|
||||
other's objects.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
endpoint = os.environ.get("S3_ENDPOINT")
|
||||
bucket = os.environ.get("S3_BUCKET")
|
||||
if not endpoint or not bucket:
|
||||
return None
|
||||
prefix = f"watch-{uuid.uuid4().hex[:8]}/"
|
||||
return _pair(
|
||||
spec, lambda: S3Resource(
|
||||
S3Config(bucket=bucket,
|
||||
region=os.environ.get("S3_REGION", "us-east-1"),
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=os.environ.get(
|
||||
"AWS_SECRET_ACCESS_KEY"),
|
||||
path_style=True,
|
||||
key_prefix=prefix)))
|
||||
|
||||
|
||||
async def build_gridfs(spec: dict) -> Pair | None:
|
||||
"""GridFS battery against ``MONGODB_URI``.
|
||||
|
||||
Each run takes its own database so repeat runs stay isolated.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
uri = os.environ.get("MONGODB_URI")
|
||||
if not uri:
|
||||
return None
|
||||
database = f"watch_{uuid.uuid4().hex[:8]}"
|
||||
return _pair(
|
||||
spec, lambda: GridFSResource(
|
||||
GridFSConfig(uri=uri, database=database, bucket="fs")))
|
||||
|
||||
|
||||
async def build_onedrive(spec: dict) -> Pair | None:
|
||||
"""OneDrive battery against the in-process Graph fake.
|
||||
|
||||
This is the ReaddirWalk path: Graph keys its tree by item id and has
|
||||
no whole-subtree listing, so the walk descends one
|
||||
``/children`` request per directory against a fresh private index.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
module = _load(SERVER_DIR / "onedrive_server.py", "integ_watch_onedrive")
|
||||
state, _server, _runner = await module.start_fake_graph()
|
||||
return _pair(
|
||||
spec, lambda: OneDriveResource(
|
||||
OneDriveConfig(access_token="integ-token",
|
||||
graph_base_url=state.base)))
|
||||
|
||||
|
||||
async def build_box(spec: dict) -> Pair | None:
|
||||
"""Box battery against the in-process fake.
|
||||
|
||||
The second ReaddirWalk target, and the one that proves the walk is
|
||||
not Graph-shaped: Box addresses folders by its own ids and answers a
|
||||
different listing endpoint.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
module = _load(SERVER_DIR / "box_server.py", "integ_watch_box")
|
||||
state, _server, _runner = await module.start_fake_box()
|
||||
folder = state.add_folder("0", "watch")
|
||||
return _pair(
|
||||
spec, lambda: BoxResource(
|
||||
BoxConfig(access_token="integ-box-token",
|
||||
endpoint=state.base,
|
||||
root_folder_id=folder["id"])))
|
||||
|
||||
|
||||
async def build_hf(spec: dict) -> Pair | None:
|
||||
"""Hugging Face battery against the in-process Hub fake.
|
||||
|
||||
Covers the shared opendal walk on a lister that omits per-entry
|
||||
metadata, which is the stat-backfill branch Nextcloud never reaches.
|
||||
The fake freezes ``modified``, so only the ETag can move: a case that
|
||||
passes here passes on the ETag alone.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
module = _load(SERVER_DIR / "hf_server.py", "integ_watch_hf")
|
||||
_hub, server, _runner = await module.start_fake_hub()
|
||||
bucket = f"integ/watch-{uuid.uuid4().hex[:8]}"
|
||||
return _pair(
|
||||
spec, lambda: HfBucketsResource(
|
||||
HfBucketsConfig(
|
||||
bucket=bucket, token="integ-token", endpoint=server.endpoint)))
|
||||
|
||||
|
||||
async def build_gdrive(spec: dict) -> Pair | None:
|
||||
"""Google Drive battery against the external gws fake.
|
||||
|
||||
The gws server is TypeScript and shared across runs, so this needs
|
||||
``GWS_URL`` and each run resets it and takes its own folder.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
url = os.environ.get("GWS_URL")
|
||||
if not url:
|
||||
return None
|
||||
url = url.rstrip("/")
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(f"{url}/reset", json={}) as resp:
|
||||
resp.raise_for_status()
|
||||
folder = f"watch-{uuid.uuid4().hex[:8]}"
|
||||
async with session.post(f"{url}/drive/v3/files",
|
||||
json={
|
||||
"name":
|
||||
folder,
|
||||
"mimeType":
|
||||
"application/vnd.google-apps.folder",
|
||||
}) as resp:
|
||||
resp.raise_for_status()
|
||||
folder_id = (await resp.json())["id"]
|
||||
return _pair(
|
||||
spec, lambda: GoogleDriveResource(
|
||||
GoogleDriveConfig(client_id="integ-client",
|
||||
client_secret="integ-secret",
|
||||
refresh_token="integ-refresh",
|
||||
api_base=url,
|
||||
folder_id=folder_id)))
|
||||
|
||||
|
||||
async def build_github(spec: dict) -> Pair | None:
|
||||
"""GitHub battery against the in-process fake.
|
||||
|
||||
The one target whose writer is not a workspace, because a git ref
|
||||
has no write ops to lend one. It is also the only backend whose
|
||||
fingerprint is a git blob sha, so a rewrite of identical bytes is
|
||||
correctly reported as nothing at all.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
module = _load(SERVER_DIR / "github_server.py", "integ_watch_github")
|
||||
state, _server, _runner = await module.start_fake_github()
|
||||
state.repo(GITHUB_OWNER, GITHUB_REPO)
|
||||
config = GitHubConfig(token=SecretStr("integ-github-token"),
|
||||
owner=GITHUB_OWNER,
|
||||
repo=GITHUB_REPO,
|
||||
ref=GITHUB_REF,
|
||||
base_url=state.base)
|
||||
resource = await GitHubResource.build(config)
|
||||
ws = Workspace({spec["mount"]: resource}, mode=MountMode.WRITE)
|
||||
return ws, GitHubWriter(config, GITHUB_OWNER, GITHUB_REPO, GITHUB_REF)
|
||||
|
||||
|
||||
BUILDERS: dict[str, Callable[[dict], Awaitable[Pair | None]]] = {
|
||||
"disk": build_disk,
|
||||
"ssh": build_ssh,
|
||||
"dropbox": build_dropbox,
|
||||
"s3": build_s3,
|
||||
"gridfs": build_gridfs,
|
||||
"onedrive": build_onedrive,
|
||||
"box": build_box,
|
||||
"hf_buckets": build_hf,
|
||||
"gdrive": build_gdrive,
|
||||
"github": build_github,
|
||||
}
|
||||
+69
-20
@@ -21,6 +21,7 @@ from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
from backends import BUILDERS as BACKEND_BUILDERS
|
||||
from webhook_server import make_app
|
||||
|
||||
from mirage import MountMode, Workspace
|
||||
@@ -53,7 +54,7 @@ def _nextcloud_config(url: str) -> NextcloudConfig:
|
||||
)
|
||||
|
||||
|
||||
def _build_nextcloud(spec: dict) -> tuple[Workspace, object] | None:
|
||||
async def _build_nextcloud(spec: dict) -> tuple[Workspace, object] | None:
|
||||
"""Build the watched workspace and a separate external writer.
|
||||
|
||||
Returns None when the deployment env is absent, so a local run
|
||||
@@ -72,7 +73,8 @@ def _build_nextcloud(spec: dict) -> tuple[Workspace, object] | None:
|
||||
return ws, external
|
||||
|
||||
|
||||
def _build_nextcloud_nested(spec: dict) -> tuple[Workspace, object] | None:
|
||||
async def _build_nextcloud_nested(
|
||||
spec: dict) -> tuple[Workspace, object] | None:
|
||||
"""Build the nested-mount battery's workspace: the outer mount at
|
||||
the account root plus a second mount, rooted at a subfolder of the
|
||||
same account, nested inside the outer mount's subtree.
|
||||
@@ -97,7 +99,7 @@ def _build_nextcloud_nested(spec: dict) -> tuple[Workspace, object] | None:
|
||||
return ws, external
|
||||
|
||||
|
||||
BUILDERS = {"nextcloud": _build_nextcloud}
|
||||
BUILDERS = {"nextcloud": _build_nextcloud, **BACKEND_BUILDERS}
|
||||
|
||||
|
||||
def _files_prefix() -> str:
|
||||
@@ -105,14 +107,23 @@ def _files_prefix() -> str:
|
||||
return f"/{os.environ.get('NEXTCLOUD_USERNAME', 'admin')}/files"
|
||||
|
||||
|
||||
def _watch_rel(spec: dict) -> str:
|
||||
"""The watch dir as the external writer spells it, mount-relative.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
return spec["watch_dir"][len(spec["mount"].rstrip("/")):].strip("/")
|
||||
|
||||
|
||||
def _framed_root(spec: dict) -> PathSpec:
|
||||
"""Build the mount-framed watch_dir root the delta hook pulls over.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
rel = spec["watch_dir"][len(spec["mount"].rstrip("/")):].strip("/")
|
||||
return PathSpec.from_str_path(spec["watch_dir"], resource_path=rel)
|
||||
return PathSpec.from_str_path(spec["watch_dir"],
|
||||
resource_path=_watch_rel(spec))
|
||||
|
||||
|
||||
async def _mutate(op: object, mutate: dict) -> None:
|
||||
@@ -362,11 +373,22 @@ async def _run_case(ws: Workspace, op: object, trigger, stream: EventStream,
|
||||
async def _seed(ws: Workspace, op: object, spec: dict) -> None:
|
||||
"""Reset the watch dir and lay down the seed files.
|
||||
|
||||
Both halves of the reset are load-bearing. The external writer
|
||||
empties the directory, because a mount that is a read view of its
|
||||
backend cannot ``rm -rf`` its own watch dir (github's is one: a ref
|
||||
changes by being committed to), and a battery whose reset silently
|
||||
did nothing carries one scope's files into the next, where their
|
||||
CREATEs arrive as UPDATEs. The watched workspace then runs its own
|
||||
``rm -rf``, which is what drops the listing it had cached.
|
||||
|
||||
Args:
|
||||
ws (Workspace): Watched workspace.
|
||||
op (object): External writer operator.
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
root = _watch_rel(spec) + "/"
|
||||
await op.create_dir(root)
|
||||
await op.remove_all(root)
|
||||
await ws.execute(f"rm -rf {spec['watch_dir']}")
|
||||
await ws.execute(f"mkdir -p {spec['watch_dir']}")
|
||||
for name in spec["seed"]:
|
||||
@@ -450,7 +472,7 @@ async def _overflow_core(spec: dict, ws: Workspace, op: object, trigger,
|
||||
await stream.close()
|
||||
|
||||
|
||||
def _overflow_workspace(spec: dict) -> tuple[Workspace, object]:
|
||||
async def _overflow_workspace(spec: dict) -> tuple[Workspace, object]:
|
||||
"""Build the overflow battery's own workspace with a tiny queue.
|
||||
|
||||
A dedicated workspace is required because the custom queue factory
|
||||
@@ -459,7 +481,7 @@ def _overflow_workspace(spec: dict) -> tuple[Workspace, object]:
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
ws, op = BUILDERS[spec["resource"]](spec)
|
||||
ws, op = await BUILDERS[spec["resource"]](spec)
|
||||
ws.attach_watch_runtime(
|
||||
Watcher(ws.registry,
|
||||
queue_factory=partial(
|
||||
@@ -477,7 +499,7 @@ async def _run_overflow_pull(spec: dict, results: list) -> None:
|
||||
"""
|
||||
if "overflow" not in spec:
|
||||
return
|
||||
ws, op = _overflow_workspace(spec)
|
||||
ws, op = await _overflow_workspace(spec)
|
||||
try:
|
||||
await _seed(ws, op, spec)
|
||||
resource = ws.registry.mount_for(spec["mount"]).resource
|
||||
@@ -499,7 +521,7 @@ async def _run_overflow_push(spec: dict, results: list) -> None:
|
||||
"""
|
||||
if "overflow" not in spec:
|
||||
return
|
||||
ws, op = _overflow_workspace(spec)
|
||||
ws, op = await _overflow_workspace(spec)
|
||||
try:
|
||||
await _seed(ws, op, spec)
|
||||
runner = web.AppRunner(make_app(ws, _files_prefix(), spec["mount"]))
|
||||
@@ -577,7 +599,7 @@ async def _run_nested_pull(spec: dict, results: list) -> None:
|
||||
"""
|
||||
if "nested" not in spec:
|
||||
return
|
||||
built = _build_nextcloud_nested(spec)
|
||||
built = await _build_nextcloud_nested(spec)
|
||||
if built is None:
|
||||
return
|
||||
ws, op = built
|
||||
@@ -605,7 +627,7 @@ async def _run_nested_push(spec: dict, results: list) -> None:
|
||||
"""
|
||||
if "nested" not in spec:
|
||||
return
|
||||
built = _build_nextcloud_nested(spec)
|
||||
built = await _build_nextcloud_nested(spec)
|
||||
if built is None:
|
||||
return
|
||||
ws, op = built
|
||||
@@ -653,6 +675,10 @@ async def _run_pull(spec: dict, ws: Workspace,
|
||||
spec["cases"], "pull", "pull"))
|
||||
|
||||
for scope in spec.get("scopes", []):
|
||||
# A scope whose mutation the backend has no op for (hf has no
|
||||
# rename) is declared inapplicable rather than run and excused.
|
||||
if spec["resource"] in scope.get("skip_resources", []):
|
||||
continue
|
||||
await _seed(ws, op, spec)
|
||||
agen = ws.watch(scope["watch"])
|
||||
poller = ConsumerPoller(resource.delta_hook(), ws, hook_root)
|
||||
@@ -718,31 +744,54 @@ async def _run_file(spec: dict) -> list[tuple[str, bool, str]]:
|
||||
builder = BUILDERS.get(spec["resource"])
|
||||
if builder is None:
|
||||
return [(spec["resource"], False, "no builder")]
|
||||
modes = {"pull": _run_pull, "push": _run_push}
|
||||
# Push mode needs a provider that can send a webhook, which only the
|
||||
# Nextcloud deployment has; every other backend declares pull only.
|
||||
wanted = spec.get("modes", ["pull", "push"])
|
||||
results: list[tuple[str, bool, str]] = []
|
||||
for mode in (_run_pull, _run_push):
|
||||
built = builder(spec)
|
||||
for name in wanted:
|
||||
built = await builder(spec)
|
||||
if built is None:
|
||||
print(f"skip [{spec['resource']}]: deployment env absent",
|
||||
file=sys.stderr)
|
||||
return []
|
||||
ws, op = built
|
||||
try:
|
||||
results.extend(await mode(spec, ws, op))
|
||||
results.extend(await modes[name](spec, ws, op))
|
||||
finally:
|
||||
await ws.close()
|
||||
closer = getattr(op, "close", None)
|
||||
if closer is not None:
|
||||
await closer()
|
||||
return results
|
||||
|
||||
|
||||
def _expand(spec: dict) -> list[dict]:
|
||||
"""Fan one case file out over the resources it names.
|
||||
|
||||
A file that declares ``resources`` runs its whole body once per
|
||||
backend, so the shared batteries are written down once rather than
|
||||
copied per target.
|
||||
|
||||
Args:
|
||||
spec (dict): Parsed case file.
|
||||
"""
|
||||
names = spec.get("resources")
|
||||
if not names:
|
||||
return [spec]
|
||||
return [{**spec, "resource": name} for name in names]
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
files = sorted(p for p in CASE_DIR.glob("*.json"))
|
||||
failed = 0
|
||||
for path in files:
|
||||
spec = json.loads(path.read_text())
|
||||
for case_id, ok, detail in await _run_file(spec):
|
||||
status = "PASS" if ok else "FAIL"
|
||||
print(f"{status} [{spec['resource']}] {case_id}: {detail}")
|
||||
if not ok:
|
||||
failed += 1
|
||||
for spec in _expand(json.loads(path.read_text())):
|
||||
for case_id, ok, detail in await _run_file(spec):
|
||||
status = "PASS" if ok else "FAIL"
|
||||
print(f"{status} [{spec['resource']}] {case_id}: {detail}")
|
||||
if not ok:
|
||||
failed += 1
|
||||
if failed:
|
||||
print(f"FAIL: {failed} watch case(s) failed", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.accessor.base import Accessor
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
|
||||
|
||||
class GitHubAccessor(Accessor):
|
||||
@@ -23,10 +24,18 @@ class GitHubAccessor(Accessor):
|
||||
repo,
|
||||
ref,
|
||||
default_branch,
|
||||
tree: dict[str, TreeEntry] | None = None,
|
||||
truncated=False):
|
||||
self.config = config
|
||||
self.owner = owner
|
||||
self.repo = repo
|
||||
self.ref = ref
|
||||
self.default_branch = default_branch
|
||||
# The recursive git tree, keyed repo-relative with no leading
|
||||
# slash, which is this mount's whole listing. find, du and grep's
|
||||
# scope counter read it straight, the way TypeScript's always
|
||||
# have: repo-relative path logic belongs on a git tree, not on an
|
||||
# index whose keys are the mount's business. Reseated by every
|
||||
# refill, so it is as fresh as the last one.
|
||||
self.tree: dict[str, TreeEntry] = tree if tree is not None else {}
|
||||
self.truncated = truncated
|
||||
|
||||
Vendored
+19
-9
@@ -36,8 +36,9 @@ class CacheManager:
|
||||
"""Args:
|
||||
file_cache (FileCacheMixin | None): Workspace file cache
|
||||
store; entries are keyed by mount-absolute path.
|
||||
index (IndexCacheStore): The mount resource's index
|
||||
cache; listings are keyed by mount-absolute path.
|
||||
index (IndexCacheStore): The mount resource's index cache;
|
||||
listings are keyed by mount-absolute path, which every
|
||||
backend agrees on.
|
||||
prefix (str): Mount prefix (e.g. "/data/").
|
||||
caches_reads (bool): Whether the resource caches reads; the
|
||||
file cache only holds paths for read-caching backends.
|
||||
@@ -47,6 +48,19 @@ class CacheManager:
|
||||
self._prefix = prefix.rstrip("/")
|
||||
self._caches_reads = caches_reads
|
||||
|
||||
async def _evict_dir(self, virtual: str) -> None:
|
||||
"""Drop one directory's cached listing.
|
||||
|
||||
Both spellings of the directory go, because a backend may have
|
||||
keyed it with or without its trailing slash and an eviction that
|
||||
hits no key is silent.
|
||||
|
||||
Args:
|
||||
virtual (str): Mount-absolute path of the directory.
|
||||
"""
|
||||
await self._index.invalidate_dir(virtual)
|
||||
await self._index.invalidate_dir(virtual + "/")
|
||||
|
||||
def _virtual(self, path: PathSpec) -> str:
|
||||
mount_path = path.mount_path
|
||||
if not mount_path.startswith("/"):
|
||||
@@ -94,8 +108,7 @@ class CacheManager:
|
||||
virtual = self._virtual(path)
|
||||
if self._caches_reads and self._file_cache is not None:
|
||||
await self._file_cache.remove(virtual)
|
||||
await self._index.invalidate_dir(virtual)
|
||||
await self._index.invalidate_dir(virtual + "/")
|
||||
await self._evict_dir(virtual)
|
||||
await self._invalidate_parent(virtual)
|
||||
|
||||
async def invalidate_ancestors(self, path: PathSpec) -> None:
|
||||
@@ -114,8 +127,7 @@ class CacheManager:
|
||||
parent = self._virtual(path).rsplit("/", 1)[0]
|
||||
while parent and parent != self._prefix:
|
||||
parent = parent.rsplit("/", 1)[0]
|
||||
await self._index.invalidate_dir(parent or "/")
|
||||
await self._index.invalidate_dir(parent + "/")
|
||||
await self._evict_dir(parent or "/")
|
||||
|
||||
async def drop_prefix(self) -> None:
|
||||
"""Drop every cached body under this mount, path unspecified.
|
||||
@@ -135,6 +147,4 @@ class CacheManager:
|
||||
await self._file_cache.evict_prefix(self._prefix + "/")
|
||||
|
||||
async def _invalidate_parent(self, virtual: str) -> None:
|
||||
parent = virtual.rsplit("/", 1)[0] or "/"
|
||||
await self._index.invalidate_dir(parent)
|
||||
await self._index.invalidate_dir(parent + "/")
|
||||
await self._evict_dir(virtual.rsplit("/", 1)[0] or "/")
|
||||
|
||||
@@ -27,24 +27,33 @@ from mirage.provision.types import ProvisionResult
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
async def _subtree(index: IndexCacheStore,
|
||||
path: PathSpec) -> list[tuple[str, int]]:
|
||||
key = "/" + path.resource_path if path.resource_path else "/"
|
||||
prefix = key.rstrip("/") + "/"
|
||||
found = [(ep, entry.size) for ep, entry in (await index.entries()).items()
|
||||
if (ep == key or ep.startswith(prefix)) and entry.size is not None
|
||||
]
|
||||
def _subtree(accessor: GitHubAccessor,
|
||||
path: PathSpec) -> list[tuple[str, int]]:
|
||||
"""Every sized entry at or under ``path``, in mount-relative space.
|
||||
|
||||
Read off the git tree rather than the index, mirroring TypeScript's
|
||||
du: the tree is keyed repo-relative, which is the space these
|
||||
comparisons are in.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): backend handle holding the tree.
|
||||
path (PathSpec): subtree root.
|
||||
"""
|
||||
key = path.resource_path.strip("/")
|
||||
prefix = key + "/" if key else ""
|
||||
found = [("/" + p, entry.size) for p, entry in accessor.tree.items()
|
||||
if (p == key or p.startswith(prefix)) and entry.size is not None]
|
||||
found.sort()
|
||||
return found
|
||||
|
||||
|
||||
async def _du_size(index: IndexCacheStore, path: PathSpec) -> int:
|
||||
return sum(size for _, size in await _subtree(index, path))
|
||||
async def _du_size(accessor: GitHubAccessor, path: PathSpec) -> int:
|
||||
return sum(size for _, size in _subtree(accessor, path))
|
||||
|
||||
|
||||
async def _du_entries(index: IndexCacheStore,
|
||||
async def _du_entries(accessor: GitHubAccessor,
|
||||
path: PathSpec) -> tuple[list[tuple[str, int]], int]:
|
||||
found = await _subtree(index, path)
|
||||
found = _subtree(accessor, path)
|
||||
return found, sum(size for _, size in found)
|
||||
|
||||
|
||||
@@ -70,5 +79,5 @@ async def du(accessor: GitHubAccessor, paths: list[PathSpec], texts: list[str],
|
||||
return await du_generic(paths, list(texts), opts,
|
||||
partial(_resolve, accessor, opts.index),
|
||||
partial(_stat, accessor, opts.index),
|
||||
partial(_du_size, opts.index),
|
||||
partial(_du_entries, opts.index))
|
||||
partial(_du_size, accessor),
|
||||
partial(_du_entries, accessor))
|
||||
|
||||
@@ -51,9 +51,7 @@ async def find(
|
||||
return await find_generic(paths,
|
||||
texts,
|
||||
opts,
|
||||
find_core=partial(find_core,
|
||||
accessor,
|
||||
index=opts.index),
|
||||
find_core=partial(find_core, accessor),
|
||||
stat=partial(stat_core,
|
||||
accessor,
|
||||
index=opts.index))
|
||||
|
||||
@@ -67,7 +67,7 @@ async def narrow_scope(
|
||||
search actually narrowed the set.
|
||||
"""
|
||||
key = scope_relative_key(paths[0])
|
||||
file_count = count_scope_files(await index.entries(), key)
|
||||
file_count = count_scope_files(accessor.tree, key)
|
||||
query = search_query(pattern,
|
||||
fixed_string) if pattern is not None else None
|
||||
literal = (pattern is not None
|
||||
|
||||
@@ -75,6 +75,7 @@ async def readdir(
|
||||
continue
|
||||
is_dir = it.get("type") == "folder"
|
||||
filename = it["name"]
|
||||
sha1 = it.get("sha1")
|
||||
entry = IndexEntry(
|
||||
id=it["id"],
|
||||
name=filename,
|
||||
@@ -82,6 +83,7 @@ async def readdir(
|
||||
remote_time=it.get("modified_at") or "",
|
||||
vfs_name=filename,
|
||||
size=None if is_dir else it.get("size"),
|
||||
extra={"sha1": sha1} if sha1 else {},
|
||||
)
|
||||
entries.append((filename, entry, is_dir))
|
||||
|
||||
|
||||
@@ -41,15 +41,23 @@ def _stat_from_item(item: dict[str, Any]) -> FileStat:
|
||||
extra={"box_id": item["id"]},
|
||||
)
|
||||
remote_time = item.get("modified_at") or ""
|
||||
# Box returns the content sha1 in the same listing, so prefer it:
|
||||
# it is content-addressed, where modified_at cannot tell two writes
|
||||
# in the same second apart and does not move at all on a re-upload
|
||||
# of identical bytes.
|
||||
sha1 = item.get("sha1") or None
|
||||
return FileStat(
|
||||
name=vfs_name,
|
||||
size=item.get("size"),
|
||||
type=guess_type(vfs_name),
|
||||
modified=remote_time,
|
||||
fingerprint=remote_time or None,
|
||||
fingerprint=sha1 or remote_time or None,
|
||||
extra={
|
||||
"box_id": item["id"],
|
||||
"resource_type": rt,
|
||||
**({
|
||||
"sha1": sha1
|
||||
} if sha1 else {}),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -106,14 +114,18 @@ async def stat(
|
||||
modified=result.entry.remote_time,
|
||||
extra={"box_id": result.entry.id},
|
||||
)
|
||||
sha1 = result.entry.extra.get("sha1")
|
||||
return FileStat(
|
||||
name=result.entry.vfs_name or result.entry.name,
|
||||
size=result.entry.size,
|
||||
type=guess_type(result.entry.vfs_name),
|
||||
modified=result.entry.remote_time,
|
||||
fingerprint=result.entry.remote_time or None,
|
||||
fingerprint=sha1 or result.entry.remote_time or None,
|
||||
extra={
|
||||
"box_id": result.entry.id,
|
||||
"resource_type": result.entry.resource_type,
|
||||
**({
|
||||
"sha1": sha1
|
||||
} if sha1 else {}),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from functools import partial
|
||||
|
||||
from mirage.accessor.box import BoxAccessor
|
||||
from mirage.core.box.readdir import readdir
|
||||
from mirage.core.box.stat import stat
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.walk import ReaddirWalk
|
||||
|
||||
|
||||
def build_delta_hook(accessor: BoxAccessor) -> DeltaHook:
|
||||
"""Build the Box delta hook.
|
||||
|
||||
Box keys its tree by folder id and has no recursive listing, so the
|
||||
pull is one ``/folders/{id}/items`` request per directory. Box does
|
||||
offer an account-wide ``/events`` feed, which is the cheaper signal
|
||||
and belongs in a push receiver, not here.
|
||||
|
||||
Fingerprints on ``modified_at``, which is what Box stat reports.
|
||||
|
||||
Args:
|
||||
accessor (BoxAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(
|
||||
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
|
||||
@@ -0,0 +1,132 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.core.timeutil import epoch_to_iso
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
def _reraise(error: OSError) -> None:
|
||||
"""Fail the walk on a directory it could not read.
|
||||
|
||||
``os.walk`` swallows every listing error by default, which for a
|
||||
snapshot differ means an unreadable subtree is indistinguishable
|
||||
from an empty one: it diffs into a DELETE for every child, then a
|
||||
CREATE for each when access comes back. Absence is the one error
|
||||
that is genuinely a DELETE, and the caller drops it.
|
||||
|
||||
Args:
|
||||
error (OSError): The failure ``os.walk`` was about to discard.
|
||||
"""
|
||||
raise error
|
||||
|
||||
|
||||
def _walk_sync(root: Path,
|
||||
path: str) -> list[tuple[str, bool, str | None, int | None]]:
|
||||
"""Collect (mount-relative path, is_dir, mtime, size) under a path.
|
||||
|
||||
Runs in a worker thread; ``os.walk`` and ``stat`` are blocking.
|
||||
Symlinks are not followed, matching every other disk walk in the
|
||||
repo and keeping a link loop from hanging the poll.
|
||||
|
||||
Args:
|
||||
root (Path): Mount root on the local filesystem.
|
||||
path (str): Mount-relative directory to walk.
|
||||
"""
|
||||
start = _resolve(root, path)
|
||||
out: list[tuple[str, bool, str | None, int | None]] = []
|
||||
for dirpath, dirnames, filenames in os.walk(start, onerror=_reraise):
|
||||
current = Path(dirpath)
|
||||
for name in dirnames:
|
||||
relative = (current / name).relative_to(root).as_posix()
|
||||
out.append(("/" + relative, True, None, None))
|
||||
for name in filenames:
|
||||
full = current / name
|
||||
relative = full.relative_to(root).as_posix()
|
||||
try:
|
||||
info = full.lstat()
|
||||
except FileNotFoundError:
|
||||
# Same rule one entry down: a file that vanished between
|
||||
# the listing and the stat is a DELETE the next pull
|
||||
# reports, an unreadable one is not.
|
||||
continue
|
||||
out.append(("/" + relative, False, epoch_to_iso(info.st_mtime),
|
||||
info.st_size))
|
||||
return out
|
||||
|
||||
|
||||
class DiskWalk:
|
||||
"""Recursive ``os.walk`` feeding the generic listing differ.
|
||||
|
||||
Reads the filesystem directly, never through mirage's caches, as
|
||||
the DeltaHook contract requires. Fingerprints on mtime, the same
|
||||
value ``disk`` stat reports, so an editor that rewrites identical
|
||||
bytes still registers as an UPDATE. That is the local filesystem's
|
||||
own resolution, not a mirage choice: nothing cheaper than hashing
|
||||
every file can tell those two apart.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: DiskAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (DiskAccessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
try:
|
||||
found = await asyncio.to_thread(_walk_sync, self._accessor.root,
|
||||
root.mount_path)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
for relative, is_dir, modified, size in found:
|
||||
virtual = (prefix.rstrip("/") + relative if prefix else relative)
|
||||
if is_dir:
|
||||
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
continue
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=stat_fingerprint(None, modified, size),
|
||||
size=size,
|
||||
modified=modified)
|
||||
|
||||
|
||||
def build_delta_hook(accessor: DiskAccessor) -> DeltaHook:
|
||||
"""Build the disk delta hook.
|
||||
|
||||
Args:
|
||||
accessor (DiskAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(DiskWalk(accessor))
|
||||
@@ -0,0 +1,113 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from mirage.accessor.dropbox import DropboxAccessor
|
||||
from mirage.core.dropbox._client import DropboxApiError
|
||||
from mirage.core.dropbox.api import list_folder
|
||||
from mirage.core.dropbox.paths import dropbox_path_of
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
|
||||
|
||||
class DropboxWalk:
|
||||
"""One recursive ``list_folder`` feeding the generic listing differ.
|
||||
|
||||
Reads the account directly, never through mirage's caches, as the
|
||||
DeltaHook contract requires.
|
||||
|
||||
Fingerprints on ``content_hash``, Dropbox's own content digest, so
|
||||
an upload of identical bytes is correctly reported as no change;
|
||||
``rev`` is the fallback, and it moves on any write.
|
||||
|
||||
Dropbox also offers a cursor: the same endpoint returns one, and
|
||||
``list_folder/continue`` replays only what changed since. That is a
|
||||
faster pull, not a more correct one, and it cannot replace this
|
||||
walk, because the server may invalidate a cursor at any time and
|
||||
the only answer to that is a full listing. When the fast path is
|
||||
added it belongs behind ``pull``, with this walk as its reset path.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: DropboxAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (DropboxAccessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
accessor = self._accessor
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
api_root = dropbox_path_of(accessor, root)
|
||||
try:
|
||||
found = await list_folder(accessor.token_manager,
|
||||
api_root,
|
||||
recursive=True)
|
||||
except DropboxApiError as exc:
|
||||
# list_folder 409s on a missing path and on a file operand;
|
||||
# either way there is nothing under this root to report.
|
||||
if exc.status == 409:
|
||||
return
|
||||
raise
|
||||
# Dropbox paths are case-insensitive: `path_display` carries the
|
||||
# server's casing while `root_path` carries the user's, so a
|
||||
# configured `/team` whose displayed path is `/Team` matched
|
||||
# nothing and every event landed outside the watch scope. The
|
||||
# comparison folds case; the slice keeps the server's casing for
|
||||
# everything below the root, and is safe because `path_lower` is
|
||||
# `path_display` lowercased, same length.
|
||||
base = accessor.root_path
|
||||
folded = base.lower()
|
||||
for entry in found:
|
||||
display = entry.get("path_display") or entry.get("path_lower")
|
||||
if not display:
|
||||
continue
|
||||
relative = display[len(base):] if base and display.lower(
|
||||
).startswith(folded) else display
|
||||
relative = relative.strip("/")
|
||||
if not relative:
|
||||
continue
|
||||
virtual = (prefix.rstrip("/") + "/" + relative if prefix else "/" +
|
||||
relative)
|
||||
if entry.get(".tag") == "folder":
|
||||
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
continue
|
||||
modified = entry.get("server_modified") or entry.get(
|
||||
"client_modified") or None
|
||||
size = entry.get("size")
|
||||
size = size if isinstance(size, int) else None
|
||||
version = entry.get("content_hash") or entry.get("rev")
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=stat_fingerprint(
|
||||
version, modified, size),
|
||||
size=size,
|
||||
modified=modified)
|
||||
|
||||
|
||||
def build_delta_hook(accessor: DropboxAccessor) -> DeltaHook:
|
||||
"""Build the Dropbox delta hook.
|
||||
|
||||
Args:
|
||||
accessor (DropboxAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(DropboxWalk(accessor))
|
||||
@@ -0,0 +1,43 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from functools import partial
|
||||
|
||||
from mirage.accessor.gdrive import GDriveAccessor
|
||||
from mirage.core.gdrive.readdir import readdir
|
||||
from mirage.core.gdrive.stat import stat
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.walk import ReaddirWalk
|
||||
|
||||
|
||||
def build_delta_hook(accessor: GDriveAccessor) -> DeltaHook:
|
||||
"""Build the Google Drive delta hook.
|
||||
|
||||
Drive addresses files by id and returns ``parents`` rather than
|
||||
paths, so a whole-corpus ``files.list`` would still have to rebuild
|
||||
the tree before it could name anything; the walk descends per folder
|
||||
instead, which is the same shape ``find`` already uses here.
|
||||
|
||||
Fingerprints on ``modifiedTime``, which is what Drive stat reports.
|
||||
Drive also has ``changes.list`` with a page token, an account-wide
|
||||
feed that is cheaper than any walk and would have to be filtered
|
||||
back down to the watch root; that belongs behind ``pull`` as a fast
|
||||
path, with this walk as its reset.
|
||||
|
||||
Args:
|
||||
accessor (GDriveAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(
|
||||
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
|
||||
@@ -13,7 +13,6 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.cache.index import NULL_INDEX, IndexCacheStore
|
||||
from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
|
||||
emit_start_path, keep,
|
||||
start_basename, tree_has_empty)
|
||||
@@ -38,8 +37,6 @@ async def find(
|
||||
path_pattern: str | None = None,
|
||||
empty: bool = False,
|
||||
tree: PredNode | None = None,
|
||||
*,
|
||||
index: IndexCacheStore = NULL_INDEX,
|
||||
) -> list[str]:
|
||||
base = path.mount_path.strip("/")
|
||||
base_depth = 0 if base == "" else base.count("/") + 1
|
||||
@@ -54,30 +51,33 @@ async def find(
|
||||
empty=empty)
|
||||
start_kind = "d" if base == "" else None
|
||||
start_size = 0
|
||||
start_remote_time = ""
|
||||
has_child = False
|
||||
entries = await index.entries()
|
||||
# The git tree, not the index: it is keyed repo-relative, which is
|
||||
# the space this walk compares in, and it stays right however the
|
||||
# mount keys its index. Mirrors TypeScript's find.
|
||||
entries = accessor.tree
|
||||
non_empty_dirs: set[str] = set()
|
||||
if tree_has_empty(tree):
|
||||
# Every intermediate folder is itself an entry, so marking direct
|
||||
# parents is enough to classify all non-empty directories.
|
||||
# Tree keys carry no leading slash, so a top-level entry has no
|
||||
# parent segment at all; rsplit would hand back the entry itself
|
||||
# and mark every top-level directory non-empty.
|
||||
non_empty_dirs = {
|
||||
entry_path.rsplit("/", 1)[0] or "/"
|
||||
entry_path.rsplit("/", 1)[0] if "/" in entry_path else ""
|
||||
for entry_path in entries
|
||||
}
|
||||
for entry_path in sorted(entries):
|
||||
p = entry_path.lstrip("/")
|
||||
for p in sorted(entries):
|
||||
if p == base:
|
||||
meta = entries[entry_path]
|
||||
start_kind = "d" if meta.resource_type == "folder" else "f"
|
||||
meta = entries[p]
|
||||
start_kind = "d" if meta.type == "tree" else "f"
|
||||
start_size = meta.size or 0
|
||||
start_remote_time = meta.remote_time
|
||||
continue
|
||||
if base and not p.startswith(base + "/"):
|
||||
continue
|
||||
has_child = True
|
||||
entry_meta = entries[entry_path]
|
||||
is_dir = entry_meta.resource_type == "folder"
|
||||
entry_meta = entries[p]
|
||||
is_dir = entry_meta.type == "tree"
|
||||
full_path = "/" + p
|
||||
depth = p.count("/") + 1 - base_depth
|
||||
if maxdepth is not None and depth > maxdepth:
|
||||
@@ -86,8 +86,7 @@ async def find(
|
||||
size = 0 if is_dir else (entry_meta.size or 0)
|
||||
is_empty = None
|
||||
if tree_has_empty(tree):
|
||||
is_empty = (entry_path.rstrip("/") not in non_empty_dirs
|
||||
if is_dir else size == 0)
|
||||
is_empty = (p not in non_empty_dirs if is_dir else size == 0)
|
||||
entry = FindEntry(key=full_path,
|
||||
name=p.rsplit("/", 1)[-1],
|
||||
kind="d" if is_dir else "f",
|
||||
@@ -99,11 +98,14 @@ async def find(
|
||||
continue
|
||||
if max_size is not None and size > max_size:
|
||||
continue
|
||||
if not matches_mtime(entry_meta.remote_time, mtime_min, mtime_max):
|
||||
# A git tree carries no timestamp, so every entry's mtime is
|
||||
# unknown, which -mtime excludes. That is what the index answered
|
||||
# too: nothing here ever set IndexEntry.remote_time.
|
||||
if not matches_mtime("", mtime_min, mtime_max):
|
||||
continue
|
||||
results.append(full_path)
|
||||
if ((start_kind is not None or has_child)
|
||||
and matches_mtime(start_remote_time, mtime_min, mtime_max)):
|
||||
and matches_mtime("", mtime_min, mtime_max)):
|
||||
root_kind = start_kind or "d"
|
||||
emit_start_path(
|
||||
results,
|
||||
|
||||
@@ -18,9 +18,10 @@ from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.cache.index import NULL_INDEX, IndexCacheStore, LookupStatus
|
||||
from mirage.core.github._client import github_get
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.tree import refill_index
|
||||
from mirage.core.github.tree import ensure_live_index, refill_index
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.errors import enoent
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
|
||||
|
||||
async def read_bytes(config: GitHubConfig, owner: str, repo: str,
|
||||
@@ -42,9 +43,9 @@ async def read(
|
||||
index: IndexCacheStore = NULL_INDEX,
|
||||
) -> bytes:
|
||||
virtual = path_spec.virtual
|
||||
path = path_spec.mount_path
|
||||
|
||||
key = "/" + path.strip("/")
|
||||
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
|
||||
key = path_spec.mount_path.strip("/")
|
||||
key = prefix + "/" + key if key else prefix or "/"
|
||||
# Freshness is tracked per directory, never per entry, so a blob's row
|
||||
# is exactly as fresh as its parent's listing and `get` can never
|
||||
# report staleness of its own. The parent is therefore the probe:
|
||||
@@ -53,11 +54,12 @@ async def read(
|
||||
# miss is not a probe either -- against a live index it is a real
|
||||
# absence, and refetching the whole tree on every ENOENT costs a
|
||||
# recursive-tree call per miss.
|
||||
await ensure_live_index(accessor, index, prefix)
|
||||
if not accessor.truncated:
|
||||
cut = key.rfind("/")
|
||||
parent = key[:cut] if cut > 0 else "/"
|
||||
if (await index.list_dir(parent)).status == LookupStatus.EXPIRED:
|
||||
await refill_index(accessor, index)
|
||||
await refill_index(accessor, index, prefix)
|
||||
result = await index.get(key)
|
||||
if result.status == LookupStatus.NOT_FOUND or result.entry is None:
|
||||
raise enoent(virtual)
|
||||
|
||||
@@ -16,7 +16,8 @@ import logging
|
||||
|
||||
from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
|
||||
LookupStatus)
|
||||
from mirage.core.github.tree import fetch_dir_tree, refill_index
|
||||
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
|
||||
refill_index)
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.errors import enoent
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
@@ -29,47 +30,51 @@ async def readdir(accessor,
|
||||
index: IndexCacheStore = NULL_INDEX) -> list[str]:
|
||||
virtual = path_spec.virtual
|
||||
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
|
||||
path = path_spec.directory if path_spec.pattern else path_spec.virtual
|
||||
if prefix and path.startswith(prefix):
|
||||
rest = path[len(prefix):]
|
||||
if prefix.endswith("/") or rest == "" or rest.startswith("/"):
|
||||
path = rest or "/"
|
||||
path = path.rstrip("/") or "/"
|
||||
listing = await index.list_dir(path)
|
||||
path = (path_spec.dir if path_spec.pattern else path_spec).mount_path
|
||||
key = path.strip("/")
|
||||
virtual_key = prefix + "/" + key if key else prefix or "/"
|
||||
await ensure_live_index(accessor, index, prefix)
|
||||
listing = await index.list_dir(virtual_key)
|
||||
# The index is the whole listing here, not a cache in front of one, so
|
||||
# an *expired* answer means the tree aged out, not that the path is
|
||||
# gone. Refetch once and ask again. A NOT_FOUND against a live index
|
||||
# is a real absence and must not cost a tree fetch.
|
||||
if listing.status == LookupStatus.EXPIRED and not accessor.truncated:
|
||||
if await refill_index(accessor, index):
|
||||
listing = await index.list_dir(path)
|
||||
if await refill_index(accessor, index, prefix):
|
||||
listing = await index.list_dir(virtual_key)
|
||||
if listing.entries is not None:
|
||||
if prefix and listing.entries and not listing.entries[0].startswith(
|
||||
prefix):
|
||||
return [prefix + e for e in listing.entries]
|
||||
return listing.entries
|
||||
if listing.status == LookupStatus.NOT_FOUND:
|
||||
if accessor.truncated:
|
||||
return await _fallback_readdir(accessor, path, index, virtual,
|
||||
prefix)
|
||||
return await _fallback_readdir(accessor, virtual_key, index,
|
||||
virtual, prefix)
|
||||
raise enoent(virtual)
|
||||
return []
|
||||
|
||||
|
||||
async def _fallback_readdir(
|
||||
accessor,
|
||||
path: str,
|
||||
virtual_key: str,
|
||||
index: IndexCacheStore,
|
||||
virtual: str,
|
||||
prefix: str = "",
|
||||
prefix: str,
|
||||
) -> list[str]:
|
||||
"""Per-directory tree fetch when recursive tree was truncated."""
|
||||
parent_sha = await _resolve_dir_sha(accessor, path, index)
|
||||
"""Per-directory tree fetch when recursive tree was truncated.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): backend handle.
|
||||
virtual_key (str): Mount-absolute directory key.
|
||||
index (IndexCacheStore): the mount's index.
|
||||
virtual (str): Virtual path, for the error message.
|
||||
prefix (str): the mount prefix, which the descent has to strip to
|
||||
walk the repository's own path segments.
|
||||
"""
|
||||
parent_sha = await _resolve_dir_sha(accessor, virtual_key, index, prefix)
|
||||
if parent_sha is None:
|
||||
raise enoent(virtual)
|
||||
entries = await fetch_dir_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, parent_sha)
|
||||
norm = "/" + path.strip("/")
|
||||
norm = virtual_key.rstrip("/")
|
||||
child_keys: list[str] = []
|
||||
dir_entries: list[tuple[str, IndexEntry]] = []
|
||||
for entry in entries:
|
||||
@@ -85,24 +90,33 @@ async def _fallback_readdir(
|
||||
child_keys.append(child_path)
|
||||
await index.set_dir(norm, dir_entries)
|
||||
log.debug("fallback readdir populated %d entries for %s", len(entries),
|
||||
path)
|
||||
virtual_keys = sorted((prefix + k if prefix else k) for k in child_keys)
|
||||
return virtual_keys
|
||||
virtual_key)
|
||||
return sorted(child_keys)
|
||||
|
||||
|
||||
async def _resolve_dir_sha(accessor, path: str,
|
||||
index: IndexCacheStore) -> str | None:
|
||||
async def _resolve_dir_sha(accessor,
|
||||
virtual_key: str,
|
||||
index: IndexCacheStore,
|
||||
prefix: str = "") -> str | None:
|
||||
"""Get the tree SHA for a directory path.
|
||||
|
||||
Walks from root if needed, fetching per-directory trees.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): backend handle.
|
||||
virtual_key (str): Mount-absolute directory key.
|
||||
index (IndexCacheStore): the mount's index.
|
||||
prefix (str): the mount prefix the keys are built against.
|
||||
"""
|
||||
norm = "/" + path.strip("/")
|
||||
norm = virtual_key.rstrip("/") or "/"
|
||||
result = await index.get(norm)
|
||||
if result.entry is not None:
|
||||
return result.entry.id
|
||||
parts = norm.strip("/").split("/")
|
||||
stem = prefix.rstrip("/")
|
||||
rest = norm[len(stem):] if stem and norm.startswith(stem) else norm
|
||||
parts = [p for p in rest.strip("/").split("/") if p]
|
||||
current_sha = accessor.ref
|
||||
current_path = ""
|
||||
current_path = stem
|
||||
for part in parts:
|
||||
entries = await fetch_dir_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, current_sha)
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.cache.index import IndexEntry
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
|
||||
@@ -45,24 +45,26 @@ def is_repo_root(key: str) -> bool:
|
||||
return key in ("", "/")
|
||||
|
||||
|
||||
def count_scope_files(entries: dict[str, IndexEntry], key: str) -> int:
|
||||
"""Count indexed files under a repo-relative scope key.
|
||||
def count_scope_files(tree: dict[str, TreeEntry], key: str) -> int:
|
||||
"""Count files under a repo-relative scope key.
|
||||
|
||||
Counted off the git tree rather than the index, mirroring
|
||||
TypeScript's: the tree keys are repo-relative with no leading slash,
|
||||
which is the space ``key`` is already in.
|
||||
|
||||
Args:
|
||||
entries (dict[str, IndexEntry]): Index entries keyed by repo-relative
|
||||
path with a leading slash.
|
||||
tree (dict[str, TreeEntry]): The recursive git tree.
|
||||
key (str): Repo-relative scope key from :func:`scope_relative_key`.
|
||||
|
||||
Returns:
|
||||
int: Number of file entries at or below the scope.
|
||||
"""
|
||||
if is_repo_root(key):
|
||||
return sum(1 for e in entries.values() if e.resource_type == "file")
|
||||
norm = "/" + key.strip("/")
|
||||
return sum(1 for e in tree.values() if e.type == "blob")
|
||||
norm = key.strip("/")
|
||||
prefix = norm + "/"
|
||||
return sum(
|
||||
1 for p, e in entries.items()
|
||||
if e.resource_type == "file" and (p == norm or p.startswith(prefix)))
|
||||
return sum(1 for p, e in tree.items()
|
||||
if e.type == "blob" and (p == norm or p.startswith(prefix)))
|
||||
|
||||
|
||||
def should_use_search(
|
||||
|
||||
@@ -29,19 +29,13 @@ async def stat(accessor,
|
||||
index: IndexCacheStore = NULL_INDEX) -> FileStat:
|
||||
virtual = path_spec.virtual
|
||||
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
|
||||
path = path_spec.virtual
|
||||
|
||||
if prefix and path.startswith(prefix):
|
||||
rest = path[len(prefix):]
|
||||
if prefix.endswith("/") or rest == "" or rest.startswith("/"):
|
||||
path = rest or "/"
|
||||
if path == "/" or path == "":
|
||||
rel = path_spec.mount_path.strip("/")
|
||||
if not rel:
|
||||
return FileStat(name="/", type=FileType.DIRECTORY)
|
||||
key = "/" + path.strip("/") if path.strip("/") else "/"
|
||||
key = prefix + "/" + rel if prefix else "/" + rel
|
||||
result = await index.get(key)
|
||||
if result.entry is None:
|
||||
parent_idx = key.rsplit("/", 1)[0] or "/"
|
||||
parent_path = (prefix + parent_idx) if prefix else parent_idx
|
||||
parent_path = key.rsplit("/", 1)[0] or "/"
|
||||
try:
|
||||
await _readdir(
|
||||
accessor,
|
||||
|
||||
@@ -17,7 +17,8 @@ from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from mirage.cache.index import NULL_INDEX, IndexCacheStore, IndexEntry
|
||||
from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
|
||||
LookupStatus)
|
||||
from mirage.core.github._client import github_get
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
@@ -101,28 +102,40 @@ async def fetch_dir_tree(
|
||||
|
||||
|
||||
def index_rows(
|
||||
tree: dict[str, TreeEntry]
|
||||
) -> tuple[dict[str, IndexEntry], dict[str, list[str]]]:
|
||||
tree: dict[str, TreeEntry],
|
||||
prefix: str) -> tuple[dict[str, IndexEntry], dict[str, list[str]]]:
|
||||
"""Turn a git tree into the index's entry and children tables.
|
||||
|
||||
Shared so the mount's initial seed and a later refill build the same
|
||||
rows; TypeScript keeps its twin in this module too (`populateIndex`).
|
||||
Keyed by mount-absolute path, the way every other backend keys its
|
||||
index, so the shared cache machinery can spell an eviction without
|
||||
knowing which backend it is talking to. The tree itself stays
|
||||
repo-relative; ``prefix`` is what lifts it.
|
||||
|
||||
Shared so the mount's seed and a later refill build the same rows;
|
||||
TypeScript keeps its twin in this module too (`populateIndex`).
|
||||
|
||||
Args:
|
||||
tree (dict[str, TreeEntry]): the recursive tree, keyed by
|
||||
repo-relative path.
|
||||
prefix (str): the mount prefix ("/gh"), or "" for a root mount.
|
||||
|
||||
Returns:
|
||||
tuple[dict[str, IndexEntry], dict[str, list[str]]]: entries keyed
|
||||
by absolute path, and each directory's sorted children.
|
||||
by mount-absolute path, and each directory's sorted children.
|
||||
"""
|
||||
stem = prefix.rstrip("/")
|
||||
dirs: dict[str, list[tuple[str, IndexEntry]]] = defaultdict(list)
|
||||
# The repository root always exists, so it gets a row even when the
|
||||
# tree is empty. Without it an empty repository is byte for byte a
|
||||
# dropped index, and `ensure_live_index` would refetch on every read
|
||||
# of one; `ls` on it also read as ENOENT rather than as empty.
|
||||
dirs[stem or "/"] = []
|
||||
for path, entry in tree.items():
|
||||
parts = path.rsplit("/", 1)
|
||||
if len(parts) == 2:
|
||||
parent, name = "/" + parts[0], parts[1]
|
||||
parent, name = stem + "/" + parts[0], parts[1]
|
||||
else:
|
||||
parent, name = "/", parts[0]
|
||||
parent, name = stem or "/", parts[0]
|
||||
dirs[parent].append(
|
||||
(name,
|
||||
IndexEntry(
|
||||
@@ -132,20 +145,31 @@ def index_rows(
|
||||
size=entry.size,
|
||||
)))
|
||||
entries = {
|
||||
("/" + parent.strip("/") + "/" + name).replace("//", "/"): entry
|
||||
(parent.rstrip("/") + "/" + name): entry
|
||||
for parent, rows in dirs.items()
|
||||
for name, entry in rows
|
||||
}
|
||||
children = {
|
||||
parent:
|
||||
sorted(("/" + parent.strip("/") + "/" + name).replace("//", "/")
|
||||
for name, _ in rows)
|
||||
parent: sorted(parent.rstrip("/") + "/" + name for name, _ in rows)
|
||||
for parent, rows in dirs.items()
|
||||
}
|
||||
return entries, children
|
||||
|
||||
|
||||
async def refill_index(accessor, index: IndexCacheStore) -> bool:
|
||||
def seed_index(accessor, index: IndexCacheStore, prefix: str) -> None:
|
||||
"""Write the accessor's tree into ``index`` under ``prefix``.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): the mount's accessor, holding the tree.
|
||||
index (IndexCacheStore): the index to seed.
|
||||
prefix (str): the mount prefix the keys are built against.
|
||||
"""
|
||||
entries, children = index_rows(accessor.tree, prefix)
|
||||
index.seed(entries, children,
|
||||
datetime.now(timezone.utc) + timedelta(days=365))
|
||||
|
||||
|
||||
async def refill_index(accessor, index: IndexCacheStore, prefix: str) -> bool:
|
||||
"""Refetch the recursive tree and re-seed the index from it.
|
||||
|
||||
The mount fetches the whole tree once and seeds the index with it, so
|
||||
@@ -160,6 +184,7 @@ async def refill_index(accessor, index: IndexCacheStore) -> bool:
|
||||
accessor (GitHubAccessor): the mount's accessor, holding the
|
||||
config and the ref to refetch.
|
||||
index (IndexCacheStore): the index to re-seed.
|
||||
prefix (str): the mount prefix the index keys are built against.
|
||||
|
||||
Returns:
|
||||
bool: whether a refill happened; False when there is no index to
|
||||
@@ -170,7 +195,55 @@ async def refill_index(accessor, index: IndexCacheStore) -> bool:
|
||||
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, accessor.ref)
|
||||
accessor.truncated = truncated
|
||||
entries, children = index_rows(tree)
|
||||
index.seed(entries, children,
|
||||
datetime.now(timezone.utc) + timedelta(days=365))
|
||||
accessor.tree = tree
|
||||
seed_index(accessor, index, prefix)
|
||||
return True
|
||||
|
||||
|
||||
async def ensure_live_index(accessor, index: IndexCacheStore,
|
||||
prefix: str) -> bool:
|
||||
"""Refetch when the index holds no listing at all.
|
||||
|
||||
Every reader here treats a missing listing as a real absence, which
|
||||
is right against a *live* index and wrong against one that was never
|
||||
filled or has been dropped, and invalidation drops rather than
|
||||
expires: `invalidate_dir` removes the directory's row outright, so
|
||||
the EXPIRED probe each reader already runs never fires. An external
|
||||
change (a watch event is the only thing that invalidates a mount
|
||||
with no write ops) therefore left the whole mount answering ENOENT
|
||||
permanently, since the seeded expiry is a year out.
|
||||
|
||||
The root listing is what tells live from not, in one lookup and no
|
||||
request: the tree is written whole, so while the index is live every
|
||||
directory has a row and the mount root always does. One refill makes
|
||||
it live again, so this cannot cost a fetch per miss, which is what
|
||||
kept the readers from probing on absence in the first place.
|
||||
|
||||
Not live always **refetches**, and never re-seeds the tree the mount
|
||||
was built with. That tree is only true at build time: the first read
|
||||
of a mount can come long after it, and reusing it then served an
|
||||
index built from a repository five external writes ago. It is still
|
||||
what ``accessor.tree`` starts as, so find and du have something to
|
||||
read before any listing happens, and every refill reseats it.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): the mount's accessor.
|
||||
index (IndexCacheStore): the index to check and fill.
|
||||
prefix (str): the mount prefix the index keys are built against.
|
||||
|
||||
Returns:
|
||||
bool: whether the index was filled.
|
||||
"""
|
||||
if index is NULL_INDEX:
|
||||
return False
|
||||
# The liveness probe comes before anything on the accessor, so a live
|
||||
# index still answers every read without one.
|
||||
if (await index.list_dir(prefix.rstrip("/") or "/")).status \
|
||||
!= LookupStatus.NOT_FOUND:
|
||||
return False
|
||||
# A truncated tree is not the whole listing, so the invariant this
|
||||
# rests on does not hold and readdir's per-directory fallback owns
|
||||
# the miss instead.
|
||||
if accessor.truncated:
|
||||
return False
|
||||
return await refill_index(accessor, index, prefix)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.core.github.tree import fetch_tree
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.errors import IncompleteWalkError
|
||||
|
||||
|
||||
class GitHubWalk:
|
||||
"""One recursive git tree fetch feeding the generic listing differ.
|
||||
|
||||
``GET /git/trees/{ref}?recursive=1`` returns every path in the
|
||||
repository with its object sha, so a pull is one request whatever
|
||||
the repository's shape, and the fingerprint is the sha itself.
|
||||
That is the strongest fingerprint any mirage backend has: git is
|
||||
content-addressed, so identical bytes have an identical sha and a
|
||||
rewrite that changes nothing correctly reports nothing.
|
||||
|
||||
A mount is pinned to one ref, so what this detects is that ref
|
||||
moving. Nothing is reported while the branch sits still, however
|
||||
much is pushed elsewhere in the repository.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: GitHubAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (GitHubAccessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
|
||||
Raises:
|
||||
IncompleteWalkError: The repository is large enough that
|
||||
GitHub truncated the recursive tree, so the listing is
|
||||
not a complete picture of the ref and diffing it would
|
||||
report every unlisted path as deleted.
|
||||
"""
|
||||
accessor = self._accessor
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, accessor.ref)
|
||||
if truncated:
|
||||
raise IncompleteWalkError(
|
||||
f"github tree for {accessor.owner}/{accessor.repo}"
|
||||
f"@{accessor.ref} was truncated; cannot diff a partial tree")
|
||||
# A complete tree for the ref is exactly what the accessor holds,
|
||||
# and find/du/grep's scope counter read it directly. Discarding it
|
||||
# here left them answering from the tree the mount was built with
|
||||
# until an unrelated read happened to refill the index, so a pull
|
||||
# that reported a CREATE was followed by a find that could not see
|
||||
# the file.
|
||||
accessor.tree = tree
|
||||
stem = root.mount_path.strip("/")
|
||||
base = (stem + "/") if stem else ""
|
||||
for entry in tree.values():
|
||||
if base and not entry.path.startswith(base):
|
||||
continue
|
||||
virtual = (prefix.rstrip("/") + "/" +
|
||||
entry.path if prefix else "/" + entry.path)
|
||||
if entry.type == "tree":
|
||||
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
continue
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=entry.sha,
|
||||
size=entry.size)
|
||||
|
||||
|
||||
def build_delta_hook(accessor: GitHubAccessor) -> DeltaHook:
|
||||
"""Build the GitHub delta hook.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(GitHubWalk(accessor))
|
||||
@@ -0,0 +1,89 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from mirage.accessor.gridfs import GridFSAccessor
|
||||
from mirage.core.gridfs._client import (_prefix, _strip_prefix, iter_latest,
|
||||
prefix_query)
|
||||
from mirage.core.timeutil import to_iso_z
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.walk import synth_dirs
|
||||
|
||||
|
||||
class GridFSWalk:
|
||||
"""One flat ``fs.files`` aggregation feeding the generic differ.
|
||||
|
||||
GridFS stores a flat filename space, so the whole subtree comes back
|
||||
from a single prefix query rather than one round trip per directory,
|
||||
and the aggregation already reduces each filename to its newest
|
||||
revision. Reads the collection directly, never through mirage's
|
||||
caches, as the DeltaHook contract requires.
|
||||
|
||||
Fingerprints on the revision's ObjectId, which is exactly what
|
||||
``gridfs`` stat reports. That is an exact version: every upload
|
||||
mints a new document, so a rewrite always moves it and an untouched
|
||||
file never does.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: GridFSAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (GridFSAccessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
config = self._accessor.config
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
pfx = _prefix(root.mount_path, config)
|
||||
files: list[str] = []
|
||||
markers: list[str] = []
|
||||
async for doc in iter_latest(self._accessor, prefix_query(pfx)):
|
||||
filename = doc["filename"]
|
||||
relative = _strip_prefix(filename, config)
|
||||
virtual = (prefix.rstrip("/") + "/" +
|
||||
relative.lstrip("/") if prefix else "/" +
|
||||
relative.lstrip("/"))
|
||||
if filename.endswith("/"):
|
||||
# A directory marker, the same convention readdir reads
|
||||
# as an immediate child directory.
|
||||
markers.append(virtual.rstrip("/"))
|
||||
continue
|
||||
files.append(virtual)
|
||||
upload = doc.get("uploadDate")
|
||||
modified = to_iso_z(upload) if upload else None
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=str(doc["_id"]),
|
||||
size=doc["length"],
|
||||
modified=modified)
|
||||
for entry in synth_dirs(root.virtual, files, markers):
|
||||
yield entry
|
||||
|
||||
|
||||
def build_delta_hook(accessor: GridFSAccessor) -> DeltaHook:
|
||||
"""Build the GridFS delta hook.
|
||||
|
||||
Args:
|
||||
accessor (GridFSAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(GridFSWalk(accessor))
|
||||
@@ -0,0 +1,33 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.accessor._hf import _HfAccessor
|
||||
from mirage.core.opendal.watch import OpendalWalk
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
|
||||
|
||||
def build_delta_hook(accessor: _HfAccessor) -> DeltaHook:
|
||||
"""Build the delta hook shared by every Hugging Face resource.
|
||||
|
||||
One recursive tree listing per pull, fingerprinted on the Hub's
|
||||
ETag. A mount pinned to an immutable ``revision`` cannot report a
|
||||
change, because the revision it reads is frozen by definition; the
|
||||
hook is only meaningful against a moving ref such as ``main``.
|
||||
|
||||
Args:
|
||||
accessor (_HfAccessor): Backend handle for any of the four hf
|
||||
resources.
|
||||
"""
|
||||
return ListingDeltaHook(OpendalWalk(accessor))
|
||||
@@ -12,75 +12,20 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from opendal.exceptions import NotFound
|
||||
|
||||
from mirage.accessor.nextcloud import NextcloudAccessor
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.core.opendal.watch import OpendalWalk
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
|
||||
|
||||
class NextcloudWalk:
|
||||
"""Recursive WebDAV walk feeding the generic listing differ.
|
||||
|
||||
Reads through the opendal operator directly (a single recursive
|
||||
PROPFIND), never through mirage's caches, as the DeltaHook contract
|
||||
requires. Fingerprints use mirage's default (native ETag when the
|
||||
listing carries one, mtime|size otherwise).
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: NextcloudAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (NextcloudAccessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
base = root.resource_path.strip("/")
|
||||
list_path = base + "/" if base else "/"
|
||||
op = self._accessor.operator()
|
||||
try:
|
||||
entries = await op.list(list_path, recursive=True)
|
||||
except NotFound:
|
||||
return
|
||||
async for entry in entries:
|
||||
relative = entry.path
|
||||
if not relative or relative == list_path:
|
||||
continue
|
||||
is_dir = relative.endswith("/")
|
||||
resource_rel = relative.rstrip("/")
|
||||
virtual = (prefix.rstrip("/") + "/" +
|
||||
resource_rel if prefix else "/" + resource_rel)
|
||||
meta = entry.metadata
|
||||
if is_dir:
|
||||
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
continue
|
||||
modified = meta.last_modified.isoformat() \
|
||||
if meta and meta.last_modified else None
|
||||
size = meta.content_length if meta else None
|
||||
fingerprint = stat_fingerprint(meta.etag if meta else None,
|
||||
modified, size)
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=fingerprint,
|
||||
size=size,
|
||||
modified=modified)
|
||||
NextcloudWalk = OpendalWalk
|
||||
|
||||
|
||||
def build_delta_hook(accessor: NextcloudAccessor) -> DeltaHook:
|
||||
"""Build the Nextcloud delta hook.
|
||||
|
||||
One recursive PROPFIND per pull, fingerprinted on the WebDAV ETag.
|
||||
|
||||
Args:
|
||||
accessor (NextcloudAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(NextcloudWalk(accessor))
|
||||
return ListingDeltaHook(OpendalWalk(accessor))
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from functools import partial
|
||||
|
||||
from mirage.accessor.onedrive import OneDriveAccessor
|
||||
from mirage.core.onedrive.readdir import readdir
|
||||
from mirage.core.onedrive.stat import stat
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.walk import ReaddirWalk
|
||||
|
||||
|
||||
def build_delta_hook(accessor: OneDriveAccessor) -> DeltaHook:
|
||||
"""Build the OneDrive delta hook.
|
||||
|
||||
Fingerprints on the item's ``cTag``, which Graph moves only when the
|
||||
content changes (``eTag`` also moves on a metadata edit), so a
|
||||
rename does not read as a content change.
|
||||
|
||||
Graph has a native ``/delta`` feed with a resumable token, which is
|
||||
cheaper than this walk and reports deletes directly. It is a fast
|
||||
path rather than a replacement: Graph can answer ``resyncRequired``
|
||||
at any time, and the only response to that is a full listing. When
|
||||
it is added it belongs behind ``pull``, with this walk as its reset.
|
||||
|
||||
Args:
|
||||
accessor (OneDriveAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(
|
||||
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
|
||||
@@ -0,0 +1,18 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.core.opendal.types import OperatorAccessor
|
||||
from mirage.core.opendal.watch import OpendalWalk, build_delta_hook
|
||||
|
||||
__all__ = ["OperatorAccessor", "OpendalWalk", "build_delta_hook"]
|
||||
@@ -0,0 +1,28 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
import opendal
|
||||
|
||||
|
||||
class OperatorAccessor(Protocol):
|
||||
"""An accessor that reaches its backend through an opendal operator.
|
||||
|
||||
``NextcloudAccessor`` and the four ``_HfAccessor`` subclasses satisfy
|
||||
this structurally, which is what lets one walk serve both.
|
||||
"""
|
||||
|
||||
def operator(self) -> opendal.AsyncOperator:
|
||||
...
|
||||
@@ -0,0 +1,116 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import opendal
|
||||
from opendal.exceptions import NotFound
|
||||
from opendal.types import Metadata
|
||||
|
||||
from mirage.core.opendal.types import OperatorAccessor
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
|
||||
|
||||
class OpendalWalk:
|
||||
"""Recursive opendal list feeding the generic listing differ.
|
||||
|
||||
Reads through the operator directly (one recursive LIST), never
|
||||
through mirage's caches, as the DeltaHook contract requires.
|
||||
Fingerprints use mirage's default: the native ETag when the listing
|
||||
carries one, ``mtime|size`` otherwise.
|
||||
|
||||
Some opendal services answer LIST without per-entry metadata (the hf
|
||||
lister does; WebDAV's PROPFIND does not), which would leave every
|
||||
file unfingerprinted and reduce detection to create/delete. When a
|
||||
listed file carries no metadata at all, one ``stat`` per affected
|
||||
file fills the gap, mirroring what the hf readdir already does for
|
||||
sizes. A backend whose lister is complete never pays for it.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: OperatorAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (OperatorAccessor): Backend handle exposing an
|
||||
opendal operator.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
base = root.resource_path.strip("/")
|
||||
list_path = base + "/" if base else "/"
|
||||
op = self._accessor.operator()
|
||||
try:
|
||||
entries = await op.list(list_path, recursive=True)
|
||||
except NotFound:
|
||||
return
|
||||
async for entry in entries:
|
||||
relative = entry.path
|
||||
if not relative or relative == list_path:
|
||||
continue
|
||||
is_dir = relative.endswith("/")
|
||||
resource_rel = relative.rstrip("/")
|
||||
virtual = (prefix.rstrip("/") + "/" +
|
||||
resource_rel if prefix else "/" + resource_rel)
|
||||
if is_dir:
|
||||
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
continue
|
||||
meta = entry.metadata
|
||||
if meta is None or (meta.etag is None
|
||||
and meta.last_modified is None
|
||||
and meta.content_length is None):
|
||||
meta = await self._stat(op, resource_rel)
|
||||
modified = meta.last_modified.isoformat() \
|
||||
if meta and meta.last_modified else None
|
||||
size = meta.content_length if meta else None
|
||||
fingerprint = stat_fingerprint(meta.etag if meta else None,
|
||||
modified, size)
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=fingerprint,
|
||||
size=size,
|
||||
modified=modified)
|
||||
|
||||
async def _stat(self, op: opendal.AsyncOperator,
|
||||
key: str) -> Metadata | None:
|
||||
"""Fetch one entry's metadata when the listing omitted it.
|
||||
|
||||
Args:
|
||||
op (opendal.AsyncOperator): Open operator.
|
||||
key (str): Operator-relative key of the entry.
|
||||
"""
|
||||
try:
|
||||
return await op.stat(key)
|
||||
except NotFound:
|
||||
# Deleted between the listing and the stat; the next pull
|
||||
# reports the DELETE from the snapshot diff.
|
||||
return None
|
||||
|
||||
|
||||
def build_delta_hook(accessor: OperatorAccessor) -> DeltaHook:
|
||||
"""Build a delta hook for any opendal-backed accessor.
|
||||
|
||||
Args:
|
||||
accessor (OperatorAccessor): Backend handle exposing an opendal
|
||||
operator.
|
||||
"""
|
||||
return ListingDeltaHook(OpendalWalk(accessor))
|
||||
@@ -0,0 +1,101 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from mirage.accessor.s3 import S3Accessor
|
||||
from mirage.core.s3._client import (_client_kwargs, _key, _strip_prefix,
|
||||
async_session)
|
||||
from mirage.core.timeutil import to_iso_z
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.walk import synth_dirs
|
||||
|
||||
|
||||
class S3Walk:
|
||||
"""Recursive ``list_objects_v2`` feeding the generic listing differ.
|
||||
|
||||
One paginated LIST with no Delimiter covers the whole subtree, so a
|
||||
pull costs one request per 1000 keys rather than one per directory.
|
||||
Reads the bucket directly, never through mirage's caches, as the
|
||||
DeltaHook contract requires.
|
||||
|
||||
Fingerprints on the object's ETag, which for a single-part upload is
|
||||
the MD5 of the content, so an overwrite with identical bytes is
|
||||
correctly reported as no change. Multipart ETags are a digest of the
|
||||
part digests, which still changes with the content.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: S3Accessor) -> None:
|
||||
"""Args:
|
||||
accessor (S3Accessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
config = self._accessor.config
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
stem = _key(root.mount_path, config).rstrip("/")
|
||||
base = (stem + "/") if stem else ""
|
||||
files: list[str] = []
|
||||
markers: list[str] = []
|
||||
session = async_session(config)
|
||||
async with session.client(**_client_kwargs(config)) as client:
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
async for page in paginator.paginate(Bucket=config.bucket,
|
||||
Prefix=stem):
|
||||
for obj in page.get("Contents") or []:
|
||||
okey = obj["Key"]
|
||||
if not (okey == stem or okey.startswith(base)):
|
||||
continue
|
||||
relative = _strip_prefix(okey, config)
|
||||
virtual = (prefix.rstrip("/") + "/" +
|
||||
relative.lstrip("/") if prefix else "/" +
|
||||
relative.lstrip("/"))
|
||||
if okey.endswith("/"):
|
||||
# A directory marker: mirage's own mkdir writes
|
||||
# one. It carries an ETag and a size, but it is
|
||||
# not a file, so synth_dirs reports it instead.
|
||||
markers.append(virtual.rstrip("/"))
|
||||
continue
|
||||
files.append(virtual)
|
||||
last_mod = obj.get("LastModified")
|
||||
modified = to_iso_z(last_mod) if last_mod else None
|
||||
size = obj.get("Size")
|
||||
etag = (obj.get("ETag") or "").strip('"') or None
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=stat_fingerprint(
|
||||
etag, modified, size),
|
||||
size=size,
|
||||
modified=modified)
|
||||
for entry in synth_dirs(root.virtual, files, markers):
|
||||
yield entry
|
||||
|
||||
|
||||
def build_delta_hook(accessor: S3Accessor) -> DeltaHook:
|
||||
"""Build the delta hook shared by S3 and every S3-compatible alias.
|
||||
|
||||
Args:
|
||||
accessor (S3Accessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(S3Walk(accessor))
|
||||
@@ -0,0 +1,36 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from functools import partial
|
||||
|
||||
from mirage.accessor.sharepoint import SharePointAccessor
|
||||
from mirage.core.sharepoint.readdir import readdir
|
||||
from mirage.core.sharepoint.stat import stat
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
from mirage.watch.walk import ReaddirWalk
|
||||
|
||||
|
||||
def build_delta_hook(accessor: SharePointAccessor) -> DeltaHook:
|
||||
"""Build the SharePoint delta hook.
|
||||
|
||||
Same Graph drive surface as OneDrive, so the same ``cTag``
|
||||
fingerprint and the same standing offer of a native ``/delta`` fast
|
||||
path; see ``mirage.core.onedrive.watch``.
|
||||
|
||||
Args:
|
||||
accessor (SharePointAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(
|
||||
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
|
||||
@@ -0,0 +1,111 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import asyncssh
|
||||
|
||||
from mirage.accessor.ssh import SSHAccessor
|
||||
from mirage.core.ssh._client import _abs
|
||||
from mirage.core.ssh.config import SSHConfig
|
||||
from mirage.core.timeutil import epoch_to_iso
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
from mirage.watch.base import DeltaHook
|
||||
from mirage.watch.delta import ListingDeltaHook
|
||||
|
||||
|
||||
async def _descend(
|
||||
sftp: asyncssh.SFTPClient, config: SSHConfig,
|
||||
path: str) -> AsyncIterator[tuple[str, bool, str | None, int | None]]:
|
||||
"""Yield (mount-relative path, is_dir, mtime, size) under a path.
|
||||
|
||||
One ``readdir`` per directory, which is one round trip per
|
||||
directory; SFTP has no recursive listing. Each entry already
|
||||
carries its attributes, so no extra stat is needed.
|
||||
|
||||
Args:
|
||||
sftp (asyncssh.SFTPClient): Open SFTP channel.
|
||||
config (SSHConfig): Backend config, for the remote root.
|
||||
path (str): Mount-relative directory to descend into.
|
||||
"""
|
||||
try:
|
||||
listing = await sftp.readdir(_abs(config, path))
|
||||
except asyncssh.SFTPNoSuchFile:
|
||||
return
|
||||
for entry in listing:
|
||||
filename = (entry.filename.decode("utf-8") if isinstance(
|
||||
entry.filename, bytes) else entry.filename)
|
||||
if filename in (".", ".."):
|
||||
continue
|
||||
child = f"{path.rstrip('/')}/{filename}"
|
||||
attrs = entry.attrs
|
||||
if attrs.type == asyncssh.FILEXFER_TYPE_DIRECTORY:
|
||||
yield child, True, None, None
|
||||
async for row in _descend(sftp, config, child):
|
||||
yield row
|
||||
continue
|
||||
modified = epoch_to_iso(
|
||||
attrs.mtime) if attrs.mtime is not None else None
|
||||
yield child, False, modified, attrs.size
|
||||
|
||||
|
||||
class SSHWalk:
|
||||
"""Recursive SFTP descent feeding the generic listing differ.
|
||||
|
||||
Reads the remote host directly, never through mirage's caches, as
|
||||
the DeltaHook contract requires. Fingerprints on mtime, the same
|
||||
value ``ssh`` stat reports.
|
||||
|
||||
Unlike the object stores, this costs one round trip per directory,
|
||||
because SFTP has no recursive listing to ask for. Poll cadence
|
||||
should account for the shape of the tree.
|
||||
"""
|
||||
|
||||
def __init__(self, accessor: SSHAccessor) -> None:
|
||||
"""Args:
|
||||
accessor (SSHAccessor): Backend handle.
|
||||
"""
|
||||
self._accessor = accessor
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
accessor = self._accessor
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
sftp = await accessor.sftp()
|
||||
async for relative, is_dir, modified, size in _descend(
|
||||
sftp, accessor.config, root.mount_path):
|
||||
virtual = (prefix.rstrip("/") + relative if prefix else relative)
|
||||
if is_dir:
|
||||
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
continue
|
||||
yield WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=stat_fingerprint(None, modified, size),
|
||||
size=size,
|
||||
modified=modified)
|
||||
|
||||
|
||||
def build_delta_hook(accessor: SSHAccessor) -> DeltaHook:
|
||||
"""Build the SSH delta hook.
|
||||
|
||||
Args:
|
||||
accessor (SSHAccessor): Backend handle.
|
||||
"""
|
||||
return ListingDeltaHook(SSHWalk(accessor))
|
||||
@@ -20,12 +20,14 @@ from mirage.commands.builtin.box import COMMANDS as BOX_COMMANDS
|
||||
from mirage.core.box._client import BoxTokenManager
|
||||
from mirage.core.box.config import BoxConfig
|
||||
from mirage.core.box.readdir import readdir
|
||||
from mirage.core.box.watch import build_delta_hook
|
||||
from mirage.ops.box import OPS as BOX_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.box.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir)
|
||||
|
||||
@@ -51,6 +53,9 @@ class BoxResource(BaseResource):
|
||||
for op in BOX_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -33,6 +33,7 @@ from mirage.core.disk.stat import stat as disk_stat
|
||||
from mirage.core.disk.stream import read_stream
|
||||
from mirage.core.disk.truncate import truncate
|
||||
from mirage.core.disk.unlink import unlink
|
||||
from mirage.core.disk.watch import build_delta_hook
|
||||
from mirage.core.disk.write import write_bytes
|
||||
from mirage.ops.disk import OPS as DISK_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -40,6 +41,7 @@ from mirage.resource.disk.prompt import PROMPT
|
||||
from mirage.types import CapacityResult, CapacityState, PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -91,6 +93,9 @@ class DiskResource(BaseResource):
|
||||
# same directory are one store, however they were spelled.
|
||||
return f"{self.name}:{self.root}"
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -24,12 +24,14 @@ from mirage.core.dropbox.rename import rename
|
||||
from mirage.core.dropbox.rmdir import rmdir
|
||||
from mirage.core.dropbox.stat import stat
|
||||
from mirage.core.dropbox.unlink import unlink
|
||||
from mirage.core.dropbox.watch import build_delta_hook
|
||||
from mirage.core.dropbox.write import write_bytes
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.dropbox.config import DropboxConfig
|
||||
from mirage.resource.dropbox.prompt import PROMPT
|
||||
from mirage.types import ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir)
|
||||
|
||||
@@ -72,6 +74,9 @@ class DropboxResource(BaseResource):
|
||||
for op in DROPBOX_VFS_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
return await _resolve_glob(self.accessor, paths, self._index)
|
||||
|
||||
|
||||
@@ -16,12 +16,14 @@ from typing import Any
|
||||
|
||||
from mirage.accessor.gdrive import GDriveAccessor
|
||||
from mirage.core.gdrive.readdir import readdir
|
||||
from mirage.core.gdrive.watch import build_delta_hook
|
||||
from mirage.core.google._client import TokenManager
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.gdrive.config import GoogleDriveConfig
|
||||
from mirage.resource.gdrive.prompt import PROMPT
|
||||
from mirage.types import ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir)
|
||||
|
||||
@@ -51,6 +53,9 @@ class GoogleDriveResource(BaseResource):
|
||||
for op in GDRIVE_VFS_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
return await _resolve_glob(self.accessor, paths, index=self._index)
|
||||
|
||||
|
||||
@@ -12,20 +12,20 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.cache.index import IndexConfig
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.readdir import readdir
|
||||
from mirage.core.github.repo import fetch_default_branch
|
||||
from mirage.core.github.tree import fetch_tree, index_rows
|
||||
from mirage.core.github.tree import fetch_tree
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.core.github.watch import build_delta_hook
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.github.prompt import PROMPT
|
||||
from mirage.types import ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir)
|
||||
|
||||
@@ -73,14 +73,14 @@ class GitHubResource(BaseResource):
|
||||
truncated (bool): whether GitHub truncated that tree, in
|
||||
which case readdir falls back to per-directory fetches.
|
||||
"""
|
||||
super().__init__()
|
||||
self.accessor = GitHubAccessor(config,
|
||||
owner,
|
||||
repo,
|
||||
ref,
|
||||
default_branch,
|
||||
tree=tree,
|
||||
truncated=truncated)
|
||||
self._populate_index(tree)
|
||||
super().__init__()
|
||||
from mirage.commands.builtin.github import COMMANDS as _github_cmds
|
||||
from mirage.ops.github import OPS as _github_vfs_ops
|
||||
|
||||
@@ -138,23 +138,8 @@ class GitHubResource(BaseResource):
|
||||
tree,
|
||||
truncated=truncated)
|
||||
|
||||
def _populate_index(self, tree: dict[str, TreeEntry]) -> None:
|
||||
entries, children = index_rows(tree)
|
||||
self._github_index_entries = entries
|
||||
self._github_index_children = children
|
||||
self._github_index_expiry = (datetime.now(timezone.utc) +
|
||||
timedelta(days=365))
|
||||
self._seed_github_index()
|
||||
|
||||
def _seed_github_index(self) -> None:
|
||||
self._index.seed(self._github_index_entries,
|
||||
self._github_index_children,
|
||||
self._github_index_expiry)
|
||||
|
||||
def set_index(self, config: IndexConfig | None = None) -> None:
|
||||
super().set_index(config)
|
||||
if hasattr(self, "_github_index_entries"):
|
||||
self._seed_github_index()
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
return await _resolve_glob(self.accessor, paths, self._index)
|
||||
|
||||
@@ -34,6 +34,7 @@ from mirage.core.gridfs.stat import stat as gridfs_stat
|
||||
from mirage.core.gridfs.stream import range_read, read_stream
|
||||
from mirage.core.gridfs.truncate import truncate
|
||||
from mirage.core.gridfs.unlink import unlink
|
||||
from mirage.core.gridfs.watch import build_delta_hook
|
||||
from mirage.core.gridfs.write import write_bytes
|
||||
from mirage.ops.gridfs import OPS as GRIDFS_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -41,6 +42,7 @@ from mirage.resource.gridfs.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -86,6 +88,9 @@ class GridFSResource(BaseResource):
|
||||
for op in GRIDFS_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
|
||||
from mirage.core.hf_buckets.stat import stat as hf_stat
|
||||
from mirage.core.hf_buckets.stream import range_read, read_stream
|
||||
from mirage.core.hf_buckets.unlink import unlink
|
||||
from mirage.core.hf_buckets.watch import build_delta_hook
|
||||
from mirage.core.hf_buckets.write import write_bytes
|
||||
from mirage.ops.hf_buckets import OPS as HF_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -36,6 +37,7 @@ from mirage.resource.hf_buckets.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -78,6 +80,9 @@ class HfBucketsResource(BaseResource):
|
||||
for op in HF_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
|
||||
from mirage.core.hf_buckets.stat import stat as hf_stat
|
||||
from mirage.core.hf_buckets.stream import range_read, read_stream
|
||||
from mirage.core.hf_buckets.unlink import unlink
|
||||
from mirage.core.hf_buckets.watch import build_delta_hook
|
||||
from mirage.core.hf_buckets.write import write_bytes
|
||||
from mirage.ops.hf_buckets import OPS as HF_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -36,6 +37,7 @@ from mirage.resource.hf_datasets.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -77,6 +79,9 @@ class HfDatasetsResource(BaseResource):
|
||||
for op in HF_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
|
||||
from mirage.core.hf_buckets.stat import stat as hf_stat
|
||||
from mirage.core.hf_buckets.stream import range_read, read_stream
|
||||
from mirage.core.hf_buckets.unlink import unlink
|
||||
from mirage.core.hf_buckets.watch import build_delta_hook
|
||||
from mirage.core.hf_buckets.write import write_bytes
|
||||
from mirage.ops.hf_buckets import OPS as HF_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -36,6 +37,7 @@ from mirage.resource.hf_models.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -77,6 +79,9 @@ class HfModelsResource(BaseResource):
|
||||
for op in HF_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
|
||||
from mirage.core.hf_buckets.stat import stat as hf_stat
|
||||
from mirage.core.hf_buckets.stream import range_read, read_stream
|
||||
from mirage.core.hf_buckets.unlink import unlink
|
||||
from mirage.core.hf_buckets.watch import build_delta_hook
|
||||
from mirage.core.hf_buckets.write import write_bytes
|
||||
from mirage.ops.hf_buckets import OPS as HF_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -36,6 +37,7 @@ from mirage.resource.hf_spaces.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -77,6 +79,9 @@ class HfSpacesResource(BaseResource):
|
||||
for op in HF_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -33,6 +33,7 @@ from mirage.core.onedrive.stat import stat as onedrive_stat
|
||||
from mirage.core.onedrive.stream import range_read, read_stream
|
||||
from mirage.core.onedrive.truncate import truncate
|
||||
from mirage.core.onedrive.unlink import unlink
|
||||
from mirage.core.onedrive.watch import build_delta_hook
|
||||
from mirage.core.onedrive.write import write_bytes
|
||||
from mirage.ops.onedrive import OPS as ONEDRIVE_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -40,6 +41,7 @@ from mirage.resource.onedrive.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir)
|
||||
|
||||
@@ -91,6 +93,9 @@ class OneDriveResource(BaseResource):
|
||||
for op in ONEDRIVE_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -34,6 +34,7 @@ from mirage.core.s3.stat import stat as s3_stat
|
||||
from mirage.core.s3.stream import range_read, read_stream
|
||||
from mirage.core.s3.truncate import truncate
|
||||
from mirage.core.s3.unlink import unlink
|
||||
from mirage.core.s3.watch import build_delta_hook
|
||||
from mirage.core.s3.write import write_bytes
|
||||
from mirage.ops.s3 import OPS as S3_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -41,6 +42,7 @@ from mirage.resource.s3.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -97,6 +99,9 @@ class S3Resource(BaseResource):
|
||||
base = f"{self.name}:{cfg.endpoint_url or 'aws'}:{cfg.bucket}"
|
||||
return f"{base}/{prefix}" if prefix else base
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -19,6 +19,7 @@ from mirage.core.sharepoint.stat import stat as sharepoint_stat
|
||||
from mirage.core.sharepoint.stream import range_read, read_stream
|
||||
from mirage.core.sharepoint.truncate import truncate
|
||||
from mirage.core.sharepoint.unlink import unlink
|
||||
from mirage.core.sharepoint.watch import build_delta_hook
|
||||
from mirage.core.sharepoint.write import write_bytes
|
||||
from mirage.ops.sharepoint import OPS as SHAREPOINT_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
@@ -26,6 +27,7 @@ from mirage.resource.sharepoint.prompt import PROMPT
|
||||
from mirage.types import PathSpec, ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir)
|
||||
|
||||
@@ -77,6 +79,9 @@ class SharePointResource(BaseResource):
|
||||
for op in SHAREPOINT_OPS:
|
||||
self.register_op(op)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
if prefix:
|
||||
paths = [
|
||||
|
||||
@@ -35,12 +35,14 @@ from mirage.core.ssh.stat import stat as ssh_stat
|
||||
from mirage.core.ssh.stream import range_read, read_stream
|
||||
from mirage.core.ssh.truncate import truncate
|
||||
from mirage.core.ssh.unlink import unlink
|
||||
from mirage.core.ssh.watch import build_delta_hook
|
||||
from mirage.core.ssh.write import write_bytes
|
||||
from mirage.ops.ssh import OPS as SSH_OPS
|
||||
from mirage.resource.base import BaseResource
|
||||
from mirage.resource.ssh.prompt import PROMPT
|
||||
from mirage.types import ResourceName
|
||||
from mirage.utils.glob_walk import make_resolve_glob
|
||||
from mirage.watch.base import DeltaHook
|
||||
|
||||
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
|
||||
|
||||
@@ -90,6 +92,9 @@ class SSHResource(BaseResource):
|
||||
for ro in SSH_OPS:
|
||||
self.register_op(ro)
|
||||
|
||||
def delta_hook(self) -> DeltaHook:
|
||||
return build_delta_hook(self.accessor)
|
||||
|
||||
async def resolve_glob(self, paths, prefix: str = ""):
|
||||
return await _resolve_glob(self.accessor, paths, self._index)
|
||||
|
||||
|
||||
@@ -22,6 +22,17 @@ class QueueOverflowError(Exception):
|
||||
"""
|
||||
|
||||
|
||||
class IncompleteWalkError(Exception):
|
||||
"""Raised when a delta walk could not see the whole watch root.
|
||||
|
||||
A snapshot diff reads every path the walk did not report as a
|
||||
DELETE, so a partial listing does not degrade into fewer events, it
|
||||
invents wrong ones. A hook that knows its listing was truncated
|
||||
raises this instead, leaving the caller's checkpoint untouched so
|
||||
the next pull can still succeed.
|
||||
"""
|
||||
|
||||
|
||||
class QueueClosed(Exception):
|
||||
"""Terminal signal from ``WatchQueue.pop`` after ``close``.
|
||||
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import (AsyncIterator, Awaitable, Callable, Iterable,
|
||||
Iterator)
|
||||
|
||||
from mirage.cache.index import IndexCacheStore
|
||||
from mirage.cache.index.ram import RAMIndexCacheStore
|
||||
from mirage.types import FileStat, FileType, PathSpec, WalkEntry
|
||||
from mirage.utils.fingerprint import stat_fingerprint
|
||||
from mirage.utils.key_prefix import mount_key, mount_prefix_of
|
||||
|
||||
ReaddirFn = Callable[[PathSpec, IndexCacheStore], Awaitable[list[str]]]
|
||||
StatFn = Callable[[PathSpec, IndexCacheStore], Awaitable[FileStat]]
|
||||
|
||||
|
||||
def _ancestors(stem: str, start: str, seen: set[str]) -> Iterator[WalkEntry]:
|
||||
"""Emit ``start`` and each ancestor up to but excluding ``stem``.
|
||||
|
||||
Args:
|
||||
stem (str): Watch root, the exclusive upper bound.
|
||||
start (str): Deepest directory to emit.
|
||||
seen (set[str]): Paths already emitted; mutated in place so a
|
||||
shared prefix is reported once across calls.
|
||||
"""
|
||||
parent = start
|
||||
while parent and parent != stem and parent not in seen:
|
||||
seen.add(parent)
|
||||
yield WalkEntry(virtual=parent, is_dir=True, fingerprint=None)
|
||||
parent = parent.rsplit("/", 1)[0]
|
||||
|
||||
|
||||
def synth_dirs(root: str, files: Iterable[str],
|
||||
dirs: Iterable[str]) -> Iterator[WalkEntry]:
|
||||
"""Directory rows a prefix store implies but does not store.
|
||||
|
||||
An object store has no directories: ``ls`` shows them because
|
||||
readdir synthesizes them from the common prefixes of the keys, and a
|
||||
walk feeding change detection has to synthesize the same ones, or a
|
||||
consumer would see a file appear inside a directory that never
|
||||
appeared.
|
||||
|
||||
``dirs`` carries prefixes the store does name explicitly (a
|
||||
zero-byte marker key, which mirage's own ``mkdir`` writes), so an
|
||||
empty directory is still reported. A prefix backed by both a marker
|
||||
and children is reported once.
|
||||
|
||||
``root`` itself is never emitted; ``find``'s start-point rule
|
||||
applies here too, the generic owns that row.
|
||||
|
||||
Args:
|
||||
root (str): Watch root's virtual path.
|
||||
files (Iterable[str]): Virtual paths of every file under the
|
||||
root; each contributes its ancestor chain.
|
||||
dirs (Iterable[str]): Virtual paths of explicitly stored
|
||||
directories; each contributes itself and its ancestors.
|
||||
"""
|
||||
stem = root.rstrip("/")
|
||||
seen: set[str] = set()
|
||||
for path in dirs:
|
||||
yield from _ancestors(stem, path.rstrip("/"), seen)
|
||||
for path in files:
|
||||
yield from _ancestors(stem, path.rsplit("/", 1)[0], seen)
|
||||
|
||||
|
||||
def entry_of(virtual: str, stat: FileStat) -> WalkEntry:
|
||||
"""One walk row built from a backend's own stat.
|
||||
|
||||
Args:
|
||||
virtual (str): Entry's virtual path.
|
||||
stat (FileStat): What the backend reported for it.
|
||||
"""
|
||||
if stat.type == FileType.DIRECTORY:
|
||||
return WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
|
||||
return WalkEntry(virtual=virtual,
|
||||
is_dir=False,
|
||||
fingerprint=stat_fingerprint(stat.fingerprint,
|
||||
stat.modified, stat.size),
|
||||
size=stat.size,
|
||||
modified=stat.modified)
|
||||
|
||||
|
||||
async def _stat_at(stat: StatFn, virtual: str, prefix: str,
|
||||
index: IndexCacheStore) -> FileStat | None:
|
||||
"""Stat one virtual path, or None when it has vanished.
|
||||
|
||||
Args:
|
||||
stat (StatFn): Backend stat.
|
||||
virtual (str): Absolute virtual path.
|
||||
prefix (str): Mount prefix.
|
||||
index (IndexCacheStore): Index the walk is populating.
|
||||
"""
|
||||
spec = PathSpec(virtual=virtual,
|
||||
directory=virtual,
|
||||
resolved=False,
|
||||
resource_path=mount_key(virtual, prefix))
|
||||
try:
|
||||
return await stat(spec, index)
|
||||
except FileNotFoundError:
|
||||
# Removed between the readdir and the stat; the next pull
|
||||
# reports the DELETE from the snapshot diff. Only absence is
|
||||
# swallowed, an API error still propagates.
|
||||
return None
|
||||
|
||||
|
||||
async def _descend(readdir: ReaddirFn, stat: StatFn, spec: PathSpec,
|
||||
index: IndexCacheStore,
|
||||
prefix: str) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under one directory, depth first.
|
||||
|
||||
Args:
|
||||
readdir (ReaddirFn): Backend readdir.
|
||||
stat (StatFn): Backend stat.
|
||||
spec (PathSpec): Directory to descend into.
|
||||
index (IndexCacheStore): Index the walk is populating.
|
||||
prefix (str): Mount prefix.
|
||||
"""
|
||||
try:
|
||||
children = await readdir(spec, index)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
for child in children:
|
||||
# Classification is stat's job, the same rule find's walk
|
||||
# follows: the one in-band proof is a trailing slash on a cold
|
||||
# listing, and the stat behind it is an index lookup against
|
||||
# the readdir that just populated it, not another request.
|
||||
trimmed = child.rstrip("/")
|
||||
if child.endswith("/"):
|
||||
yield WalkEntry(virtual=trimmed, is_dir=True, fingerprint=None)
|
||||
is_dir = True
|
||||
else:
|
||||
found = await _stat_at(stat, trimmed, prefix, index)
|
||||
if found is None:
|
||||
continue
|
||||
yield entry_of(trimmed, found)
|
||||
is_dir = found.type == FileType.DIRECTORY
|
||||
if is_dir:
|
||||
child_spec = PathSpec(virtual=trimmed,
|
||||
directory=trimmed,
|
||||
resolved=False,
|
||||
resource_path=mount_key(trimmed, prefix))
|
||||
async for row in _descend(readdir, stat, child_spec, index,
|
||||
prefix):
|
||||
yield row
|
||||
|
||||
|
||||
class ReaddirWalk:
|
||||
"""Recursive readdir descent for a backend with no recursive listing.
|
||||
|
||||
Box, Google Drive and Microsoft Graph key their trees by opaque id,
|
||||
so a child cannot be addressed without having listed its parent, and
|
||||
none of them offers a whole-subtree listing. This walks them the way
|
||||
``find`` does, one readdir per directory.
|
||||
|
||||
Each pull builds its **own** index and throws it away afterwards.
|
||||
That is what keeps the DeltaHook contract: the index is not mirage's
|
||||
read cache, so the walk cannot compare the cache to itself, and it
|
||||
starts empty every time, so nothing carries over between pulls. It
|
||||
still has to exist, because these backends resolve a path's id
|
||||
through the index their parent's readdir populated; handing them
|
||||
``NULL_INDEX`` makes every path below the root read as absent.
|
||||
"""
|
||||
|
||||
def __init__(self, readdir: ReaddirFn, stat: StatFn) -> None:
|
||||
"""Args:
|
||||
readdir (ReaddirFn): Backend readdir, already bound to its
|
||||
accessor.
|
||||
stat (StatFn): Backend stat, already bound to its accessor.
|
||||
"""
|
||||
self._readdir = readdir
|
||||
self._stat = stat
|
||||
|
||||
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
|
||||
"""Yield every entry under ``root``.
|
||||
|
||||
Args:
|
||||
root (PathSpec): Watch root (mount-virtual path).
|
||||
"""
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
index = RAMIndexCacheStore()
|
||||
async for entry in _descend(self._readdir, self._stat, root, index,
|
||||
prefix):
|
||||
yield entry
|
||||
@@ -14,12 +14,13 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.cache.index import IndexCacheStore, IndexEntry
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.commands.builtin.github.du import _du_size
|
||||
from mirage.commands.builtin.github.grep import grep
|
||||
from mirage.commands.builtin.github.narrow import narrow_scope
|
||||
from mirage.commands.builtin.github.rg import rg
|
||||
from mirage.commands.config import CommandOpts
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.io.stream import materialize
|
||||
from mirage.types import PathSpec
|
||||
from tests.fixtures.github_mock import MOCK_BLOBS
|
||||
@@ -27,15 +28,6 @@ from tests.fixtures.github_mock import MOCK_BLOBS
|
||||
_NGLOBALS = narrow_scope.__globals__
|
||||
|
||||
|
||||
class EntryOnlyIndex(IndexCacheStore):
|
||||
|
||||
async def entries(self) -> dict[str, IndexEntry]:
|
||||
return {
|
||||
"/src/main.py":
|
||||
IndexEntry(id="main", name="main.py", resource_type="file", size=7)
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def counting_read(monkeypatch):
|
||||
reads: list[str] = []
|
||||
@@ -63,8 +55,23 @@ def _subdir() -> PathSpec:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_du_uses_store_interface_not_ram_implementation():
|
||||
assert await _du_size(EntryOnlyIndex(), _subdir()) == 7
|
||||
async def test_du_sizes_from_the_git_tree():
|
||||
# du reads the tree, not the index: the tree is keyed repo-relative,
|
||||
# which is the space this comparison is in, and it stays right
|
||||
# however the mount keys its index.
|
||||
accessor = GitHubAccessor(None,
|
||||
"acme",
|
||||
"proj",
|
||||
"main",
|
||||
"main",
|
||||
tree={
|
||||
"src/main.py":
|
||||
TreeEntry(path="src/main.py",
|
||||
type="blob",
|
||||
sha="main",
|
||||
size=7)
|
||||
})
|
||||
assert await _du_size(accessor, _subdir()) == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -47,12 +47,23 @@ async def test_read_missing_path(github_env):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_empty_index(github_env):
|
||||
async def test_read_empty_index_refills(github_env):
|
||||
# An empty index is no knowledge, not knowledge of absence: the index
|
||||
# here is the whole listing, and invalidation drops rows rather than
|
||||
# expiring them, so reading a miss as ENOENT made an invalidated mount
|
||||
# answer ENOENT forever.
|
||||
accessor, _ = github_env
|
||||
data = await github_read(accessor, PathSpec.from_str_path("/README.md"),
|
||||
RAMIndexCacheStore())
|
||||
assert b"Mock Repo" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_empty_index_still_enoent_off_tree(github_env):
|
||||
accessor, _ = github_env
|
||||
empty_index = RAMIndexCacheStore()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await github_read(accessor, PathSpec.from_str_path("/README.md"),
|
||||
empty_index)
|
||||
await github_read(accessor, PathSpec.from_str_path("/nonexistent.txt"),
|
||||
RAMIndexCacheStore())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.core.disk.watch import DiskWalk, build_delta_hook
|
||||
from mirage.types import FileChangeKind, PathSpec
|
||||
|
||||
|
||||
def _accessor(root: Path) -> DiskAccessor:
|
||||
return DiskAccessor(root)
|
||||
|
||||
|
||||
def _root(virtual: str, resource_path: str) -> PathSpec:
|
||||
return PathSpec(virtual=virtual,
|
||||
directory=virtual,
|
||||
resource_path=resource_path)
|
||||
|
||||
|
||||
def _touch(root: Path, relative: str, body: bytes, mtime: float) -> None:
|
||||
target = root / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(body)
|
||||
os.utime(target, (mtime, mtime))
|
||||
|
||||
|
||||
async def _collect(walk, root):
|
||||
return [entry async for entry in walk(root)]
|
||||
|
||||
|
||||
def test_walk_reports_files_and_directories(tmp_path):
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
_touch(tmp_path, "data/sub/deep.txt", b"deep", 1_700_000_000)
|
||||
entries = asyncio.run(
|
||||
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/data", "data")))
|
||||
files = {e.virtual for e in entries if not e.is_dir}
|
||||
dirs = {e.virtual for e in entries if e.is_dir}
|
||||
assert files == {"/d/data/a.txt", "/d/data/sub/deep.txt"}
|
||||
assert dirs == {"/d/data/sub"}
|
||||
|
||||
|
||||
def test_walk_carries_size_and_mtime(tmp_path):
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
entries = asyncio.run(
|
||||
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/data", "data")))
|
||||
entry = next(e for e in entries if not e.is_dir)
|
||||
assert entry.size == 5
|
||||
assert entry.modified is not None
|
||||
assert entry.fingerprint == f"{entry.modified}|5"
|
||||
|
||||
|
||||
def test_missing_root_walks_empty(tmp_path):
|
||||
entries = asyncio.run(
|
||||
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/gone", "gone")))
|
||||
assert entries == []
|
||||
|
||||
|
||||
def test_baseline_then_create_update_delete(tmp_path):
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
_touch(tmp_path, "data/b.txt", b"beta", 1_700_000_000)
|
||||
hook = build_delta_hook(_accessor(tmp_path))
|
||||
root = _root("/d/data", "data")
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
assert first.changes == ()
|
||||
|
||||
_touch(tmp_path, "data/a.txt", b"gamma", 1_700_000_500)
|
||||
_touch(tmp_path, "data/c.txt", b"new", 1_700_000_500)
|
||||
(tmp_path / "data/b.txt").unlink()
|
||||
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
by_path = {c.path.virtual: c.kind for c in second.changes}
|
||||
assert by_path == {
|
||||
"/d/data/a.txt": FileChangeKind.UPDATE,
|
||||
"/d/data/c.txt": FileChangeKind.CREATE,
|
||||
"/d/data/b.txt": FileChangeKind.DELETE,
|
||||
}
|
||||
|
||||
|
||||
def test_untouched_tree_reports_nothing(tmp_path):
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
hook = build_delta_hook(_accessor(tmp_path))
|
||||
root = _root("/d/data", "data")
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
assert second.changes == ()
|
||||
|
||||
|
||||
def test_new_directory_is_reported(tmp_path):
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
hook = build_delta_hook(_accessor(tmp_path))
|
||||
root = _root("/d/data", "data")
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
(tmp_path / "data" / "fresh").mkdir()
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
assert [(c.kind, c.path.virtual) for c in second.changes
|
||||
] == [(FileChangeKind.CREATE, "/d/data/fresh")]
|
||||
|
||||
|
||||
def test_changed_path_carries_the_mount_framing(tmp_path):
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
hook = build_delta_hook(_accessor(tmp_path))
|
||||
root = _root("/d/data", "data")
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
_touch(tmp_path, "data/a.txt", b"gamma", 1_700_000_500)
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
changed = second.changes[0].path
|
||||
assert changed.virtual == "/d/data/a.txt"
|
||||
assert changed.resource_path == "data/a.txt"
|
||||
|
||||
|
||||
def test_missing_root_reports_nothing(tmp_path):
|
||||
entries = asyncio.run(
|
||||
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/gone", "gone")))
|
||||
assert entries == []
|
||||
|
||||
|
||||
def test_unreadable_directory_aborts_rather_than_reporting_empty(tmp_path):
|
||||
# An unreadable subtree is not an empty one. Swallowing the error
|
||||
# diffs into a DELETE for every child, then a CREATE for each once
|
||||
# access returns, so the walk fails and the checkpoint stands.
|
||||
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
|
||||
locked = tmp_path / "data" / "locked"
|
||||
locked.mkdir()
|
||||
(locked / "inner.txt").write_bytes(b"inner")
|
||||
locked.chmod(0o000)
|
||||
try:
|
||||
with pytest.raises(PermissionError):
|
||||
asyncio.run(
|
||||
_collect(DiskWalk(_accessor(tmp_path)),
|
||||
_root("/d/data", "data")))
|
||||
finally:
|
||||
locked.chmod(0o755)
|
||||
@@ -0,0 +1,71 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.accessor.dropbox import DropboxAccessor
|
||||
from mirage.core.dropbox._client import DropboxTokenManager
|
||||
from mirage.core.dropbox.watch import DropboxWalk
|
||||
from mirage.resource.dropbox.config import DropboxConfig
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _accessor(root_path: str) -> DropboxAccessor:
|
||||
config = DropboxConfig(client_id="c",
|
||||
client_secret="s",
|
||||
refresh_token="r",
|
||||
root_path=root_path)
|
||||
return DropboxAccessor(config, DropboxTokenManager(config))
|
||||
|
||||
|
||||
def _root() -> PathSpec:
|
||||
return PathSpec(virtual="/m", directory="/m", resource_path="")
|
||||
|
||||
|
||||
async def _collect(walk, root):
|
||||
return [entry async for entry in walk(root)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_casing_of_the_root_is_still_stripped() -> None:
|
||||
# Dropbox paths are case-insensitive: path_display carries the
|
||||
# server's casing and root_path the user's. Comparing them exactly
|
||||
# left the root on the front of every virtual path, which put every
|
||||
# event outside the watch scope and silently disabled delivery.
|
||||
listing = [{
|
||||
".tag": "file",
|
||||
"path_display": "/Team/notes.txt",
|
||||
"path_lower": "/team/notes.txt",
|
||||
"size": 4,
|
||||
"rev": "r1",
|
||||
}]
|
||||
with patch("mirage.core.dropbox.watch.list_folder", return_value=listing):
|
||||
entries = await _collect(DropboxWalk(_accessor("/team")), _root())
|
||||
assert [e.virtual for e in entries] == ["/m/notes.txt"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_casing_below_the_root_is_preserved() -> None:
|
||||
listing = [{
|
||||
".tag": "file",
|
||||
"path_display": "/Team/Notes/Report.TXT",
|
||||
"path_lower": "/team/notes/report.txt",
|
||||
"size": 4,
|
||||
"rev": "r1",
|
||||
}]
|
||||
with patch("mirage.core.dropbox.watch.list_folder", return_value=listing):
|
||||
entries = await _collect(DropboxWalk(_accessor("/team")), _root())
|
||||
assert [e.virtual for e in entries] == ["/m/Notes/Report.TXT"]
|
||||
@@ -16,34 +16,32 @@ from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.cache.index import IndexCacheStore, IndexEntry
|
||||
from mirage.cache.index.ram import RAMIndexCacheStore
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.core.github.find import find
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _index() -> RAMIndexCacheStore:
|
||||
index = RAMIndexCacheStore()
|
||||
index._entries.update({
|
||||
"/src":
|
||||
IndexEntry(id="a", name="src", resource_type="folder", size=None),
|
||||
"/src/main.py":
|
||||
IndexEntry(id="b", name="main.py", resource_type="file", size=120),
|
||||
"/src/utils":
|
||||
IndexEntry(id="c", name="utils", resource_type="folder", size=None),
|
||||
"/src/utils/helpers.py":
|
||||
IndexEntry(id="d", name="helpers.py", resource_type="file", size=80),
|
||||
"/README.md":
|
||||
IndexEntry(id="e", name="README.md", resource_type="file", size=50),
|
||||
})
|
||||
return index
|
||||
def _accessor() -> GitHubAccessor:
|
||||
"""A mount whose git tree holds the fixture repository.
|
||||
|
||||
|
||||
class EntryOnlyIndex(IndexCacheStore):
|
||||
|
||||
async def entries(self) -> dict[str, IndexEntry]:
|
||||
return await _index().entries()
|
||||
find reads the tree, not the index: the tree keys are repo-relative,
|
||||
which is the space find compares in.
|
||||
"""
|
||||
tree = {
|
||||
"src":
|
||||
TreeEntry(path="src", type="tree", sha="a", size=None),
|
||||
"src/main.py":
|
||||
TreeEntry(path="src/main.py", type="blob", sha="b", size=120),
|
||||
"src/utils":
|
||||
TreeEntry(path="src/utils", type="tree", sha="c", size=None),
|
||||
"src/utils/helpers.py":
|
||||
TreeEntry(path="src/utils/helpers.py", type="blob", sha="d", size=80),
|
||||
"README.md":
|
||||
TreeEntry(path="README.md", type="blob", sha="e", size=50),
|
||||
}
|
||||
return GitHubAccessor(None, "acme", "proj", "main", "main", tree=tree)
|
||||
|
||||
|
||||
def _spec(path: str, prefix: str = "") -> PathSpec:
|
||||
@@ -54,55 +52,48 @@ def _spec(path: str, prefix: str = "") -> PathSpec:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_all_from_root():
|
||||
results = await find(None, _spec("/"), index=_index())
|
||||
results = await find(_accessor(), _spec("/"))
|
||||
assert results == [
|
||||
"/", "/README.md", "/src", "/src/main.py", "/src/utils",
|
||||
"/src/utils/helpers.py"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_uses_store_interface_not_ram_implementation():
|
||||
results = await find(None, _spec("/"), index=EntryOnlyIndex())
|
||||
assert "/src/main.py" in results
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_name_pattern():
|
||||
results = await find(None, _spec("/"), name="*.py", index=_index())
|
||||
results = await find(_accessor(), _spec("/"), name="*.py")
|
||||
assert results == ["/src/main.py", "/src/utils/helpers.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_type_directory():
|
||||
results = await find(None, _spec("/src"), type="d", index=_index())
|
||||
results = await find(_accessor(), _spec("/src"), type="d")
|
||||
assert results == ["/src", "/src/utils"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_type_file_under_subdir():
|
||||
results = await find(None, _spec("/src"), type="f", index=_index())
|
||||
results = await find(_accessor(), _spec("/src"), type="f")
|
||||
assert results == ["/src/main.py", "/src/utils/helpers.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_maxdepth():
|
||||
results = await find(None, _spec("/src"), maxdepth=1, index=_index())
|
||||
results = await find(_accessor(), _spec("/src"), maxdepth=1)
|
||||
assert results == ["/src", "/src/main.py", "/src/utils"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_mindepth():
|
||||
results = await find(None, _spec("/src"), mindepth=2, index=_index())
|
||||
results = await find(_accessor(), _spec("/src"), mindepth=2)
|
||||
assert results == ["/src/utils/helpers.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_strips_mount_prefix():
|
||||
results = await find(None,
|
||||
results = await find(_accessor(),
|
||||
_spec("/github/src", prefix="/github"),
|
||||
type="f",
|
||||
index=_index())
|
||||
type="f")
|
||||
assert results == ["/src/main.py", "/src/utils/helpers.py"]
|
||||
|
||||
|
||||
@@ -110,58 +101,51 @@ async def test_find_strips_mount_prefix():
|
||||
async def test_find_size_filters():
|
||||
# Directories contribute size 0 to -size, so the root is excluded
|
||||
# under a positive minimum (#318).
|
||||
results = await find(None, _spec("/"), min_size=100, index=_index())
|
||||
results = await find(_accessor(), _spec("/"), min_size=100)
|
||||
assert results == ["/src/main.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_file_start_path():
|
||||
results = await find(None, _spec("/src/main.py"), index=_index())
|
||||
results = await find(_accessor(), _spec("/src/main.py"))
|
||||
assert results == ["/src/main.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_size_filters_file_start():
|
||||
too_big = await find(None,
|
||||
_spec("/src/main.py"),
|
||||
max_size=50,
|
||||
index=_index())
|
||||
too_big = await find(_accessor(), _spec("/src/main.py"), max_size=50)
|
||||
assert too_big == []
|
||||
big_enough = await find(None,
|
||||
_spec("/src/main.py"),
|
||||
min_size=100,
|
||||
index=_index())
|
||||
big_enough = await find(_accessor(), _spec("/src/main.py"), min_size=100)
|
||||
assert big_enough == ["/src/main.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_mtime_filters_unknown_and_out_of_window_entries():
|
||||
index = _index()
|
||||
index._entries["/src/main.py"] = index._entries["/src/main.py"].model_copy(
|
||||
update={"remote_time": "2026-07-15T12:00:00+00:00"})
|
||||
|
||||
async def test_find_mtime_excludes_every_entry_a_git_tree_has_no_times():
|
||||
# A git tree carries no timestamps, so every entry's mtime is unknown
|
||||
# and -mtime excludes it. This was already true through the index,
|
||||
# which nothing ever gave a remote_time.
|
||||
results = await find(
|
||||
None,
|
||||
_accessor(),
|
||||
_spec("/"),
|
||||
mtime_min=datetime(2026, 7, 15, tzinfo=timezone.utc).timestamp(),
|
||||
mtime_max=datetime(2026, 7, 16, tzinfo=timezone.utc).timestamp(),
|
||||
index=index,
|
||||
)
|
||||
|
||||
assert results == ["/src/main.py"]
|
||||
assert results == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_empty_matches_empty_files_and_directories():
|
||||
index = _index()
|
||||
index._entries["/empty.txt"] = IndexEntry(id="empty-file",
|
||||
name="empty.txt",
|
||||
resource_type="file",
|
||||
size=0)
|
||||
index._entries["/empty-dir"] = IndexEntry(id="empty-dir",
|
||||
name="empty-dir",
|
||||
resource_type="folder")
|
||||
accessor = _accessor()
|
||||
accessor.tree["empty.txt"] = TreeEntry(path="empty.txt",
|
||||
type="blob",
|
||||
sha="empty-file",
|
||||
size=0)
|
||||
accessor.tree["empty-dir"] = TreeEntry(path="empty-dir",
|
||||
type="tree",
|
||||
sha="empty-dir",
|
||||
size=None)
|
||||
|
||||
results = await find(None, _spec("/"), empty=True, index=index)
|
||||
results = await find(accessor, _spec("/"), empty=True)
|
||||
|
||||
assert results == ["/empty-dir", "/empty.txt"]
|
||||
|
||||
@@ -58,6 +58,14 @@ def _index() -> RAMIndexCacheStore:
|
||||
index._entries["/src/main.py"] = entry
|
||||
index._children["/src"] = ["/src/main.py"]
|
||||
index._expiry["/src"] = datetime.now(timezone.utc) + timedelta(days=365)
|
||||
# The root row is what makes this a live index rather than a dropped
|
||||
# one; without it every read here would be a refill, which is the
|
||||
# distinction ensure_live_index draws.
|
||||
index._entries["/src"] = IndexEntry(id="aaa",
|
||||
name="src",
|
||||
resource_type="folder")
|
||||
index._children["/"] = ["/src"]
|
||||
index._expiry["/"] = datetime.now(timezone.utc) + timedelta(days=365)
|
||||
return index
|
||||
|
||||
|
||||
|
||||
@@ -24,20 +24,21 @@ from mirage.utils.key_prefix import mount_key
|
||||
|
||||
@pytest.fixture
|
||||
def entries():
|
||||
"""A git tree, keyed repo-relative with no leading slash."""
|
||||
|
||||
def _f():
|
||||
return SimpleNamespace(resource_type="file")
|
||||
return SimpleNamespace(type="blob")
|
||||
|
||||
def _d():
|
||||
return SimpleNamespace(resource_type="folder")
|
||||
return SimpleNamespace(type="tree")
|
||||
|
||||
return {
|
||||
"/README.md": _f(),
|
||||
"/src": _d(),
|
||||
"/src/main.py": _f(),
|
||||
"/src/utils.py": _f(),
|
||||
"/src/models": _d(),
|
||||
"/src/models/user.py": _f(),
|
||||
"README.md": _f(),
|
||||
"src": _d(),
|
||||
"src/main.py": _f(),
|
||||
"src/utils.py": _f(),
|
||||
"src/models": _d(),
|
||||
"src/models/user.py": _f(),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -17,8 +17,12 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.cache.index import NULL_INDEX
|
||||
from mirage.cache.index.ram import RAMIndexCacheStore
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.tree import fetch_dir_tree, fetch_tree
|
||||
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
|
||||
fetch_tree, index_rows)
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
|
||||
|
||||
@@ -133,3 +137,117 @@ async def test_fetch_tree_passes_params(mock_get, config):
|
||||
repo="proj",
|
||||
ref="v1",
|
||||
)
|
||||
|
||||
|
||||
def _tree_payload() -> dict:
|
||||
return {
|
||||
"truncated":
|
||||
False,
|
||||
"tree": [
|
||||
{
|
||||
"path": "data",
|
||||
"type": "tree",
|
||||
"sha": "t1",
|
||||
"size": None
|
||||
},
|
||||
{
|
||||
"path": "data/keep.txt",
|
||||
"type": "blob",
|
||||
"sha": "b1",
|
||||
"size": 4
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _accessor(config):
|
||||
tree = {
|
||||
"data":
|
||||
TreeEntry(path="data", type="tree", sha="t1", size=None),
|
||||
"data/keep.txt":
|
||||
TreeEntry(path="data/keep.txt", type="blob", sha="b1", size=4),
|
||||
}
|
||||
return GitHubAccessor(config, "acme", "proj", "main", "main", tree=tree)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.tree.github_get")
|
||||
async def test_ensure_live_index_refetches_the_build_tree(mock_get, config):
|
||||
# The build tree is only true at build time: a mount's first read can
|
||||
# come long after it, so reusing it would key an index built from a
|
||||
# repository several external writes ago.
|
||||
mock_get.return_value = _tree_payload()
|
||||
index = RAMIndexCacheStore(ttl=600)
|
||||
accessor = _accessor(config)
|
||||
assert await ensure_live_index(accessor, index, "/gh") is True
|
||||
mock_get.assert_awaited_once()
|
||||
assert (await index.list_dir("/gh/data")).entries == ["/gh/data/keep.txt"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.tree.github_get")
|
||||
async def test_ensure_live_index_refetches_a_dropped_listing(mock_get, config):
|
||||
mock_get.return_value = _tree_payload()
|
||||
index = RAMIndexCacheStore(ttl=600)
|
||||
accessor = _accessor(config)
|
||||
await ensure_live_index(accessor, index, "/gh")
|
||||
# What invalidation does: drop the row rather than expire it, which
|
||||
# is why the readers' EXPIRED probe never fires.
|
||||
await index.invalidate_dir("/gh")
|
||||
await index.invalidate_dir("/gh/data")
|
||||
assert await ensure_live_index(accessor, index, "/gh") is True
|
||||
assert mock_get.await_count == 2
|
||||
assert (await index.list_dir("/gh/data")).entries == ["/gh/data/keep.txt"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.tree.github_get")
|
||||
async def test_ensure_live_index_leaves_a_live_index_alone(mock_get, config):
|
||||
mock_get.return_value = _tree_payload()
|
||||
index = RAMIndexCacheStore(ttl=600)
|
||||
accessor = _accessor(config)
|
||||
await ensure_live_index(accessor, index, "/gh")
|
||||
mock_get.reset_mock()
|
||||
assert await ensure_live_index(accessor, index, "/gh") is False
|
||||
mock_get.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.tree.github_get")
|
||||
async def test_ensure_live_index_skips_a_truncated_tree(mock_get, config):
|
||||
index = RAMIndexCacheStore(ttl=600)
|
||||
accessor = _accessor(config)
|
||||
accessor.truncated = True
|
||||
assert await ensure_live_index(accessor, index, "/gh") is False
|
||||
mock_get.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_live_index_skips_the_null_index(config):
|
||||
assert await ensure_live_index(_accessor(config), NULL_INDEX, "") is False
|
||||
|
||||
|
||||
def test_index_rows_key_by_mount_absolute_path():
|
||||
# Every other backend keys its index this way, which is what lets the
|
||||
# shared CacheManager spell an eviction without knowing the backend.
|
||||
tree = {
|
||||
"data":
|
||||
TreeEntry(path="data", type="tree", sha="t1", size=None),
|
||||
"data/keep.txt":
|
||||
TreeEntry(path="data/keep.txt", type="blob", sha="b1", size=4),
|
||||
}
|
||||
entries, children = index_rows(tree, "/gh")
|
||||
assert sorted(entries) == ["/gh/data", "/gh/data/keep.txt"]
|
||||
assert sorted(children) == ["/gh", "/gh/data"]
|
||||
|
||||
|
||||
def test_index_rows_root_mount_keeps_bare_paths():
|
||||
entries, children = index_rows(
|
||||
{"a.txt": TreeEntry(path="a.txt", type="blob", sha="b", size=1)}, "")
|
||||
assert sorted(entries) == ["/a.txt"]
|
||||
assert sorted(children) == ["/"]
|
||||
|
||||
|
||||
def test_index_rows_gives_an_empty_repo_a_root_row():
|
||||
_entries, children = index_rows({}, "/gh")
|
||||
assert children == {"/gh": []}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
from mirage.core.github.watch import GitHubWalk
|
||||
from mirage.types import PathSpec
|
||||
from mirage.watch.errors import IncompleteWalkError
|
||||
|
||||
CONFIG = GitHubConfig(token="t")
|
||||
|
||||
|
||||
def _accessor(tree: dict[str, TreeEntry]) -> GitHubAccessor:
|
||||
return GitHubAccessor(config=CONFIG,
|
||||
owner="acme",
|
||||
repo="proj",
|
||||
ref="main",
|
||||
default_branch="main",
|
||||
tree=tree)
|
||||
|
||||
|
||||
def _root() -> PathSpec:
|
||||
return PathSpec(virtual="/gh", directory="/gh", resource_path="")
|
||||
|
||||
|
||||
def _entry(path: str, sha: str) -> TreeEntry:
|
||||
return TreeEntry(path=path, type="blob", sha=sha, size=3)
|
||||
|
||||
|
||||
async def _collect(walk, root):
|
||||
return [entry async for entry in walk(root)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_refreshes_the_accessor_tree() -> None:
|
||||
# find, du and grep's scope counter read accessor.tree directly, so a
|
||||
# pull that fetched a newer tree and dropped it left them reporting the
|
||||
# repository as it stood when the mount was built.
|
||||
stale = {"a.txt": _entry("a.txt", "sha-a")}
|
||||
fresh = {
|
||||
"a.txt": _entry("a.txt", "sha-a"),
|
||||
"b.txt": _entry("b.txt", "sha-b"),
|
||||
}
|
||||
accessor = _accessor(stale)
|
||||
with patch("mirage.core.github.watch.fetch_tree",
|
||||
return_value=(fresh, False)):
|
||||
await _collect(GitHubWalk(accessor), _root())
|
||||
assert accessor.tree == fresh
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_truncated_tree_is_not_adopted() -> None:
|
||||
# A partial tree would make find report the missing half as deleted.
|
||||
stale = {"a.txt": _entry("a.txt", "sha-a")}
|
||||
accessor = _accessor(stale)
|
||||
with patch("mirage.core.github.watch.fetch_tree", return_value=({}, True)):
|
||||
with pytest.raises(IncompleteWalkError):
|
||||
await _collect(GitHubWalk(accessor), _root())
|
||||
assert accessor.tree == stale
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_walk_reports_blobs_with_their_sha() -> None:
|
||||
tree = {"a.txt": _entry("a.txt", "sha-a")}
|
||||
accessor = _accessor(tree)
|
||||
with patch("mirage.core.github.watch.fetch_tree",
|
||||
return_value=(tree, False)):
|
||||
entries = await _collect(GitHubWalk(accessor), _root())
|
||||
assert [(e.virtual, e.fingerprint)
|
||||
for e in entries] == [("/gh/a.txt", "sha-a")]
|
||||
@@ -0,0 +1,122 @@
|
||||
import asyncio
|
||||
|
||||
from mirage.accessor.s3 import S3Accessor
|
||||
from mirage.core.s3.watch import S3Walk, build_delta_hook
|
||||
from mirage.resource.s3 import S3Config
|
||||
from mirage.types import FileChangeKind, PathSpec
|
||||
from tests.e2e.s3_mock import patch_s3_multi
|
||||
|
||||
BUCKET = "watch-bucket"
|
||||
|
||||
|
||||
def _accessor(key_prefix: str | None = None) -> S3Accessor:
|
||||
return S3Accessor(
|
||||
S3Config(bucket=BUCKET,
|
||||
region="us-east-1",
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
key_prefix=key_prefix))
|
||||
|
||||
|
||||
def _root(virtual: str, resource_path: str) -> PathSpec:
|
||||
return PathSpec(virtual=virtual,
|
||||
directory=virtual,
|
||||
resource_path=resource_path)
|
||||
|
||||
|
||||
async def _collect(walk, root):
|
||||
return [entry async for entry in walk(root)]
|
||||
|
||||
|
||||
def test_walk_yields_files_with_etag_fingerprints():
|
||||
store = {BUCKET: {"data/a.txt": b"alpha", "data/b.txt": b"beta"}}
|
||||
with patch_s3_multi(store):
|
||||
entries = asyncio.run(
|
||||
_collect(S3Walk(_accessor()), _root("/s3/data", "data")))
|
||||
files = {e.virtual: e for e in entries if not e.is_dir}
|
||||
assert set(files) == {"/s3/data/a.txt", "/s3/data/b.txt"}
|
||||
assert files["/s3/data/a.txt"].size == 5
|
||||
# ETag, not the mtime|size composite: the mock's LastModified is a
|
||||
# constant, so a composite would collide across files of equal size.
|
||||
assert "|" not in (files["/s3/data/a.txt"].fingerprint or "")
|
||||
assert (files["/s3/data/a.txt"].fingerprint
|
||||
!= files["/s3/data/b.txt"].fingerprint)
|
||||
|
||||
|
||||
def test_walk_synthesizes_intermediate_directories():
|
||||
store = {BUCKET: {"data/sub/deep/x.txt": b"x"}}
|
||||
with patch_s3_multi(store):
|
||||
entries = asyncio.run(
|
||||
_collect(S3Walk(_accessor()), _root("/s3/data", "data")))
|
||||
dirs = {e.virtual for e in entries if e.is_dir}
|
||||
assert dirs == {"/s3/data/sub", "/s3/data/sub/deep"}
|
||||
|
||||
|
||||
def test_walk_reports_an_explicit_marker_as_its_own_directory():
|
||||
store = {BUCKET: {"data/empty/": b""}}
|
||||
with patch_s3_multi(store):
|
||||
entries = asyncio.run(
|
||||
_collect(S3Walk(_accessor()), _root("/s3/data", "data")))
|
||||
assert [e.virtual for e in entries if e.is_dir] == ["/s3/data/empty"]
|
||||
assert not [e for e in entries if not e.is_dir]
|
||||
|
||||
|
||||
def test_walk_strips_the_key_prefix():
|
||||
store = {BUCKET: {"team/x/data/a.txt": b"alpha"}}
|
||||
with patch_s3_multi(store):
|
||||
entries = asyncio.run(
|
||||
_collect(S3Walk(_accessor("team/x/")), _root("/s3/data", "data")))
|
||||
assert [e.virtual for e in entries if not e.is_dir] == ["/s3/data/a.txt"]
|
||||
|
||||
|
||||
def test_baseline_pull_reports_nothing_then_detects_a_write():
|
||||
store = {BUCKET: {"data/a.txt": b"alpha"}}
|
||||
hook = build_delta_hook(_accessor())
|
||||
root = _root("/s3/data", "data")
|
||||
with patch_s3_multi(store):
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
assert first.changes == ()
|
||||
store[BUCKET]["data/a.txt"] = b"gamma"
|
||||
store[BUCKET]["data/new.txt"] = b"new"
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
by_path = {c.path.virtual: c.kind for c in second.changes}
|
||||
assert by_path == {
|
||||
"/s3/data/a.txt": FileChangeKind.UPDATE,
|
||||
"/s3/data/new.txt": FileChangeKind.CREATE,
|
||||
}
|
||||
|
||||
|
||||
def test_delete_is_detected():
|
||||
store = {BUCKET: {"data/a.txt": b"alpha", "data/b.txt": b"beta"}}
|
||||
hook = build_delta_hook(_accessor())
|
||||
root = _root("/s3/data", "data")
|
||||
with patch_s3_multi(store):
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
del store[BUCKET]["data/b.txt"]
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
assert [(c.kind, c.path.virtual) for c in second.changes
|
||||
] == [(FileChangeKind.DELETE, "/s3/data/b.txt")]
|
||||
|
||||
|
||||
def test_same_bytes_rewritten_is_not_a_change():
|
||||
store = {BUCKET: {"data/a.txt": b"alpha"}}
|
||||
hook = build_delta_hook(_accessor())
|
||||
root = _root("/s3/data", "data")
|
||||
with patch_s3_multi(store):
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
store[BUCKET]["data/a.txt"] = b"alpha"
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
assert second.changes == ()
|
||||
|
||||
|
||||
def test_changed_path_carries_the_mount_framing():
|
||||
store = {BUCKET: {"data/a.txt": b"alpha"}}
|
||||
hook = build_delta_hook(_accessor())
|
||||
root = _root("/s3/data", "data")
|
||||
with patch_s3_multi(store):
|
||||
first = asyncio.run(hook.pull(root, None))
|
||||
store[BUCKET]["data/a.txt"] = b"gamma"
|
||||
second = asyncio.run(hook.pull(root, first.checkpoint))
|
||||
changed = second.changes[0].path
|
||||
assert changed.virtual == "/s3/data/a.txt"
|
||||
assert changed.resource_path == "data/a.txt"
|
||||
+25
-15
@@ -37,6 +37,7 @@ _CORE_MODULES = [
|
||||
"mirage.core.s3.mkdir",
|
||||
"mirage.core.s3.create",
|
||||
"mirage.core.s3.truncate",
|
||||
"mirage.core.s3.watch",
|
||||
]
|
||||
|
||||
|
||||
@@ -59,6 +60,24 @@ def _mock_s3_error(code: str) -> Exception:
|
||||
return exc
|
||||
|
||||
|
||||
def _content_entry(key: str, data: bytes) -> dict[str, object]:
|
||||
"""One Contents row, shaped like the real list_objects_v2.
|
||||
|
||||
Real S3 carries an ETag on every listed object, which is what the
|
||||
watch walk fingerprints on, so the mock has to carry one too.
|
||||
|
||||
Args:
|
||||
key (str): Object key.
|
||||
data (bytes): Object content.
|
||||
"""
|
||||
return {
|
||||
"Key": key,
|
||||
"Size": len(data),
|
||||
"LastModified": LAST_MODIFIED,
|
||||
"ETag": f'"{hashlib.md5(data).hexdigest()}"',
|
||||
}
|
||||
|
||||
|
||||
def _paginate_directory(objects, prefix):
|
||||
common_prefixes: set[str] = set()
|
||||
contents: list[dict[str, object]] = []
|
||||
@@ -67,21 +86,13 @@ def _paginate_directory(objects, prefix):
|
||||
continue
|
||||
relative = key[len(prefix):]
|
||||
if not relative:
|
||||
contents.append({
|
||||
"Key": key,
|
||||
"Size": len(data),
|
||||
"LastModified": LAST_MODIFIED
|
||||
})
|
||||
contents.append(_content_entry(key, data))
|
||||
continue
|
||||
if "/" in relative:
|
||||
child = relative.split("/", 1)[0]
|
||||
common_prefixes.add(prefix + child + "/")
|
||||
continue
|
||||
contents.append({
|
||||
"Key": key,
|
||||
"Size": len(data),
|
||||
"LastModified": LAST_MODIFIED
|
||||
})
|
||||
contents.append(_content_entry(key, data))
|
||||
return {
|
||||
"CommonPrefixes": [{
|
||||
"Prefix": v
|
||||
@@ -92,11 +103,10 @@ def _paginate_directory(objects, prefix):
|
||||
|
||||
def _paginate_flat(objects, prefix):
|
||||
return {
|
||||
"Contents": [{
|
||||
"Key": k,
|
||||
"Size": len(v),
|
||||
"LastModified": LAST_MODIFIED
|
||||
} for k, v in sorted(objects.items()) if k.startswith(prefix)]
|
||||
"Contents": [
|
||||
_content_entry(k, v) for k, v in sorted(objects.items())
|
||||
if k.startswith(prefix)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
Vendored
+4
@@ -131,5 +131,9 @@ def mock_github_api(monkeypatch):
|
||||
_fetch_default_branch)
|
||||
monkeypatch.setattr("mirage.resource.github.github.fetch_tree",
|
||||
_fetch_tree)
|
||||
# refill_index reads the name in its own module, and an empty index is
|
||||
# a refill trigger now, so leaving this one real let a test reach the
|
||||
# live API.
|
||||
monkeypatch.setattr("mirage.core.github.tree.fetch_tree", _fetch_tree)
|
||||
monkeypatch.setattr("mirage.core.github.read.read_bytes", _read_bytes)
|
||||
monkeypatch.setattr("mirage.core.github.search.search_code", _search_code)
|
||||
|
||||
@@ -28,16 +28,36 @@ OWNER = "test-owner"
|
||||
REPO = "test-repo"
|
||||
|
||||
|
||||
def _offline(tree: dict,
|
||||
truncated: bool = False,
|
||||
default_branch: str = "main"):
|
||||
"""Patch every path that would reach the network.
|
||||
|
||||
``mirage.core.github.tree.fetch_tree`` is the second one: a read fills
|
||||
the index by refetching, because the tree a mount was built with is
|
||||
only true at build time.
|
||||
|
||||
Args:
|
||||
tree (dict): The recursive tree to answer with.
|
||||
truncated (bool): Whether to report it truncated.
|
||||
default_branch (str): Branch the repo endpoint reports.
|
||||
"""
|
||||
return (patch("mirage.resource.github.github.fetch_default_branch",
|
||||
return_value=default_branch),
|
||||
patch("mirage.resource.github.github.fetch_tree",
|
||||
return_value=(tree, truncated)),
|
||||
patch("mirage.core.github.tree.fetch_tree",
|
||||
return_value=(tree, truncated)))
|
||||
|
||||
|
||||
async def _make_resource(ref: str = "main",
|
||||
default_branch: str = "main",
|
||||
tree: dict | None = None,
|
||||
truncated: bool = False) -> GitHubResource:
|
||||
if tree is None:
|
||||
tree = {}
|
||||
with patch("mirage.resource.github.github.fetch_default_branch",
|
||||
return_value=default_branch), \
|
||||
patch("mirage.resource.github.github.fetch_tree",
|
||||
return_value=(tree, truncated)):
|
||||
branch_p, build_p, core_p = _offline(tree, truncated, default_branch)
|
||||
with branch_p, build_p, core_p:
|
||||
return await GitHubResource.build(
|
||||
config=CONFIG,
|
||||
owner=OWNER,
|
||||
@@ -137,13 +157,15 @@ async def test_stat_returns_sha_fingerprint() -> None:
|
||||
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
|
||||
}
|
||||
resource = await _make_resource(tree=tree)
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"), resource.index)
|
||||
with _offline(tree)[2]:
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"),
|
||||
resource.index)
|
||||
assert result.fingerprint == "abc123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replacing_index_preserves_preloaded_tree() -> None:
|
||||
async def test_replacing_index_still_serves_the_tree() -> None:
|
||||
tree = {
|
||||
"src/main.py":
|
||||
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
|
||||
@@ -151,15 +173,19 @@ async def test_replacing_index_preserves_preloaded_tree() -> None:
|
||||
resource = await _make_resource(tree=tree)
|
||||
resource.set_index(IndexConfig())
|
||||
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"), resource.index)
|
||||
# The fresh store is empty, which reads as not-live, so the next read
|
||||
# fills it by refetching rather than reporting the path gone.
|
||||
with _offline(tree)[2]:
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"),
|
||||
resource.index)
|
||||
assert result.fingerprint == "abc123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_raises_when_path_not_in_tree() -> None:
|
||||
resource = await _make_resource()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
with _offline({})[2], pytest.raises(FileNotFoundError):
|
||||
await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/nonexistent.py"), resource.index)
|
||||
|
||||
@@ -194,6 +220,10 @@ async def test_the_constructor_reaches_no_network() -> None:
|
||||
branch.assert_not_called()
|
||||
fetch.assert_not_called()
|
||||
assert resource.accessor.default_branch == "main"
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"), resource.index)
|
||||
# A read is where the network comes in: the index is keyed by mount
|
||||
# prefix, so filling it waits for a PathSpec and refetches then.
|
||||
with _offline(tree)[2]:
|
||||
result = await stat(resource.accessor,
|
||||
PathSpec.from_str_path("/src/main.py"),
|
||||
resource.index)
|
||||
assert result.fingerprint == "abc123"
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.cache.index import IndexCacheStore
|
||||
from mirage.types import FileStat, FileType, PathSpec
|
||||
from mirage.watch.walk import ReaddirWalk, entry_of, synth_dirs
|
||||
|
||||
|
||||
def _root(virtual: str, resource_path: str) -> PathSpec:
|
||||
return PathSpec(virtual=virtual,
|
||||
directory=virtual,
|
||||
resource_path=resource_path)
|
||||
|
||||
|
||||
def test_synth_dirs_emits_every_ancestor_excluding_the_root():
|
||||
dirs = [
|
||||
e.virtual for e in synth_dirs("/m/data", ["/m/data/a/b/c.txt"], [])
|
||||
]
|
||||
assert dirs == ["/m/data/a/b", "/m/data/a"]
|
||||
|
||||
|
||||
def test_synth_dirs_reports_a_shared_prefix_once():
|
||||
dirs = list(
|
||||
synth_dirs("/m/data", ["/m/data/a/x.txt", "/m/data/a/y.txt"], []))
|
||||
assert [e.virtual for e in dirs] == ["/m/data/a"]
|
||||
|
||||
|
||||
def test_synth_dirs_emits_a_stored_directory_with_no_children():
|
||||
dirs = list(synth_dirs("/m/data", [], ["/m/data/empty"]))
|
||||
assert [e.virtual for e in dirs] == ["/m/data/empty"]
|
||||
|
||||
|
||||
def test_synth_dirs_does_not_double_report_a_marker_with_children():
|
||||
dirs = list(synth_dirs("/m/data", ["/m/data/a/x.txt"], ["/m/data/a"]))
|
||||
assert [e.virtual for e in dirs] == ["/m/data/a"]
|
||||
|
||||
|
||||
def test_synth_dirs_rows_are_directories_without_fingerprints():
|
||||
dirs = list(synth_dirs("/m/data", ["/m/data/a/x.txt"], []))
|
||||
assert all(e.is_dir and e.fingerprint is None for e in dirs)
|
||||
|
||||
|
||||
def test_synth_dirs_emits_nothing_for_a_file_at_the_root():
|
||||
assert list(synth_dirs("/m/data", ["/m/data/x.txt"], [])) == []
|
||||
|
||||
|
||||
def test_entry_of_reports_a_directory_without_a_fingerprint():
|
||||
entry = entry_of("/m/d", FileStat(name="d", type=FileType.DIRECTORY))
|
||||
assert entry.is_dir is True
|
||||
assert entry.fingerprint is None
|
||||
|
||||
|
||||
def test_entry_of_prefers_the_backend_fingerprint():
|
||||
stat = FileStat(name="f.txt", size=3, modified="T", fingerprint="etag-1")
|
||||
assert entry_of("/m/f.txt", stat).fingerprint == "etag-1"
|
||||
|
||||
|
||||
def test_entry_of_falls_back_to_the_composite():
|
||||
stat = FileStat(name="f.txt", size=3, modified="T")
|
||||
assert entry_of("/m/f.txt", stat).fingerprint == "T|3"
|
||||
|
||||
|
||||
def _backend(tree: dict[str, dict]) -> ReaddirWalk:
|
||||
|
||||
async def readdir(spec: PathSpec, index: IndexCacheStore) -> list[str]:
|
||||
node = tree.get(spec.virtual)
|
||||
if node is None or "children" not in node:
|
||||
raise FileNotFoundError(spec.virtual)
|
||||
return list(node["children"])
|
||||
|
||||
async def stat(spec: PathSpec, index: IndexCacheStore) -> FileStat:
|
||||
node = tree.get(spec.virtual)
|
||||
if node is None:
|
||||
raise FileNotFoundError(spec.virtual)
|
||||
return node["stat"]
|
||||
|
||||
return ReaddirWalk(readdir, stat)
|
||||
|
||||
|
||||
async def _collect(walk: ReaddirWalk, spec: PathSpec) -> list:
|
||||
return [entry async for entry in walk(spec)]
|
||||
|
||||
|
||||
def test_readdir_walk_descends_and_reports_leaves():
|
||||
walk = _backend({
|
||||
"/m/data": {
|
||||
"children": ["/m/data/a.txt", "/m/data/sub"],
|
||||
"stat": FileStat(name="data", type=FileType.DIRECTORY),
|
||||
},
|
||||
"/m/data/a.txt": {
|
||||
"stat":
|
||||
FileStat(name="a.txt", size=5, modified="T1", fingerprint="fp-a")
|
||||
},
|
||||
"/m/data/sub": {
|
||||
"children": ["/m/data/sub/deep.txt"],
|
||||
"stat": FileStat(name="sub", type=FileType.DIRECTORY),
|
||||
},
|
||||
"/m/data/sub/deep.txt": {
|
||||
"stat":
|
||||
FileStat(name="deep.txt",
|
||||
size=4,
|
||||
modified="T2",
|
||||
fingerprint="fp-d")
|
||||
},
|
||||
})
|
||||
entries = asyncio.run(_collect(walk, _root("/m/data", "data")))
|
||||
assert [e.virtual for e in entries] == [
|
||||
"/m/data/a.txt",
|
||||
"/m/data/sub",
|
||||
"/m/data/sub/deep.txt",
|
||||
]
|
||||
assert [e.virtual for e in entries if e.is_dir] == ["/m/data/sub"]
|
||||
|
||||
|
||||
def test_readdir_walk_trusts_a_trailing_slash_without_a_stat():
|
||||
# No stat entry for the child at all: the slash is the proof, so a
|
||||
# stat would raise and the walk would lose the subtree.
|
||||
walk = _backend({
|
||||
"/m/data": {
|
||||
"children": ["/m/data/sub/"],
|
||||
"stat": FileStat(name="data", type=FileType.DIRECTORY),
|
||||
},
|
||||
"/m/data/sub": {
|
||||
"children": [],
|
||||
"stat": FileStat(name="sub", type=FileType.DIRECTORY),
|
||||
},
|
||||
})
|
||||
entries = asyncio.run(_collect(walk, _root("/m/data", "data")))
|
||||
assert [(e.virtual, e.is_dir) for e in entries] == [("/m/data/sub", True)]
|
||||
|
||||
|
||||
def test_readdir_walk_skips_an_entry_that_vanished_mid_walk():
|
||||
walk = _backend({
|
||||
"/m/data": {
|
||||
"children": ["/m/data/gone.txt", "/m/data/here.txt"],
|
||||
"stat": FileStat(name="data", type=FileType.DIRECTORY),
|
||||
},
|
||||
"/m/data/here.txt": {
|
||||
"stat":
|
||||
FileStat(name="here.txt", size=1, modified="T", fingerprint="fp")
|
||||
},
|
||||
})
|
||||
entries = asyncio.run(_collect(walk, _root("/m/data", "data")))
|
||||
assert [e.virtual for e in entries] == ["/m/data/here.txt"]
|
||||
|
||||
|
||||
def test_readdir_walk_treats_a_missing_root_as_empty():
|
||||
entries = asyncio.run(_collect(_backend({}), _root("/m/gone", "gone")))
|
||||
assert entries == []
|
||||
|
||||
|
||||
def test_readdir_walk_propagates_a_non_absence_error():
|
||||
|
||||
async def readdir(spec: PathSpec, index: IndexCacheStore) -> list[str]:
|
||||
raise PermissionError("rate limited")
|
||||
|
||||
async def stat(spec: PathSpec, index: IndexCacheStore) -> FileStat:
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
with pytest.raises(PermissionError):
|
||||
asyncio.run(
|
||||
_collect(ReaddirWalk(readdir, stat), _root("/m/data", "data")))
|
||||
|
||||
|
||||
def test_readdir_walk_starts_from_an_empty_index_on_every_call():
|
||||
seen: list[IndexCacheStore] = []
|
||||
|
||||
async def readdir(spec: PathSpec, index: IndexCacheStore) -> list[str]:
|
||||
seen.append(index)
|
||||
return ["/m/data/a.txt"] if spec.virtual == "/m/data" else []
|
||||
|
||||
async def stat(spec: PathSpec, index: IndexCacheStore) -> FileStat:
|
||||
return FileStat(name="a.txt", size=1, modified="T", fingerprint="fp")
|
||||
|
||||
walk = ReaddirWalk(readdir, stat)
|
||||
root = _root("/m/data", "data")
|
||||
asyncio.run(_collect(walk, root))
|
||||
asyncio.run(_collect(walk, root))
|
||||
# Two pulls, two distinct index instances: nothing a pull learned
|
||||
# can leak into the next one's snapshot.
|
||||
assert seen[0] is not seen[-1]
|
||||
@@ -26,10 +26,7 @@ import {
|
||||
import { read as githubRead } from '@struktoai/mirage-core/core/github/read'
|
||||
import { readdir as githubReaddir } from '@struktoai/mirage-core/core/github/readdir'
|
||||
import { stat as githubStat } from '@struktoai/mirage-core/core/github/stat'
|
||||
import {
|
||||
buildTreeMap as githubBuildTreeMap,
|
||||
populateIndex as githubPopulateIndex,
|
||||
} from '@struktoai/mirage-core/core/github/tree'
|
||||
import { buildTreeMap as githubBuildTreeMap } from '@struktoai/mirage-core/core/github/tree'
|
||||
import { GITHUB_OPS } from '@struktoai/mirage-core/ops/github/index'
|
||||
import type { RegisteredOp } from '@struktoai/mirage-core/ops/registry'
|
||||
import type { Resource } from '@struktoai/mirage-core/resource/base'
|
||||
@@ -87,8 +84,9 @@ export class GitHubResource implements Resource {
|
||||
truncated,
|
||||
tree: treeMap,
|
||||
})
|
||||
// Not seeded here: the index is keyed by mount prefix, which only a
|
||||
// PathSpec knows, so the first read seeds it from the accessor's tree.
|
||||
const index = new RAMIndexCacheStore({ ttl: 86_400 })
|
||||
await githubPopulateIndex(index, tree)
|
||||
return new GitHubResource(config, accessor, index)
|
||||
}
|
||||
|
||||
|
||||
+16
-10
@@ -46,6 +46,19 @@ export class CacheManager {
|
||||
this.cachesReads = cachesReads
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop one directory's cached listing.
|
||||
*
|
||||
* Both spellings of the directory go, because a backend may have keyed
|
||||
* it with or without its trailing slash and an eviction that hits no
|
||||
* key is silent.
|
||||
*/
|
||||
private async evictDir(virtual: string): Promise<void> {
|
||||
if (this.index === null) return
|
||||
await this.index.invalidateDir(virtual)
|
||||
await this.index.invalidateDir(virtual + '/')
|
||||
}
|
||||
|
||||
private virtual(path: string | PathSpec): string {
|
||||
let p = path instanceof PathSpec ? path.mountPath : path
|
||||
if (!p.startsWith('/')) p = '/' + p
|
||||
@@ -87,10 +100,7 @@ export class CacheManager {
|
||||
if (this.cachesReads && this.fileCache !== null) {
|
||||
await this.fileCache.remove(virtual)
|
||||
}
|
||||
if (this.index !== null) {
|
||||
await this.index.invalidateDir(virtual)
|
||||
await this.index.invalidateDir(virtual + '/')
|
||||
}
|
||||
await this.evictDir(virtual)
|
||||
await this.invalidateParent(virtual)
|
||||
}
|
||||
|
||||
@@ -109,8 +119,7 @@ export class CacheManager {
|
||||
let parent = virtual.slice(0, Math.max(virtual.lastIndexOf('/'), 0))
|
||||
while (parent !== '' && parent !== this.prefix) {
|
||||
parent = parent.slice(0, Math.max(parent.lastIndexOf('/'), 0))
|
||||
await this.index.invalidateDir(parent === '' ? '/' : parent)
|
||||
await this.index.invalidateDir(parent + '/')
|
||||
await this.evictDir(parent === '' ? '/' : parent)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,10 +142,7 @@ export class CacheManager {
|
||||
}
|
||||
|
||||
private async invalidateParent(virtual: string): Promise<void> {
|
||||
if (this.index === null) return
|
||||
const lastSlash = virtual.lastIndexOf('/')
|
||||
const parent = lastSlash > 0 ? virtual.slice(0, lastSlash) : '/'
|
||||
await this.index.invalidateDir(parent)
|
||||
await this.index.invalidateDir(parent + '/')
|
||||
await this.evictDir(lastSlash > 0 ? virtual.slice(0, lastSlash) : '/')
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ export async function readdir(
|
||||
}
|
||||
const isDir = it.type === 'folder'
|
||||
const filename = it.name
|
||||
const sha1 = typeof it.sha1 === 'string' && it.sha1 !== '' ? it.sha1 : null
|
||||
const entry = new IndexEntry({
|
||||
id: it.id,
|
||||
name: filename,
|
||||
@@ -89,6 +90,7 @@ export async function readdir(
|
||||
remoteTime: it.modified_at ?? '',
|
||||
vfsName: filename,
|
||||
size: isDir ? null : typeof it.size === 'number' ? it.size : null,
|
||||
extra: sha1 === null ? {} : { sha1 },
|
||||
})
|
||||
entries.push({ name: filename, entry, isDir })
|
||||
}
|
||||
|
||||
@@ -34,14 +34,19 @@ function statFromItem(item: BoxItem): FileStat {
|
||||
})
|
||||
}
|
||||
const size = typeof item.size === 'number' ? item.size : null
|
||||
// Box returns the content sha1 in the same listing, so prefer it: it
|
||||
// is content-addressed, where modified_at cannot tell two writes in
|
||||
// the same second apart and does not move at all on a re-upload of
|
||||
// identical bytes.
|
||||
const sha1 = typeof item.sha1 === 'string' && item.sha1 !== '' ? item.sha1 : null
|
||||
const modified = item.modified_at ?? ''
|
||||
return new FileStat({
|
||||
name: vfsName,
|
||||
size,
|
||||
type: guessType(vfsName),
|
||||
modified: item.modified_at ?? '',
|
||||
fingerprint:
|
||||
item.modified_at !== undefined && item.modified_at !== '' ? item.modified_at : null,
|
||||
extra: { box_id: item.id, resource_type: rt },
|
||||
modified,
|
||||
fingerprint: sha1 ?? (modified !== '' ? modified : null),
|
||||
extra: { box_id: item.id, resource_type: rt, ...(sha1 === null ? {} : { sha1 }) },
|
||||
})
|
||||
}
|
||||
|
||||
@@ -109,15 +114,18 @@ export async function stat(
|
||||
extra: { box_id: result.entry.id },
|
||||
})
|
||||
}
|
||||
const cachedSha1 = result.entry.extra.sha1
|
||||
const sha1 = typeof cachedSha1 === 'string' && cachedSha1 !== '' ? cachedSha1 : null
|
||||
return new FileStat({
|
||||
name: result.entry.vfsName !== '' ? result.entry.vfsName : result.entry.name,
|
||||
size: result.entry.size,
|
||||
type: guessType(result.entry.vfsName),
|
||||
modified: result.entry.remoteTime,
|
||||
fingerprint: result.entry.remoteTime !== '' ? result.entry.remoteTime : null,
|
||||
fingerprint: sha1 ?? (result.entry.remoteTime !== '' ? result.entry.remoteTime : null),
|
||||
extra: {
|
||||
box_id: result.entry.id,
|
||||
resource_type: result.entry.resourceType,
|
||||
...(sha1 === null ? {} : { sha1 }),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { BoxAccessor } from '../../accessor/box.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import type { FileStat, PathSpec } from '../../types.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { ReaddirWalk } from '../../watch/walk.ts'
|
||||
import { readdir } from './readdir.ts'
|
||||
import { stat } from './stat.ts'
|
||||
|
||||
/**
|
||||
* Build the Box delta hook.
|
||||
*
|
||||
* Box keys its tree by folder id and has no recursive listing, so the pull is
|
||||
* one `/folders/{id}/items` request per directory. Box does offer an
|
||||
* account-wide `/events` feed, which is the cheaper signal and belongs in a
|
||||
* push receiver, not here.
|
||||
*
|
||||
* Fingerprints on `modified_at`, which is what Box stat reports.
|
||||
*/
|
||||
export function buildDeltaHook(accessor: BoxAccessor): DeltaHook {
|
||||
const walk = new ReaddirWalk(
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
|
||||
)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type * as ApiModule from './api.ts'
|
||||
|
||||
vi.mock('./api.ts', async () => {
|
||||
const actual = await vi.importActual<typeof ApiModule>('./api.ts')
|
||||
return { ...actual, listFolder: vi.fn() }
|
||||
})
|
||||
|
||||
import { DropboxAccessor } from '../../accessor/dropbox.ts'
|
||||
import { PathSpec, type WalkEntry } from '../../types.ts'
|
||||
import type { DropboxTokenManager } from './_client.ts'
|
||||
import * as api from './api.ts'
|
||||
import { DropboxWalk } from './watch.ts'
|
||||
|
||||
const STUB_TM = {} as DropboxTokenManager
|
||||
|
||||
function accessor(rootPath: string): DropboxAccessor {
|
||||
return new DropboxAccessor({ tokenManager: STUB_TM, rootPath })
|
||||
}
|
||||
|
||||
function root(): PathSpec {
|
||||
return new PathSpec({ virtual: '/m', directory: '/m', resourcePath: '' })
|
||||
}
|
||||
|
||||
async function collect(walk: DropboxWalk, at: PathSpec): Promise<WalkEntry[]> {
|
||||
const out: WalkEntry[] = []
|
||||
for await (const entry of walk.walk(at)) out.push(entry)
|
||||
return out
|
||||
}
|
||||
|
||||
describe('DropboxWalk root stripping', () => {
|
||||
it('strips the root when the server casing differs', async () => {
|
||||
// Dropbox paths are case-insensitive: path_display carries the
|
||||
// server's casing and rootPath the user's. Comparing them exactly
|
||||
// left the root on the front of every virtual path, which put every
|
||||
// event outside the watch scope and silently disabled delivery.
|
||||
vi.mocked(api.listFolder).mockResolvedValue([
|
||||
{
|
||||
'.tag': 'file',
|
||||
id: 'id:1',
|
||||
name: 'notes.txt',
|
||||
path_display: '/Team/notes.txt',
|
||||
path_lower: '/team/notes.txt',
|
||||
size: 4,
|
||||
rev: 'r1',
|
||||
},
|
||||
])
|
||||
const entries = await collect(new DropboxWalk(accessor('/team')), root())
|
||||
expect(entries.map((e) => e.virtual)).toEqual(['/m/notes.txt'])
|
||||
})
|
||||
|
||||
it('preserves the casing below the root', async () => {
|
||||
vi.mocked(api.listFolder).mockResolvedValue([
|
||||
{
|
||||
'.tag': 'file',
|
||||
id: 'id:1',
|
||||
name: 'Report.TXT',
|
||||
path_display: '/Team/Notes/Report.TXT',
|
||||
path_lower: '/team/notes/report.txt',
|
||||
size: 4,
|
||||
rev: 'r1',
|
||||
},
|
||||
])
|
||||
const entries = await collect(new DropboxWalk(accessor('/team')), root())
|
||||
expect(entries.map((e) => e.virtual)).toEqual(['/m/Notes/Report.TXT'])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,102 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { DropboxAccessor } from '../../accessor/dropbox.ts'
|
||||
import type { PathSpec, WalkEntry } from '../../types.ts'
|
||||
import { mountPrefixOf } from '../../utils/key_prefix.ts'
|
||||
import { stripSlash } from '../../utils/slash.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { statFingerprint } from '../../watch/fingerprint.ts'
|
||||
import { DropboxApiError } from './_client.ts'
|
||||
import { listFolder } from './api.ts'
|
||||
import { dropboxPathOf } from './paths.ts'
|
||||
|
||||
/**
|
||||
* One recursive `list_folder` feeding the generic listing differ.
|
||||
*
|
||||
* Reads the account directly, never through mirage's caches, as the DeltaHook
|
||||
* contract requires.
|
||||
*
|
||||
* Fingerprints on `content_hash`, Dropbox's own content digest, so an upload of
|
||||
* identical bytes is correctly reported as no change; `rev` is the fallback,
|
||||
* and it moves on any write.
|
||||
*
|
||||
* Dropbox also offers a cursor: the same endpoint returns one, and
|
||||
* `list_folder/continue` replays only what changed since. That is a faster
|
||||
* pull, not a more correct one, and it cannot replace this walk, because the
|
||||
* server may invalidate a cursor at any time and the only answer to that is a
|
||||
* full listing. When the fast path is added it belongs behind `pull`, with this
|
||||
* walk as its reset path.
|
||||
*/
|
||||
export class DropboxWalk {
|
||||
private readonly accessor: DropboxAccessor
|
||||
|
||||
constructor(accessor: DropboxAccessor) {
|
||||
this.accessor = accessor
|
||||
}
|
||||
|
||||
async *walk(root: PathSpec): AsyncGenerator<WalkEntry> {
|
||||
const accessor = this.accessor
|
||||
const prefix = mountPrefixOf(root.virtual, root.resourcePath)
|
||||
const apiRoot = dropboxPathOf(accessor, root)
|
||||
let found
|
||||
try {
|
||||
found = await listFolder(accessor.tokenManager, apiRoot, { recursive: true })
|
||||
} catch (error) {
|
||||
// list_folder 409s on a missing path and on a file operand;
|
||||
// either way there is nothing under this root to report.
|
||||
if (error instanceof DropboxApiError && error.status === 409) return
|
||||
throw error
|
||||
}
|
||||
// Dropbox paths are case-insensitive: `path_display` carries the
|
||||
// server's casing while `rootPath` carries the user's, so a configured
|
||||
// `/team` whose displayed path is `/Team` matched nothing and every
|
||||
// event landed outside the watch scope. The comparison folds case; the
|
||||
// slice keeps the server's casing for everything below the root, and is
|
||||
// safe because `path_lower` is `path_display` lowercased, same length.
|
||||
const base = accessor.rootPath
|
||||
const folded = base.toLowerCase()
|
||||
for (const entry of found) {
|
||||
const display = entry.path_display ?? entry.path_lower
|
||||
if (display === undefined || display === '') continue
|
||||
const trimmed =
|
||||
base !== '' && display.toLowerCase().startsWith(folded)
|
||||
? display.slice(base.length)
|
||||
: display
|
||||
const relative = stripSlash(trimmed)
|
||||
if (relative === '') continue
|
||||
const virtual = prefix !== '' ? `${prefix}/${relative}` : `/${relative}`
|
||||
if (entry['.tag'] === 'folder') {
|
||||
yield { virtual, isDir: true, fingerprint: null }
|
||||
continue
|
||||
}
|
||||
const modified = entry.server_modified ?? entry.client_modified ?? null
|
||||
const size = typeof entry.size === 'number' ? entry.size : null
|
||||
const version = entry.content_hash ?? entry.rev ?? null
|
||||
yield {
|
||||
virtual,
|
||||
isDir: false,
|
||||
fingerprint: statFingerprint(version, modified, size),
|
||||
size,
|
||||
modified,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildDeltaHook(accessor: DropboxAccessor): DeltaHook {
|
||||
const walk = new DropboxWalk(accessor)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { GDriveAccessor } from '../../accessor/gdrive.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import type { FileStat, PathSpec } from '../../types.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { ReaddirWalk } from '../../watch/walk.ts'
|
||||
import { readdir } from './readdir.ts'
|
||||
import { stat } from './stat.ts'
|
||||
|
||||
/**
|
||||
* Build the Google Drive delta hook.
|
||||
*
|
||||
* Drive addresses files by id and returns `parents` rather than paths, so a
|
||||
* whole-corpus `files.list` would still have to rebuild the tree before it
|
||||
* could name anything; the walk descends per folder instead, which is the same
|
||||
* shape `find` already uses here.
|
||||
*
|
||||
* Fingerprints on `modifiedTime`, which is what Drive stat reports. Drive also
|
||||
* has `changes.list` with a page token, an account-wide feed that is cheaper
|
||||
* than any walk and would have to be filtered back down to the watch root; that
|
||||
* belongs behind `pull` as a fast path, with this walk as its reset.
|
||||
*/
|
||||
export function buildDeltaHook(accessor: GDriveAccessor): DeltaHook {
|
||||
const walk = new ReaddirWalk(
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
|
||||
)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -63,10 +63,14 @@ function accessorFor(sha: string, probe: Probe): GitHubAccessor {
|
||||
|
||||
async function seeded(sha: string): Promise<RAMIndexCacheStore> {
|
||||
const index = new RAMIndexCacheStore()
|
||||
await populateIndex(index, [
|
||||
{ path: 'src', type: 'tree' as const, sha: 'aaa' },
|
||||
{ path: 'src/main.py', type: 'blob' as const, sha, size: 4 },
|
||||
])
|
||||
await populateIndex(
|
||||
index,
|
||||
{
|
||||
src: { path: 'src', type: 'tree', sha: 'aaa', size: null },
|
||||
'src/main.py': { path: 'src/main.py', type: 'blob', sha, size: 4 },
|
||||
},
|
||||
'',
|
||||
)
|
||||
return index
|
||||
}
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ import { mountPrefixOf } from '../../utils/key_prefix.ts'
|
||||
import type { GitHubAccessor } from '../../accessor/github.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import type { PathSpec } from '../../types.ts'
|
||||
import { refillIndex } from './tree.ts'
|
||||
import { ensureLiveIndex, refillIndex } from './tree.ts'
|
||||
import { fetchBlob } from './_client.ts'
|
||||
import { stripSlash } from '../../utils/slash.ts'
|
||||
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
|
||||
import { eisdir, enoent } from '../../utils/errors.ts'
|
||||
|
||||
function stripPrefix(path: PathSpec): string {
|
||||
@@ -31,11 +31,6 @@ function stripPrefix(path: PathSpec): string {
|
||||
return p
|
||||
}
|
||||
|
||||
function indexKey(p: string): string {
|
||||
const trimmed = stripSlash(p)
|
||||
return trimmed === '' ? '/' : `/${trimmed}`
|
||||
}
|
||||
|
||||
function parentKey(key: string): string {
|
||||
const cut = key.lastIndexOf('/')
|
||||
return cut <= 0 ? '/' : key.slice(0, cut)
|
||||
@@ -46,9 +41,12 @@ export async function read(
|
||||
path: PathSpec,
|
||||
index?: IndexCacheStore,
|
||||
): Promise<Uint8Array> {
|
||||
const prefix = mountPrefixOf(path.virtual, path.resourcePath)
|
||||
const p = stripPrefix(path)
|
||||
if (index === undefined) throw enoent(path)
|
||||
const key = indexKey(p)
|
||||
const rel = stripSlash(p)
|
||||
const key =
|
||||
rel === '' ? (prefix === '' ? '/' : rstripSlash(prefix)) : `${rstripSlash(prefix)}/${rel}`
|
||||
// Freshness is tracked per directory, never per entry, so a blob's row
|
||||
// is exactly as fresh as its parent's listing and `get` can never report
|
||||
// staleness of its own. The parent is therefore the probe: after a write
|
||||
@@ -56,9 +54,10 @@ export async function read(
|
||||
// sha, and reading it back served the old bytes. A miss is not a probe
|
||||
// either -- against a live index it is a real absence, and refetching the
|
||||
// whole tree on every ENOENT costs a recursive-tree call per miss.
|
||||
await ensureLiveIndex(accessor, index, prefix)
|
||||
if (!accessor.truncated) {
|
||||
const parent = await index.listDir(parentKey(key))
|
||||
if (parent.status === LookupStatus.EXPIRED) await refillIndex(accessor, index)
|
||||
if (parent.status === LookupStatus.EXPIRED) await refillIndex(accessor, index, prefix)
|
||||
}
|
||||
const result = await index.get(key)
|
||||
if (result.entry === undefined || result.entry === null) throw enoent(path)
|
||||
|
||||
@@ -42,9 +42,14 @@ function accessorFor(probe: { trees: number }): GitHubAccessor {
|
||||
})
|
||||
}
|
||||
|
||||
const TREE_MAP = {
|
||||
src: { path: 'src', type: 'tree', sha: 'aaa', size: null },
|
||||
'src/main.py': { path: 'src/main.py', type: 'blob', sha: 'bbb', size: 120 },
|
||||
}
|
||||
|
||||
async function seeded(): Promise<RAMIndexCacheStore> {
|
||||
const index = new RAMIndexCacheStore()
|
||||
await populateIndex(index, TREE)
|
||||
await populateIndex(index, TREE_MAP, '')
|
||||
return index
|
||||
}
|
||||
|
||||
|
||||
@@ -18,9 +18,9 @@ import { LookupStatus } from '../../cache/index/config.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import type { PathSpec } from '../../types.ts'
|
||||
import { fetchDirTree } from './_client.ts'
|
||||
import { refillIndex } from './tree.ts'
|
||||
import { ensureLiveIndex, refillIndex } from './tree.ts'
|
||||
import { IndexEntry } from '../../cache/index/config.ts'
|
||||
import { stripSlash } from '../../utils/slash.ts'
|
||||
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
|
||||
import { enoent } from '../../utils/errors.ts'
|
||||
import { compareCodePoints } from '../../utils/sort.ts'
|
||||
|
||||
@@ -33,11 +33,6 @@ function stripPrefix(path: PathSpec): string {
|
||||
return p
|
||||
}
|
||||
|
||||
function normalizeKey(p: string): string {
|
||||
const trimmed = stripSlash(p)
|
||||
return trimmed === '' ? '/' : `/${trimmed}`
|
||||
}
|
||||
|
||||
export async function readdir(
|
||||
accessor: GitHubAccessor,
|
||||
path: PathSpec,
|
||||
@@ -47,9 +42,11 @@ export async function readdir(
|
||||
throw enoent(path.virtual)
|
||||
}
|
||||
const prefix = mountPrefixOf(path.virtual, path.resourcePath)
|
||||
const stripped = stripPrefix(path)
|
||||
const key = normalizeKey(stripped)
|
||||
const rel = stripSlash(stripPrefix(path))
|
||||
const key =
|
||||
rel === '' ? (prefix === '' ? '/' : rstripSlash(prefix)) : `${rstripSlash(prefix)}/${rel}`
|
||||
|
||||
await ensureLiveIndex(accessor, index, prefix)
|
||||
let listing = await index.listDir(key)
|
||||
// The index is the whole listing here, not a cache in front of one, so an
|
||||
// *expired* answer means the tree aged out, not that the path is gone.
|
||||
@@ -57,12 +54,10 @@ export async function readdir(
|
||||
// absence and must not cost a tree fetch: refilling on any miss spends a
|
||||
// recursive-tree call on every ENOENT.
|
||||
if (listing.status === LookupStatus.EXPIRED && !accessor.truncated) {
|
||||
if (await refillIndex(accessor, index)) listing = await index.listDir(key)
|
||||
if (await refillIndex(accessor, index, prefix)) listing = await index.listDir(key)
|
||||
}
|
||||
if (listing.entries !== undefined && listing.entries !== null) {
|
||||
return prefix !== '' && listing.entries.length > 0 && !listing.entries[0]?.startsWith(prefix)
|
||||
? listing.entries.map((e) => prefix + e)
|
||||
: listing.entries
|
||||
return listing.entries
|
||||
}
|
||||
if (listing.status === LookupStatus.NOT_FOUND) {
|
||||
if (accessor.truncated) {
|
||||
@@ -79,7 +74,7 @@ async function fallbackReaddir(
|
||||
index: IndexCacheStore,
|
||||
prefix: string,
|
||||
): Promise<string[]> {
|
||||
const parentSha = await resolveDirSha(accessor, key, index)
|
||||
const parentSha = await resolveDirSha(accessor, key, index, prefix)
|
||||
if (parentSha === null) throw enoent(`${prefix}/${key}`)
|
||||
const entries = await fetchDirTree(accessor.transport, accessor.owner, accessor.repo, parentSha)
|
||||
const childKeys: string[] = []
|
||||
@@ -102,23 +97,26 @@ async function fallbackReaddir(
|
||||
}
|
||||
childKeys.sort(compareCodePoints)
|
||||
await index.setDir(key, childEntries)
|
||||
return childKeys.map((k) => (prefix !== '' ? prefix + k : k))
|
||||
return childKeys
|
||||
}
|
||||
|
||||
async function resolveDirSha(
|
||||
accessor: GitHubAccessor,
|
||||
key: string,
|
||||
index: IndexCacheStore,
|
||||
prefix: string,
|
||||
): Promise<string | null> {
|
||||
const result = await index.get(key)
|
||||
if (result.entry !== undefined && result.entry !== null) {
|
||||
return result.entry.id
|
||||
}
|
||||
const parts = stripSlash(key)
|
||||
const stem = rstripSlash(prefix)
|
||||
const rest = stem !== '' && key.startsWith(stem) ? key.slice(stem.length) : key
|
||||
const parts = stripSlash(rest)
|
||||
.split('/')
|
||||
.filter((p) => p !== '')
|
||||
let currentSha = accessor.ref
|
||||
let currentPath = ''
|
||||
let currentPath = stem
|
||||
for (const part of parts) {
|
||||
const entries = await fetchDirTree(
|
||||
accessor.transport,
|
||||
|
||||
@@ -18,7 +18,7 @@ import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import { FileStat, FileType, PathSpec } from '../../types.ts'
|
||||
import { getExtension } from '../../commands/resolve.ts'
|
||||
import { readdir as coreReaddir } from './readdir.ts'
|
||||
import { stripSlash } from '../../utils/slash.ts'
|
||||
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
|
||||
import { enoent } from '../../utils/errors.ts'
|
||||
|
||||
function stripPrefix(path: PathSpec): string {
|
||||
@@ -30,11 +30,6 @@ function stripPrefix(path: PathSpec): string {
|
||||
return p
|
||||
}
|
||||
|
||||
function indexKey(p: string): string {
|
||||
const trimmed = stripSlash(p)
|
||||
return trimmed === '' ? '/' : `/${trimmed}`
|
||||
}
|
||||
|
||||
function guessFileType(name: string): FileType {
|
||||
const ext = getExtension(name)
|
||||
if (ext === 'json') return FileType.JSON
|
||||
@@ -54,11 +49,15 @@ export async function stat(
|
||||
return new FileStat({ name: '/', type: FileType.DIRECTORY })
|
||||
}
|
||||
if (index === undefined) throw enoent(path)
|
||||
const ikey = indexKey(p)
|
||||
const ikey = `${rstripSlash(prefix)}/${trimmed}`
|
||||
let result = await index.get(ikey)
|
||||
if (result.entry === undefined || result.entry === null) {
|
||||
const parentIdx = ikey.includes('/') ? ikey.slice(0, ikey.lastIndexOf('/')) || '/' : '/'
|
||||
const parentPath = prefix !== '' ? prefix + parentIdx : parentIdx
|
||||
// `ikey` is already mount-absolute, so its parent is too: prepending
|
||||
// the prefix again asks for `/repo/repo`, whose listing never populates
|
||||
// the entry this is here to find. stat then reports ENOENT for a file
|
||||
// that exists, and the read family's implicit-directory probe finds it
|
||||
// in the parent listing and answers EISDIR instead.
|
||||
const parentPath = ikey.includes('/') ? ikey.slice(0, ikey.lastIndexOf('/')) || '/' : '/'
|
||||
try {
|
||||
await coreReaddir(
|
||||
accessor,
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { fetchDirTree, fetchTree, type GitHubTransport } from './_client.ts'
|
||||
import { GitHubAccessor } from '../../accessor/github.ts'
|
||||
import { RAMIndexCacheStore } from '../../cache/index/ram.ts'
|
||||
import { ensureLiveIndex, populateIndex } from './tree.ts'
|
||||
|
||||
const ITEMS = [
|
||||
{ path: 'extern', mode: '160000', type: 'commit', sha: 'ccc' },
|
||||
@@ -39,3 +42,107 @@ describe('github tree fetch', () => {
|
||||
expect(entries.map((e) => e.path)).toEqual(['main.py', 'src'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('ensureLiveIndex', () => {
|
||||
const TREE = {
|
||||
data: { path: 'data', type: 'tree', sha: 't1', size: null },
|
||||
'data/keep.txt': { path: 'data/keep.txt', type: 'blob', sha: 'b1', size: 4 },
|
||||
}
|
||||
|
||||
function accessor(calls: { n: number }): GitHubAccessor {
|
||||
return new GitHubAccessor({
|
||||
transport: {
|
||||
get: () => {
|
||||
calls.n += 1
|
||||
return Promise.resolve({
|
||||
tree: [
|
||||
{ path: 'data', type: 'tree' as const, sha: 't1' },
|
||||
{ path: 'data/keep.txt', type: 'blob' as const, sha: 'b1', size: 4 },
|
||||
],
|
||||
truncated: false,
|
||||
})
|
||||
},
|
||||
} as unknown as GitHubTransport,
|
||||
owner: 'acme',
|
||||
repo: 'proj',
|
||||
ref: 'main',
|
||||
defaultBranch: 'main',
|
||||
tree: TREE,
|
||||
})
|
||||
}
|
||||
|
||||
it('refetches rather than reuse the build-time tree', async () => {
|
||||
// The build tree is only true at build time: a mount's first read can
|
||||
// come long after it, so reusing it would key an index built from a
|
||||
// repository several external writes ago.
|
||||
const calls = { n: 0 }
|
||||
const index = new RAMIndexCacheStore({ ttl: 600 })
|
||||
expect(await ensureLiveIndex(accessor(calls), index, '/gh')).toBe(true)
|
||||
expect(calls.n).toBe(1)
|
||||
expect((await index.listDir('/gh/data')).entries).toEqual(['/gh/data/keep.txt'])
|
||||
})
|
||||
|
||||
it('refetches a dropped listing', async () => {
|
||||
const calls = { n: 0 }
|
||||
const acc = accessor(calls)
|
||||
const index = new RAMIndexCacheStore({ ttl: 600 })
|
||||
await ensureLiveIndex(acc, index, '/gh')
|
||||
// What invalidation does: drop the row rather than expire it, which is
|
||||
// why the readers' EXPIRED probe never fires.
|
||||
await index.invalidateDir('/gh')
|
||||
await index.invalidateDir('/gh/data')
|
||||
expect(await ensureLiveIndex(acc, index, '/gh')).toBe(true)
|
||||
expect(calls.n).toBe(2)
|
||||
expect((await index.listDir('/gh/data')).entries).toEqual(['/gh/data/keep.txt'])
|
||||
})
|
||||
|
||||
it('leaves a live index alone and sends no request', async () => {
|
||||
const calls = { n: 0 }
|
||||
const acc = accessor(calls)
|
||||
const index = new RAMIndexCacheStore({ ttl: 600 })
|
||||
await ensureLiveIndex(acc, index, '/gh')
|
||||
const before = calls.n
|
||||
expect(await ensureLiveIndex(acc, index, '/gh')).toBe(false)
|
||||
expect(calls.n).toBe(before)
|
||||
})
|
||||
|
||||
it('skips a truncated tree', async () => {
|
||||
const calls = { n: 0 }
|
||||
const acc = accessor(calls)
|
||||
acc.truncated = true
|
||||
expect(await ensureLiveIndex(acc, new RAMIndexCacheStore({ ttl: 600 }), '/gh')).toBe(false)
|
||||
expect(calls.n).toBe(0)
|
||||
})
|
||||
|
||||
it('skips a missing index', async () => {
|
||||
expect(await ensureLiveIndex(accessor({ n: 0 }), undefined, '')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('populateIndex', () => {
|
||||
const TREE = {
|
||||
data: { path: 'data', type: 'tree', sha: 't1', size: null },
|
||||
'data/keep.txt': { path: 'data/keep.txt', type: 'blob', sha: 'b1', size: 4 },
|
||||
}
|
||||
|
||||
it('keys by mount-absolute path', async () => {
|
||||
// Every other backend keys its index this way, which is what lets the
|
||||
// shared CacheManager spell an eviction without knowing the backend.
|
||||
const index = new RAMIndexCacheStore({ ttl: 600 })
|
||||
await populateIndex(index, TREE, '/gh')
|
||||
expect((await index.listDir('/gh')).entries).toEqual(['/gh/data'])
|
||||
expect((await index.listDir('/gh/data')).entries).toEqual(['/gh/data/keep.txt'])
|
||||
})
|
||||
|
||||
it('keeps bare paths on a root mount', async () => {
|
||||
const index = new RAMIndexCacheStore({ ttl: 600 })
|
||||
await populateIndex(index, TREE, '')
|
||||
expect((await index.listDir('/')).entries).toEqual(['/data'])
|
||||
})
|
||||
|
||||
it('gives an empty repo a root row', async () => {
|
||||
const index = new RAMIndexCacheStore({ ttl: 600 })
|
||||
await populateIndex(index, {}, '/gh')
|
||||
expect((await index.listDir('/gh')).entries).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -15,9 +15,11 @@
|
||||
import type { GitHubAccessor } from '../../accessor/github.ts'
|
||||
import { fetchTree } from './_client.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import { LookupStatus } from '../../cache/index/config.ts'
|
||||
import type { IndexEntry } from '../../cache/index/config.ts'
|
||||
import type { GitHubTreeItem } from './_client.ts'
|
||||
import { indexEntryFromTree, makeTreeEntry, type TreeEntry } from './tree_entry.ts'
|
||||
import { rstripSlash } from '../../utils/slash.ts'
|
||||
|
||||
export function buildTreeMap(tree: GitHubTreeItem[]): Record<string, TreeEntry> {
|
||||
const map: Record<string, TreeEntry> = {}
|
||||
@@ -25,12 +27,26 @@ export function buildTreeMap(tree: GitHubTreeItem[]): Record<string, TreeEntry>
|
||||
return map
|
||||
}
|
||||
|
||||
export async function populateIndex(index: IndexCacheStore, tree: GitHubTreeItem[]): Promise<void> {
|
||||
export async function populateIndex(
|
||||
index: IndexCacheStore,
|
||||
tree: Record<string, TreeEntry>,
|
||||
prefix: string,
|
||||
): Promise<void> {
|
||||
// Keyed by mount-absolute path, the way every other backend keys its
|
||||
// index, so the shared cache machinery can spell an eviction without
|
||||
// knowing which backend it is talking to. The tree itself stays
|
||||
// repo-relative; `prefix` is what lifts it.
|
||||
const stem = rstripSlash(prefix)
|
||||
const dirs = new Map<string, [string, IndexEntry][]>()
|
||||
for (const item of tree) {
|
||||
// The repository root always exists, so it gets a row even when the tree
|
||||
// is empty. Without it an empty repository is byte for byte a dropped
|
||||
// index, and `ensureLiveIndex` would refetch on every read of one.
|
||||
dirs.set(stem === '' ? '/' : stem, [])
|
||||
for (const item of Object.values(tree)) {
|
||||
const parts = item.path.split('/')
|
||||
const name = parts[parts.length - 1] ?? item.path
|
||||
const parent = parts.length > 1 ? `/${parts.slice(0, -1).join('/')}` : '/'
|
||||
const parent =
|
||||
parts.length > 1 ? `${stem}/${parts.slice(0, -1).join('/')}` : stem === '' ? '/' : stem
|
||||
const arr = dirs.get(parent) ?? []
|
||||
arr.push([name, indexEntryFromTree(item)])
|
||||
dirs.set(parent, arr)
|
||||
@@ -38,6 +54,19 @@ export async function populateIndex(index: IndexCacheStore, tree: GitHubTreeItem
|
||||
await Promise.all([...dirs].map(([parent, entries]) => index.setDir(parent, entries)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the accessor's tree into `index` under `prefix`.
|
||||
*
|
||||
* Mirrors Python's `seed_index`.
|
||||
*/
|
||||
async function seedIndex(
|
||||
accessor: GitHubAccessor,
|
||||
index: IndexCacheStore,
|
||||
prefix: string,
|
||||
): Promise<void> {
|
||||
await populateIndex(index, accessor.tree, prefix)
|
||||
}
|
||||
|
||||
/**
|
||||
* Refetch the recursive tree and re-seed the index from it.
|
||||
*
|
||||
@@ -61,6 +90,7 @@ export async function populateIndex(index: IndexCacheStore, tree: GitHubTreeItem
|
||||
export async function refillIndex(
|
||||
accessor: GitHubAccessor,
|
||||
index: IndexCacheStore | undefined,
|
||||
prefix: string,
|
||||
): Promise<boolean> {
|
||||
if (index === undefined) return false
|
||||
const { tree, truncated } = await fetchTree(
|
||||
@@ -71,6 +101,57 @@ export async function refillIndex(
|
||||
)
|
||||
accessor.truncated = truncated
|
||||
accessor.tree = buildTreeMap(tree)
|
||||
await populateIndex(index, tree)
|
||||
await seedIndex(accessor, index, prefix)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Refetch when the index holds no listing at all.
|
||||
*
|
||||
* Every reader here treats a missing listing as a real absence, which is
|
||||
* right against a *live* index and wrong against one that was never filled
|
||||
* or has been dropped, and invalidation drops rather than expires:
|
||||
* `invalidateDir` removes the directory's row outright, so the EXPIRED
|
||||
* probe each reader already runs never fires. An external change (a watch
|
||||
* event is the only thing that invalidates a mount with no write ops)
|
||||
* therefore left the whole mount answering ENOENT permanently, since the
|
||||
* seeded expiry is a year out.
|
||||
*
|
||||
* The root listing is what tells live from not, in one lookup and no
|
||||
* request: the tree is written whole, so while the index is live every
|
||||
* directory has a row and the mount root always does. One refill makes it
|
||||
* live again, so this cannot cost a fetch per miss, which is what kept the
|
||||
* readers from probing on absence in the first place.
|
||||
*
|
||||
* Not live always **refetches**, and never re-seeds the tree the mount was
|
||||
* built with. That tree is only true at build time: the first read of a
|
||||
* mount can come long after it, and reusing it then served an index built
|
||||
* from a repository five external writes ago. It is still what
|
||||
* `accessor.tree` starts as, so find and du have something to read before
|
||||
* any listing happens, and every refill reseats it.
|
||||
*
|
||||
* Mirrors Python's `ensure_live_index`.
|
||||
*
|
||||
* Args:
|
||||
* accessor (GitHubAccessor): the mount's accessor.
|
||||
* index (IndexCacheStore | undefined): the index to check and fill.
|
||||
* prefix (string): the mount prefix the index keys are built against.
|
||||
*
|
||||
* Returns:
|
||||
* boolean: whether the index was filled.
|
||||
*/
|
||||
export async function ensureLiveIndex(
|
||||
accessor: GitHubAccessor,
|
||||
index: IndexCacheStore | undefined,
|
||||
prefix: string,
|
||||
): Promise<boolean> {
|
||||
if (index === undefined) return false
|
||||
// The liveness probe comes before anything on the accessor, so a live
|
||||
// index still answers every read without one.
|
||||
const root = rstripSlash(prefix) === '' ? '/' : rstripSlash(prefix)
|
||||
if ((await index.listDir(root)).status !== LookupStatus.NOT_FOUND) return false
|
||||
// A truncated tree is not the whole listing, so the invariant this rests
|
||||
// on does not hold and readdir's per-directory fallback owns the miss.
|
||||
if (accessor.truncated) return false
|
||||
return refillIndex(accessor, index, prefix)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,12 @@ export function makeTreeEntry(item: GitHubTreeItem): TreeEntry {
|
||||
}
|
||||
}
|
||||
|
||||
export function indexEntryFromTree(item: GitHubTreeItem): IndexEntry {
|
||||
export function indexEntryFromTree(item: {
|
||||
path: string
|
||||
type: string
|
||||
sha: string
|
||||
size?: number | null
|
||||
}): IndexEntry {
|
||||
const parts = item.path.split('/')
|
||||
const name = parts[parts.length - 1] ?? item.path
|
||||
return new IndexEntry({
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { GitHubAccessor } from '../../accessor/github.ts'
|
||||
import { PathSpec, type WalkEntry } from '../../types.ts'
|
||||
import { IncompleteWalkError } from '../../watch/errors.ts'
|
||||
import type { GitHubTransport } from './_client.ts'
|
||||
import type { TreeEntry } from './tree_entry.ts'
|
||||
import { GitHubWalk } from './watch.ts'
|
||||
|
||||
interface TreeItem {
|
||||
path: string
|
||||
type: string
|
||||
sha: string
|
||||
size?: number
|
||||
}
|
||||
|
||||
function transport(tree: TreeItem[], truncated = false): GitHubTransport {
|
||||
return {
|
||||
get: () => Promise.resolve({ tree, truncated }),
|
||||
request: () => Promise.resolve({}),
|
||||
}
|
||||
}
|
||||
|
||||
function accessor(tree: TreeItem[], stale: Record<string, TreeEntry>, truncated = false) {
|
||||
return new GitHubAccessor({
|
||||
transport: transport(tree, truncated),
|
||||
owner: 'acme',
|
||||
repo: 'proj',
|
||||
ref: 'main',
|
||||
defaultBranch: 'main',
|
||||
tree: stale,
|
||||
})
|
||||
}
|
||||
|
||||
function root(): PathSpec {
|
||||
return new PathSpec({ virtual: '/gh', directory: '/gh', resourcePath: '' })
|
||||
}
|
||||
|
||||
async function collect(walk: GitHubWalk, at: PathSpec): Promise<WalkEntry[]> {
|
||||
const out: WalkEntry[] = []
|
||||
for await (const entry of walk.walk(at)) out.push(entry)
|
||||
return out
|
||||
}
|
||||
|
||||
const STALE: Record<string, TreeEntry> = {
|
||||
'a.txt': { path: 'a.txt', type: 'blob', sha: 'sha-a', size: 3 },
|
||||
}
|
||||
|
||||
describe('GitHubWalk', () => {
|
||||
it('refreshes the accessor tree from the pull', async () => {
|
||||
// find, du and grep's scope counter read accessor.tree directly, so a
|
||||
// pull that fetched a newer tree and dropped it left them reporting
|
||||
// the repository as it stood when the mount was built.
|
||||
const acc = accessor(
|
||||
[
|
||||
{ path: 'a.txt', type: 'blob', sha: 'sha-a', size: 3 },
|
||||
{ path: 'b.txt', type: 'blob', sha: 'sha-b', size: 3 },
|
||||
],
|
||||
STALE,
|
||||
)
|
||||
await collect(new GitHubWalk(acc), root())
|
||||
expect(Object.keys(acc.tree).sort()).toEqual(['a.txt', 'b.txt'])
|
||||
expect(acc.tree['b.txt']?.sha).toBe('sha-b')
|
||||
})
|
||||
|
||||
it('does not adopt a truncated tree', async () => {
|
||||
// A partial tree would make find report the missing half as deleted.
|
||||
const acc = accessor([], STALE, true)
|
||||
await expect(collect(new GitHubWalk(acc), root())).rejects.toThrow(IncompleteWalkError)
|
||||
expect(Object.keys(acc.tree)).toEqual(['a.txt'])
|
||||
})
|
||||
|
||||
it('reports blobs with their sha as the fingerprint', async () => {
|
||||
const acc = accessor([{ path: 'a.txt', type: 'blob', sha: 'sha-a', size: 3 }], STALE)
|
||||
const entries = await collect(new GitHubWalk(acc), root())
|
||||
expect(entries.map((e) => [e.virtual, e.fingerprint])).toEqual([['/gh/a.txt', 'sha-a']])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,88 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { GitHubAccessor } from '../../accessor/github.ts'
|
||||
import type { PathSpec, WalkEntry } from '../../types.ts'
|
||||
import { mountPrefixOf } from '../../utils/key_prefix.ts'
|
||||
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { IncompleteWalkError } from '../../watch/errors.ts'
|
||||
import { fetchTree } from './_client.ts'
|
||||
import { buildTreeMap } from './tree.ts'
|
||||
|
||||
/**
|
||||
* One recursive git tree fetch feeding the generic listing differ.
|
||||
*
|
||||
* `GET /git/trees/{ref}?recursive=1` returns every path in the repository with
|
||||
* its object sha, so a pull is one request whatever the repository's shape, and
|
||||
* the fingerprint is the sha itself. That is the strongest fingerprint any
|
||||
* mirage backend has: git is content-addressed, so identical bytes have an
|
||||
* identical sha and a rewrite that changes nothing correctly reports nothing.
|
||||
*
|
||||
* A mount is pinned to one ref, so what this detects is that ref moving.
|
||||
* Nothing is reported while the branch sits still, however much is pushed
|
||||
* elsewhere in the repository.
|
||||
*/
|
||||
export class GitHubWalk {
|
||||
private readonly accessor: GitHubAccessor
|
||||
|
||||
constructor(accessor: GitHubAccessor) {
|
||||
this.accessor = accessor
|
||||
}
|
||||
|
||||
async *walk(root: PathSpec): AsyncGenerator<WalkEntry> {
|
||||
const accessor = this.accessor
|
||||
const prefix = mountPrefixOf(root.virtual, root.resourcePath)
|
||||
const { tree, truncated } = await fetchTree(
|
||||
accessor.transport,
|
||||
accessor.owner,
|
||||
accessor.repo,
|
||||
accessor.ref,
|
||||
)
|
||||
if (truncated) {
|
||||
throw new IncompleteWalkError(
|
||||
`github tree for ${accessor.owner}/${accessor.repo}@${accessor.ref} was truncated; ` +
|
||||
'cannot diff a partial tree',
|
||||
)
|
||||
}
|
||||
// A complete tree for the ref is exactly what the accessor holds, and
|
||||
// find/du/grep's scope counter read it directly. Discarding it here
|
||||
// left them answering from the tree the mount was built with until an
|
||||
// unrelated read happened to refill the index, so a pull that reported
|
||||
// a CREATE was followed by a find that could not see the file.
|
||||
accessor.tree = buildTreeMap(tree)
|
||||
const stem = stripSlash(rstripSlash(root.resourcePath))
|
||||
const base = stem !== '' ? `${stem}/` : ''
|
||||
for (const item of tree) {
|
||||
if (base !== '' && !item.path.startsWith(base)) continue
|
||||
const virtual = prefix !== '' ? `${prefix}/${item.path}` : `/${item.path}`
|
||||
if (item.type === 'tree') {
|
||||
yield { virtual, isDir: true, fingerprint: null }
|
||||
continue
|
||||
}
|
||||
yield {
|
||||
virtual,
|
||||
isDir: false,
|
||||
fingerprint: item.sha,
|
||||
size: item.size ?? null,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildDeltaHook(accessor: GitHubAccessor): DeltaHook {
|
||||
const walk = new GitHubWalk(accessor)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { OneDriveAccessor } from '../../accessor/onedrive.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import type { FileStat, PathSpec } from '../../types.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { ReaddirWalk } from '../../watch/walk.ts'
|
||||
import { readdir, stat } from './index.ts'
|
||||
|
||||
/**
|
||||
* Build the OneDrive delta hook.
|
||||
*
|
||||
* Fingerprints on the item's `cTag`, which Graph moves only when the content
|
||||
* changes (`eTag` also moves on a metadata edit), so a rename does not read as
|
||||
* a content change.
|
||||
*
|
||||
* Graph has a native `/delta` feed with a resumable token, which is cheaper
|
||||
* than this walk and reports deletes directly. It is a fast path rather than a
|
||||
* replacement: Graph can answer `resyncRequired` at any time, and the only
|
||||
* response to that is a full listing. When it is added it belongs behind
|
||||
* `pull`, with this walk as its reset.
|
||||
*/
|
||||
export function buildDeltaHook(accessor: OneDriveAccessor): DeltaHook {
|
||||
const walk = new ReaddirWalk(
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
|
||||
)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as ClientModule from './_client.ts'
|
||||
|
||||
vi.mock('./_client.ts', async () => {
|
||||
const actual = await vi.importActual<typeof ClientModule>('./_client.ts')
|
||||
return { ...actual, loadS3Module: vi.fn(), withClient: vi.fn() }
|
||||
})
|
||||
|
||||
import { S3Accessor } from '../../accessor/s3.ts'
|
||||
import type { S3Config } from '../../resource/s3/config.ts'
|
||||
import { FileChangeKind, PathSpec, type WalkEntry } from '../../types.ts'
|
||||
import * as clientMod from './_client.ts'
|
||||
import { buildDeltaHook, S3Walk } from './watch.ts'
|
||||
|
||||
class FakeCommand {
|
||||
constructor(readonly input: unknown) {}
|
||||
}
|
||||
|
||||
interface StoredObject {
|
||||
key: string
|
||||
size: number
|
||||
etag: string
|
||||
}
|
||||
|
||||
function mockListing(objects: StoredObject[]): void {
|
||||
vi.mocked(clientMod.loadS3Module).mockResolvedValue({
|
||||
ListObjectsV2Command: FakeCommand,
|
||||
} as never)
|
||||
vi.mocked(clientMod.withClient).mockImplementation(async (_config, fn) => {
|
||||
const client = {
|
||||
send: () =>
|
||||
Promise.resolve({
|
||||
Contents: objects.map((obj) => ({
|
||||
Key: obj.key,
|
||||
Size: obj.size,
|
||||
LastModified: new Date('2026-03-31T00:00:00.000Z'),
|
||||
ETag: `"${obj.etag}"`,
|
||||
})),
|
||||
IsTruncated: false,
|
||||
}),
|
||||
}
|
||||
return (await fn(client as never)) as never
|
||||
})
|
||||
}
|
||||
|
||||
function accessor(keyPrefix?: string): S3Accessor {
|
||||
return new S3Accessor({
|
||||
bucket: 'watch-bucket',
|
||||
region: 'us-east-1',
|
||||
keyPrefix,
|
||||
} as S3Config)
|
||||
}
|
||||
|
||||
function root(virtual: string, resourcePath: string): PathSpec {
|
||||
return new PathSpec({ virtual, directory: virtual, resourcePath })
|
||||
}
|
||||
|
||||
async function collect(walk: S3Walk, spec: PathSpec): Promise<WalkEntry[]> {
|
||||
const out: WalkEntry[] = []
|
||||
for await (const entry of walk.walk(spec)) out.push(entry)
|
||||
return out
|
||||
}
|
||||
|
||||
describe('S3Walk', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('yields files fingerprinted on the ETag', async () => {
|
||||
mockListing([
|
||||
{ key: 'data/a.txt', size: 5, etag: 'etag-a' },
|
||||
{ key: 'data/b.txt', size: 4, etag: 'etag-b' },
|
||||
])
|
||||
const entries = await collect(new S3Walk(accessor()), root('/s3/data', 'data'))
|
||||
const files = entries.filter((e) => !e.isDir)
|
||||
expect(files.map((e) => e.virtual).sort()).toEqual(['/s3/data/a.txt', '/s3/data/b.txt'])
|
||||
// The ETag, not the mtime|size composite: LastModified is constant
|
||||
// here, so a composite would collide across files of equal size.
|
||||
expect(files[0]?.fingerprint).toBe('etag-a')
|
||||
expect(files[0]?.size).toBe(5)
|
||||
})
|
||||
|
||||
it('synthesizes intermediate directories', async () => {
|
||||
mockListing([{ key: 'data/sub/deep/x.txt', size: 1, etag: 'etag-x' }])
|
||||
const entries = await collect(new S3Walk(accessor()), root('/s3/data', 'data'))
|
||||
expect(entries.filter((e) => e.isDir).map((e) => e.virtual)).toEqual([
|
||||
'/s3/data/sub/deep',
|
||||
'/s3/data/sub',
|
||||
])
|
||||
})
|
||||
|
||||
it('reports an explicit marker as its own directory', async () => {
|
||||
mockListing([{ key: 'data/empty/', size: 0, etag: 'etag-marker' }])
|
||||
const entries = await collect(new S3Walk(accessor()), root('/s3/data', 'data'))
|
||||
expect(entries.filter((e) => e.isDir).map((e) => e.virtual)).toEqual(['/s3/data/empty'])
|
||||
expect(entries.filter((e) => !e.isDir)).toEqual([])
|
||||
})
|
||||
|
||||
it('strips the key prefix', async () => {
|
||||
mockListing([{ key: 'team/x/data/a.txt', size: 5, etag: 'etag-a' }])
|
||||
const entries = await collect(new S3Walk(accessor('team/x/')), root('/s3/data', 'data'))
|
||||
expect(entries.filter((e) => !e.isDir).map((e) => e.virtual)).toEqual(['/s3/data/a.txt'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('s3 delta hook', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('reports nothing on the baseline pull, then create and update', async () => {
|
||||
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
|
||||
const hook = buildDeltaHook(accessor())
|
||||
const spec = root('/s3/data', 'data')
|
||||
const first = await hook.pull(spec, null)
|
||||
expect(first.changes).toEqual([])
|
||||
|
||||
mockListing([
|
||||
{ key: 'data/a.txt', size: 5, etag: 'etag-a2' },
|
||||
{ key: 'data/new.txt', size: 3, etag: 'etag-new' },
|
||||
])
|
||||
const second = await hook.pull(spec, first.checkpoint)
|
||||
expect(Object.fromEntries(second.changes.map((c) => [c.path.virtual, c.kind]))).toEqual({
|
||||
'/s3/data/a.txt': FileChangeKind.UPDATE,
|
||||
'/s3/data/new.txt': FileChangeKind.CREATE,
|
||||
})
|
||||
})
|
||||
|
||||
it('detects a delete', async () => {
|
||||
mockListing([
|
||||
{ key: 'data/a.txt', size: 5, etag: 'etag-a' },
|
||||
{ key: 'data/b.txt', size: 4, etag: 'etag-b' },
|
||||
])
|
||||
const hook = buildDeltaHook(accessor())
|
||||
const spec = root('/s3/data', 'data')
|
||||
const first = await hook.pull(spec, null)
|
||||
|
||||
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
|
||||
const second = await hook.pull(spec, first.checkpoint)
|
||||
expect(second.changes.map((c) => [c.kind, c.path.virtual])).toEqual([
|
||||
[FileChangeKind.DELETE, '/s3/data/b.txt'],
|
||||
])
|
||||
})
|
||||
|
||||
it('treats a rewrite of identical bytes as no change', async () => {
|
||||
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
|
||||
const hook = buildDeltaHook(accessor())
|
||||
const spec = root('/s3/data', 'data')
|
||||
const first = await hook.pull(spec, null)
|
||||
const second = await hook.pull(spec, first.checkpoint)
|
||||
expect(second.changes).toEqual([])
|
||||
})
|
||||
|
||||
it('frames a changed path against the mount', async () => {
|
||||
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
|
||||
const hook = buildDeltaHook(accessor())
|
||||
const spec = root('/s3/data', 'data')
|
||||
const first = await hook.pull(spec, null)
|
||||
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a2' }])
|
||||
const second = await hook.pull(spec, first.checkpoint)
|
||||
expect(second.changes[0]?.path.virtual).toBe('/s3/data/a.txt')
|
||||
expect(second.changes[0]?.path.resourcePath).toBe('data/a.txt')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,113 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { S3Accessor } from '../../accessor/s3.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { statFingerprint } from '../../watch/fingerprint.ts'
|
||||
import { synthDirs } from '../../watch/walk.ts'
|
||||
import type { PathSpec, WalkEntry } from '../../types.ts'
|
||||
import { mountPrefixOf } from '../../utils/key_prefix.ts'
|
||||
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
|
||||
import { loadS3Module, rawPathOf, s3Key, stripKeyPrefix, withClient } from './_client.ts'
|
||||
|
||||
interface ListedObject {
|
||||
Key?: string
|
||||
Size?: number
|
||||
LastModified?: Date | string
|
||||
ETag?: string
|
||||
}
|
||||
|
||||
function isoOf(value: Date | string | undefined): string | null {
|
||||
if (value === undefined) return null
|
||||
return value instanceof Date ? value.toISOString() : value
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursive `ListObjectsV2` feeding the generic listing differ.
|
||||
*
|
||||
* One paginated LIST with no Delimiter covers the whole subtree, so a pull
|
||||
* costs one request per 1000 keys rather than one per directory. Reads the
|
||||
* bucket directly, never through mirage's caches, as the DeltaHook contract
|
||||
* requires.
|
||||
*
|
||||
* Fingerprints on the object's ETag, which for a single-part upload is the MD5
|
||||
* of the content, so an overwrite with identical bytes is correctly reported
|
||||
* as no change. Multipart ETags are a digest of the part digests, which still
|
||||
* changes with the content.
|
||||
*/
|
||||
export class S3Walk {
|
||||
private readonly accessor: S3Accessor
|
||||
|
||||
constructor(accessor: S3Accessor) {
|
||||
this.accessor = accessor
|
||||
}
|
||||
|
||||
async *walk(root: PathSpec): AsyncGenerator<WalkEntry> {
|
||||
const config = this.accessor.config
|
||||
const prefix = mountPrefixOf(root.virtual, root.resourcePath)
|
||||
const stem = rstripSlash(s3Key(rawPathOf(root), config))
|
||||
const base = stem !== '' ? `${stem}/` : ''
|
||||
const files: string[] = []
|
||||
const markers: string[] = []
|
||||
const rows: WalkEntry[] = []
|
||||
const { ListObjectsV2Command } = await loadS3Module(config)
|
||||
await withClient(config, async (client) => {
|
||||
let continuationToken: string | undefined
|
||||
do {
|
||||
const input: Record<string, unknown> = { Bucket: config.bucket, Prefix: stem }
|
||||
if (continuationToken !== undefined) input.ContinuationToken = continuationToken
|
||||
const resp = (await client.send(new ListObjectsV2Command(input))) as {
|
||||
Contents?: ListedObject[]
|
||||
IsTruncated?: boolean
|
||||
NextContinuationToken?: string
|
||||
}
|
||||
for (const obj of resp.Contents ?? []) {
|
||||
const key = obj.Key
|
||||
if (key === undefined) continue
|
||||
if (!(key === stem || key.startsWith(base))) continue
|
||||
const relative = stripSlash(stripKeyPrefix(key, config))
|
||||
const virtual = prefix !== '' ? `${prefix}/${relative}` : `/${relative}`
|
||||
if (key.endsWith('/')) {
|
||||
// A directory marker: mirage's own mkdir writes one. It
|
||||
// carries an ETag and a size, but it is not a file, so
|
||||
// synthDirs reports it instead.
|
||||
markers.push(rstripSlash(virtual))
|
||||
continue
|
||||
}
|
||||
files.push(virtual)
|
||||
const modified = isoOf(obj.LastModified)
|
||||
const size = obj.Size ?? null
|
||||
const etag = (obj.ETag ?? '').replace(/^"|"$/g, '') || null
|
||||
rows.push({
|
||||
virtual,
|
||||
isDir: false,
|
||||
fingerprint: statFingerprint(etag, modified, size),
|
||||
size,
|
||||
modified,
|
||||
})
|
||||
}
|
||||
continuationToken = resp.IsTruncated === true ? resp.NextContinuationToken : undefined
|
||||
} while (continuationToken !== undefined)
|
||||
})
|
||||
yield* rows
|
||||
yield* synthDirs(root.virtual, files, markers)
|
||||
}
|
||||
}
|
||||
|
||||
// Build the delta hook shared by S3 and every S3-compatible alias.
|
||||
export function buildDeltaHook(accessor: S3Accessor): DeltaHook {
|
||||
const walk = new S3Walk(accessor)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { SharePointAccessor } from '../../accessor/sharepoint.ts'
|
||||
import type { IndexCacheStore } from '../../cache/index/store.ts'
|
||||
import type { FileStat, PathSpec } from '../../types.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { ListingDeltaHook } from '../../watch/delta.ts'
|
||||
import { ReaddirWalk } from '../../watch/walk.ts'
|
||||
import { readdir, stat } from './index.ts'
|
||||
|
||||
/**
|
||||
* Build the SharePoint delta hook.
|
||||
*
|
||||
* Same Graph drive surface as OneDrive, so the same `cTag` fingerprint and the
|
||||
* same standing offer of a native `/delta` fast path; see
|
||||
* `core/onedrive/watch.ts`.
|
||||
*/
|
||||
export function buildDeltaHook(accessor: SharePointAccessor): DeltaHook {
|
||||
const walk = new ReaddirWalk(
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
|
||||
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
|
||||
)
|
||||
return new ListingDeltaHook(walk.walk.bind(walk))
|
||||
}
|
||||
@@ -13,6 +13,8 @@ import { ResourceName, type FileStat, type PathSpec } from '../../types.ts'
|
||||
import type { RegisteredCommand } from '../../commands/config.ts'
|
||||
import { BaseResource, type Resource } from '../base.ts'
|
||||
import { ONEDRIVE_PROMPT } from './prompt.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { buildDeltaHook } from '../../core/onedrive/watch.ts'
|
||||
|
||||
const resolveGlob = makeResolveGlob(readdir)
|
||||
|
||||
@@ -68,6 +70,10 @@ export class OneDriveResource extends BaseResource implements Resource {
|
||||
return stat(this.accessor, path, this.index)
|
||||
}
|
||||
|
||||
deltaHook(): DeltaHook {
|
||||
return buildDeltaHook(this.accessor)
|
||||
}
|
||||
|
||||
override getState(): OneDriveResourceState {
|
||||
const config: OneDriveConfigRedacted = redactOneDriveConfig(this.config)
|
||||
return { type: this.kind, config }
|
||||
|
||||
@@ -13,6 +13,8 @@ import { ResourceName, type FileStat, type PathSpec } from '../../types.ts'
|
||||
import type { RegisteredCommand } from '../../commands/config.ts'
|
||||
import { BaseResource, type Resource } from '../base.ts'
|
||||
import { SHAREPOINT_PROMPT } from './prompt.ts'
|
||||
import type { DeltaHook } from '../../watch/base.ts'
|
||||
import { buildDeltaHook } from '../../core/sharepoint/watch.ts'
|
||||
|
||||
const resolveGlob = makeResolveGlob(readdir)
|
||||
|
||||
@@ -68,6 +70,10 @@ export class SharePointResource extends BaseResource implements Resource {
|
||||
return stat(this.accessor, path, this.index)
|
||||
}
|
||||
|
||||
deltaHook(): DeltaHook {
|
||||
return buildDeltaHook(this.accessor)
|
||||
}
|
||||
|
||||
override getState(): SharePointResourceState {
|
||||
const config: SharePointConfigRedacted = redactSharePointConfig(this.config)
|
||||
return { type: this.kind, config }
|
||||
|
||||
@@ -19,6 +19,18 @@ export class QueueOverflowError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
// A snapshot diff reads every path the walk did not report as a
|
||||
// DELETE, so a partial listing does not degrade into fewer events, it
|
||||
// invents wrong ones. A hook that knows its listing was truncated
|
||||
// raises this instead, leaving the caller's checkpoint untouched so the
|
||||
// next pull can still succeed.
|
||||
export class IncompleteWalkError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'IncompleteWalkError'
|
||||
}
|
||||
}
|
||||
|
||||
export class QueueClosed extends Error {
|
||||
constructor(label: string) {
|
||||
super(label)
|
||||
|
||||
@@ -21,7 +21,7 @@ export type {
|
||||
WatchRuntime,
|
||||
} from './base.ts'
|
||||
export { ListingDeltaHook, specFor } from './delta.ts'
|
||||
export { QueueClosed, QueueOverflowError } from './errors.ts'
|
||||
export { IncompleteWalkError, QueueClosed, QueueOverflowError } from './errors.ts'
|
||||
export { statFingerprint } from './fingerprint.ts'
|
||||
export {
|
||||
type QueueFactory,
|
||||
@@ -30,3 +30,4 @@ export {
|
||||
type WatchQueue,
|
||||
} from './queue/index.ts'
|
||||
export { Watcher } from './watcher.ts'
|
||||
export { entryOf, ReaddirWalk, type WalkReaddirFn, synthDirs, type WalkStatFn } from './walk.ts'
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { IndexCacheStore } from '../cache/index/store.ts'
|
||||
import { FileStat, FileType, PathSpec, type WalkEntry } from '../types.ts'
|
||||
import { enoent } from '../utils/errors.ts'
|
||||
import { entryOf, ReaddirWalk, synthDirs } from './walk.ts'
|
||||
|
||||
function root(virtual: string, resourcePath: string): PathSpec {
|
||||
return new PathSpec({ virtual, directory: virtual, resourcePath })
|
||||
}
|
||||
|
||||
describe('synthDirs', () => {
|
||||
it('emits every ancestor of a file, excluding the root', () => {
|
||||
const dirs = [...synthDirs('/m/data', ['/m/data/a/b/c.txt'], [])].map((e) => e.virtual)
|
||||
expect(dirs).toEqual(['/m/data/a/b', '/m/data/a'])
|
||||
})
|
||||
|
||||
it('reports a shared prefix once', () => {
|
||||
const dirs = [...synthDirs('/m/data', ['/m/data/a/x.txt', '/m/data/a/y.txt'], [])]
|
||||
expect(dirs.map((e) => e.virtual)).toEqual(['/m/data/a'])
|
||||
})
|
||||
|
||||
it('emits an explicitly stored directory even with no children', () => {
|
||||
const dirs = [...synthDirs('/m/data', [], ['/m/data/empty'])]
|
||||
expect(dirs.map((e) => e.virtual)).toEqual(['/m/data/empty'])
|
||||
})
|
||||
|
||||
it('does not double-report a prefix backed by both a marker and children', () => {
|
||||
const dirs = [...synthDirs('/m/data', ['/m/data/a/x.txt'], ['/m/data/a'])]
|
||||
expect(dirs.map((e) => e.virtual)).toEqual(['/m/data/a'])
|
||||
})
|
||||
|
||||
it('marks every row as a directory with no fingerprint', () => {
|
||||
const dirs = [...synthDirs('/m/data', ['/m/data/a/x.txt'], [])]
|
||||
expect(dirs.every((e) => e.isDir && e.fingerprint === null)).toBe(true)
|
||||
})
|
||||
|
||||
it('emits nothing for a file directly under the root', () => {
|
||||
expect([...synthDirs('/m/data', ['/m/data/x.txt'], [])]).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('entryOf', () => {
|
||||
it('reports a directory with no fingerprint', () => {
|
||||
const entry = entryOf('/m/d', new FileStat({ name: 'd', type: FileType.DIRECTORY }))
|
||||
expect(entry).toEqual({ virtual: '/m/d', isDir: true, fingerprint: null })
|
||||
})
|
||||
|
||||
it('prefers the backend fingerprint over the composite', () => {
|
||||
const entry = entryOf(
|
||||
'/m/f.txt',
|
||||
new FileStat({ name: 'f.txt', size: 3, modified: 'T', fingerprint: 'etag-1' }),
|
||||
)
|
||||
expect(entry.fingerprint).toBe('etag-1')
|
||||
})
|
||||
|
||||
it('falls back to mtime|size when the backend has no version', () => {
|
||||
const entry = entryOf('/m/f.txt', new FileStat({ name: 'f.txt', size: 3, modified: 'T' }))
|
||||
expect(entry.fingerprint).toBe('T|3')
|
||||
})
|
||||
})
|
||||
|
||||
type FakeTree = Record<string, { children?: string[]; stat: FileStat }>
|
||||
|
||||
function fakeBackend(tree: FakeTree): ReaddirWalk {
|
||||
const readdir = (path: PathSpec): Promise<string[]> => {
|
||||
const node = tree[path.virtual]
|
||||
if (node?.children === undefined) return Promise.reject(enoent(path.virtual))
|
||||
return Promise.resolve(node.children)
|
||||
}
|
||||
const stat = (path: PathSpec): Promise<FileStat> => {
|
||||
const node = tree[path.virtual]
|
||||
if (node === undefined) return Promise.reject(enoent(path.virtual))
|
||||
return Promise.resolve(node.stat)
|
||||
}
|
||||
return new ReaddirWalk(readdir as never, stat as never)
|
||||
}
|
||||
|
||||
async function collect(walk: ReaddirWalk, spec: PathSpec): Promise<WalkEntry[]> {
|
||||
const out: WalkEntry[] = []
|
||||
for await (const entry of walk.walk(spec)) out.push(entry)
|
||||
return out
|
||||
}
|
||||
|
||||
describe('ReaddirWalk', () => {
|
||||
it('descends into every directory and reports leaves', async () => {
|
||||
const walk = fakeBackend({
|
||||
'/m/data': {
|
||||
children: ['/m/data/a.txt', '/m/data/sub'],
|
||||
stat: new FileStat({ name: 'data', type: FileType.DIRECTORY }),
|
||||
},
|
||||
'/m/data/a.txt': {
|
||||
stat: new FileStat({ name: 'a.txt', size: 5, modified: 'T1', fingerprint: 'fp-a' }),
|
||||
},
|
||||
'/m/data/sub': {
|
||||
children: ['/m/data/sub/deep.txt'],
|
||||
stat: new FileStat({ name: 'sub', type: FileType.DIRECTORY }),
|
||||
},
|
||||
'/m/data/sub/deep.txt': {
|
||||
stat: new FileStat({ name: 'deep.txt', size: 4, modified: 'T2', fingerprint: 'fp-d' }),
|
||||
},
|
||||
})
|
||||
const entries = await collect(walk, root('/m/data', 'data'))
|
||||
expect(entries.map((e) => e.virtual)).toEqual([
|
||||
'/m/data/a.txt',
|
||||
'/m/data/sub',
|
||||
'/m/data/sub/deep.txt',
|
||||
])
|
||||
expect(entries.filter((e) => e.isDir).map((e) => e.virtual)).toEqual(['/m/data/sub'])
|
||||
})
|
||||
|
||||
it('trusts a trailing slash without a stat round trip', async () => {
|
||||
const walk = fakeBackend({
|
||||
'/m/data': {
|
||||
children: ['/m/data/sub/'],
|
||||
stat: new FileStat({ name: 'data', type: FileType.DIRECTORY }),
|
||||
},
|
||||
// No stat entry for /m/data/sub at all: the slash is the proof,
|
||||
// so a stat would reject and the walk would lose the subtree.
|
||||
'/m/data/sub': {
|
||||
children: [],
|
||||
stat: new FileStat({ name: 'sub', type: FileType.DIRECTORY }),
|
||||
},
|
||||
})
|
||||
const entries = await collect(walk, root('/m/data', 'data'))
|
||||
expect(entries).toEqual([{ virtual: '/m/data/sub', isDir: true, fingerprint: null }])
|
||||
})
|
||||
|
||||
it('skips an entry that vanished between the readdir and the stat', async () => {
|
||||
const walk = fakeBackend({
|
||||
'/m/data': {
|
||||
children: ['/m/data/gone.txt', '/m/data/here.txt'],
|
||||
stat: new FileStat({ name: 'data', type: FileType.DIRECTORY }),
|
||||
},
|
||||
'/m/data/here.txt': {
|
||||
stat: new FileStat({ name: 'here.txt', size: 1, modified: 'T', fingerprint: 'fp' }),
|
||||
},
|
||||
})
|
||||
const entries = await collect(walk, root('/m/data', 'data'))
|
||||
expect(entries.map((e) => e.virtual)).toEqual(['/m/data/here.txt'])
|
||||
})
|
||||
|
||||
it('walks a missing root as empty rather than throwing', async () => {
|
||||
const walk = fakeBackend({})
|
||||
expect(await collect(walk, root('/m/gone', 'gone'))).toEqual([])
|
||||
})
|
||||
|
||||
it('starts from an empty index on every call', async () => {
|
||||
const seen: (IndexCacheStore | undefined)[] = []
|
||||
const readdir = (path: PathSpec, index: IndexCacheStore): Promise<string[]> => {
|
||||
seen.push(index)
|
||||
return Promise.resolve(path.virtual === '/m/data' ? ['/m/data/a.txt'] : [])
|
||||
}
|
||||
const stat = (): Promise<FileStat> =>
|
||||
Promise.resolve(new FileStat({ name: 'a.txt', size: 1, modified: 'T', fingerprint: 'fp' }))
|
||||
const walk = new ReaddirWalk(readdir as never, stat as never)
|
||||
await collect(walk, root('/m/data', 'data'))
|
||||
await collect(walk, root('/m/data', 'data'))
|
||||
// Two pulls, two distinct index instances: nothing a pull learned
|
||||
// can leak into the next one's snapshot.
|
||||
expect(seen[0]).not.toBe(seen[seen.length - 1])
|
||||
})
|
||||
})
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user