Merge pull request #811 from strukto-ai/feat/watch-delta-hooks

feat(watch): ship delta_hook for ten more backends
This commit is contained in:
Zecheng Zhang
2026-08-15 10:27:48 -07:00
committed by GitHub
119 changed files with 6192 additions and 518 deletions
+46 -1
View File
@@ -702,11 +702,56 @@ jobs:
- name: Run declarative battery on nextcloud
run: ./python/.venv/bin/python integ/runners/python/main.py --target nextcloud --strict
- name: Run watch battery on nextcloud
# disk, ssh and dropbox need nothing (tempdir, in-process SFTP
# server, in-process fake); s3 and gridfs need a real service each,
# and skip when the env is absent rather than failing.
- name: Start MinIO for the watch battery
run: |
docker run -d --name minio-watch -p 9000:9000 \
-e MINIO_ROOT_USER=minio -e MINIO_ROOT_PASSWORD=minio123 \
minio/minio:latest server /data
for i in $(seq 1 30); do
curl -sf http://localhost:9000/minio/health/live && break
sleep 1
done
curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc
chmod +x /tmp/mc
/tmp/mc alias set local http://localhost:9000 minio minio123
/tmp/mc mb local/mirage-watch
- name: Start MongoDB for the watch battery
run: |
docker run -d --name mongo-watch -p 27017:27017 mongo:8
for i in $(seq 1 30); do
docker exec mongo-watch mongosh --quiet \
--eval 'db.runCommand({ping:1}).ok' >/dev/null 2>&1 && break
sleep 1
done
# gdrive is the one watch target whose fake is TypeScript and
# out-of-process; the other five in-process fakes need nothing.
- name: Start the fake Google Workspace server
run: |
cd integ && nohup pnpm exec tsx server/gws_server.ts --port 19999 \
> /tmp/gws.log 2>&1 &
for i in $(seq 1 30); do
grep -q "GWS_URL=" /tmp/gws.log && break
sleep 1
done
cat /tmp/gws.log
- name: Run watch battery
env:
NEXTCLOUD_URL: http://localhost:8080/remote.php/dav/files/admin/
NEXTCLOUD_USERNAME: admin
NEXTCLOUD_PASSWORD: admin123
S3_ENDPOINT: http://localhost:9000
S3_BUCKET: mirage-watch
S3_REGION: us-east-1
AWS_ACCESS_KEY_ID: minio
AWS_SECRET_ACCESS_KEY: minio123
MONGODB_URI: mongodb://localhost:27017
GWS_URL: http://127.0.0.1:19999
run: ./python/.venv/bin/python integ/watch/run.py
integ-fuse:
+1
View File
@@ -106,6 +106,7 @@ jobs:
run examples/python/ram/ram_python.py integ/truth/python/ram_python.txt
run examples/python/disk/disk.py integ/truth/python/disk.txt
run examples/python/disk/disk_vfs.py integ/truth/python/disk_vfs.txt
run examples/python/disk/watch.py integ/truth/watch_delta.txt
run examples/python/other/custom_command.py integ/truth/python/custom_command.txt
run examples/python/filetype/filetype.py integ/truth/python/filetype.txt
run examples/python/redis_resource/example_redis.py integ/truth/python/redis.txt
+1
View File
@@ -100,6 +100,7 @@ jobs:
run ram/ram_vfs.ts integ/truth/typescript/ram_vfs.txt
run disk/disk.ts integ/truth/typescript/disk.txt
run disk/disk_vfs.ts integ/truth/typescript/disk_vfs.txt
run disk/watch.ts integ/truth/watch_delta.txt
run other/custom_command.ts integ/truth/typescript/custom_command.txt
run filetype/filetype.ts integ/truth/typescript/filetype.txt
run pyodide/basic.ts integ/truth/typescript/pyodide_basic.txt
+78 -14
View File
@@ -18,17 +18,45 @@ Watching splits into two halves, and they have different support stories:
would subscribe to; mapping its payload to a `FileEvent` is consumer code,
and Mirage ships a sample where marked.
| Resource | Pull (`delta_hook`) | Push signal (provider-side) | Sample |
| --- | --- | --- | --- |
| Nextcloud | ✓ ETag walk diff | `webhook_listeners` app (Nextcloud 30+) | [example](https://github.com/strukto-ai/mirage/blob/main/examples/python/nextcloud/watch.py) + integ receiver |
| S3 / S3-compatible | planned | S3 Event Notifications (SQS/EventBridge) | — |
| Google Drive | planned | `changes.watch` push channels | — |
| Dropbox | planned | Dropbox webhooks + cursor delta | — |
| OneDrive / SharePoint | planned | Graph change notifications + delta API | — |
| GitHub | planned | repository webhooks (push events) | — |
| Slack | planned | Events API (`message`, `file_shared`, ...) | — |
| Disk | planned | local inotify / FSEvents / watchdog | — |
| RAM, Redis, others | — | (no external writers to observe) | — |
| Resource | Pull (`delta_hook`) | Fingerprint | Push signal (provider-side) | Sample |
| --- | --- | --- | --- | --- |
| Nextcloud | ✓ recursive PROPFIND | WebDAV ETag | `webhook_listeners` app (Nextcloud 30+) | [example](https://github.com/strukto-ai/mirage/blob/main/examples/python/nextcloud/watch.py) + integ receiver |
| S3 / S3-compatible | ✓ recursive `list_objects_v2` | object ETag | S3 Event Notifications (SQS/EventBridge) | — |
| GitHub | ✓ recursive git tree | blob sha | repository webhooks (push events) | — |
| Dropbox | ✓ recursive `list_folder` | `content_hash` | Dropbox webhooks + cursor delta | — |
| GridFS | ✓ `fs.files` prefix query | revision ObjectId | MongoDB change streams | — |
| Hugging Face | ✓ recursive tree listing | Hub ETag | — | — |
| Disk | ✓ recursive directory walk | mtime\|size | local inotify / FSEvents / watchdog | [example](https://github.com/strukto-ai/mirage/blob/main/examples/python/disk/watch.py) |
| SSH / SFTP | ✓ per-directory descent | mtime\|size | — | — |
| Google Drive | ✓ per-folder descent | `modifiedTime` | `changes.watch` push channels | — |
| OneDrive / SharePoint | ✓ per-folder descent | `cTag` | Graph change notifications + delta API | — |
| Box | ✓ per-folder descent | content `sha1` | Box `/events` long poll | — |
| Slack | planned | — | Events API (`message`, `file_shared`, ...) | — |
| RAM, Redis, others | — | — | (no external writers to observe) | — |
The **Fingerprint** column is what decides UPDATE. A content-addressed one
(GitHub's blob sha, S3's single-part ETag, Dropbox's `content_hash`, Box's
`sha1`) reports
nothing when a write stores identical bytes. `mtime|size` cannot: rewriting a
file with its own contents moves the mtime, so disk and SSH report an UPDATE.
That is the filesystem's own resolution, not a Mirage choice.
### Cost per pull
Not every hook costs the same, and the poll cadence should follow the shape:
- **One request per pull**, whatever the tree: GitHub (one recursive tree
call), Nextcloud (one recursive PROPFIND), Hugging Face, and GridFS (one
prefix query). S3 is one request per 1000 keys.
- **One request per directory**: Google Drive, OneDrive/SharePoint and Box key
their trees by opaque id and offer no whole-subtree listing, so the walk
descends folder by folder. SFTP does too, for the same reason.
Where a provider offers a native cursor (Dropbox `list_folder/continue`, Graph
`/delta`, Drive `changes.list`), that is a **faster** pull, not a more correct
one, and it does not replace the walk: a server may invalidate a cursor at any
time, and the only answer to that is a full listing. Those fast paths belong
behind `pull()` with the walk as their reset.
## What "Delivery ✓" buys you without a hook
@@ -54,6 +82,17 @@ The guarantee is identical on every backend: caches for the changed path and
its ancestor listings are invalidated before delivery, so reads after an event
are fresh.
One thing every backend has to agree on for that to hold: the index is
keyed by the mount-absolute path (`/m/data/x`), which is what
`CacheManager` builds when it evicts. GitHub was the one exception, keying
its by the repo-relative path because its index is a materialized git
tree, and the mismatch was silent: evicting a key an index never held
succeeds, so a GitHub mount kept serving pre-change bytes with nothing
failing anywhere a caller could see. It now keys like the rest, and the
repo-relative path logic that wanted the other spelling (`find`, `du`,
grep's scope counter) reads the git tree on the accessor instead, which is
where a repo-relative path belongs.
## Adding pull detection to a backend
`ListingDeltaHook` is generic; a backend earns the **Pull** column with one
@@ -71,7 +110,32 @@ def delta_hook(self):
return ListingDeltaHook(S3Walk(self.accessor))
```
Two shared helpers cover the shapes that recur, so a new backend usually
writes neither loop itself:
- **`synth_dirs`** (`mirage/watch/walk.py`) builds the directory rows a prefix
store implies but does not store. An object store has no directories, so a
walk that reported only keys would show a file appearing inside a directory
that never appeared. S3 and GridFS use it, and it takes explicitly stored
markers too, so an empty directory made by `mkdir` is still reported.
- **`ReaddirWalk`** covers a backend with no recursive listing at all. It
descends through the backend's own `readdir` and `stat`, exactly as `find`
does, and gives each pull a **fresh private index**. That is what keeps the
DeltaHook contract: the index is not Mirage's read cache, so the walk cannot
compare the cache to itself, and it starts empty every pull. It has to exist
at all because Drive, Box and Graph resolve a path's id through the index its
parent's readdir populated; a null index makes every path below the root read
as absent.
Backends with a native cursor API (Dropbox `list_folder/continue`, Graph
delta) can skip the walk and implement `pull(root, checkpoint)` directly; the
opaque checkpoint becomes the server cursor, and consumers cannot tell the
difference.
delta) can implement `pull(root, checkpoint)` directly and let the opaque
checkpoint be the server cursor; consumers cannot tell the difference. Keep the
walk as the reset path, though. A cursor is a promise the server keeps, and
when it breaks (`path/reset`, `resyncRequired`) a full listing is the only
answer.
A walk that knows it saw only part of the tree must raise
`IncompleteWalkError` rather than return what it has. A snapshot diff reads
every unlisted path as a DELETE, so a partial listing does not degrade into
fewer events, it invents wrong ones. GitHub does this when the API truncates a
large repository's tree.
+10 -3
View File
@@ -111,8 +111,10 @@ is [`examples/python/nextcloud/watch.py`](https://github.com/strukto-ai/mirage/b
## Pull Mode
Backends that implement `delta_hook()` (Nextcloud today) answer one question:
*what changed under this root since the last checkpoint?* A baseline pull
Backends that implement `delta_hook()` answer one question:
*what changed under this root since the last checkpoint?* Eleven resource
families ship one today, listed in the [Watch Matrix](/python/watch-matrix)
with the fingerprint each one compares on. A baseline pull
(`checkpoint=None`) emits nothing; every later pull diffs against the
checkpoint you hand back. The whole consumer poller is:
@@ -127,6 +129,11 @@ while True:
await asyncio.sleep(30)
```
A runnable version needing no credentials is
[`examples/python/disk/watch.py`](https://github.com/strukto-ai/mirage/blob/main/examples/python/disk/watch.py):
it writes to the mount's directory behind mirage's back, pulls the diff, and
reads the changed file back through the workspace.
Pull is self-healing: it diffs current backend state against your checkpoint,
so events missed while your service was down surface on the next pull. A
common production shape is push-first with a pull at startup as recovery, or
@@ -180,7 +187,7 @@ loops end cleanly) and the workspace returns to its idle state; the next
`ws.notify`.
- The TypeScript API has the same delivery, queue, and delta-hook design; see
[TypeScript Watch](/typescript/watch).
- Nextcloud is the first backend with a `delta_hook`; push mode works with any
- Pull detection ships for eleven resource families; push mode works with any
backend today since `ws.notify` accepts events from any detection you run.
See the [Watch Matrix](/python/watch-matrix) for per-resource support.
- Events are in-memory and at-most-once per subscriber; durable queues and
+76 -12
View File
@@ -14,17 +14,45 @@ Watching has two independent halves:
`deltaHook()` ships with Mirage. **Push signal** names the provider mechanism
that your application can map to a `FileEvent`.
| Resource | Pull (`deltaHook`) | Push signal (provider-side) |
| --------------------- | ------------------ | ------------------------------------------ |
| Nextcloud | ✓ ETag walk diff | `webhook_listeners` app (Nextcloud 30+) |
| S3 / S3-compatible | planned | S3 Event Notifications (SQS/EventBridge) |
| Google Drive | planned | `changes.watch` push channels |
| Dropbox | planned | Dropbox webhooks + cursor delta |
| OneDrive / SharePoint | planned | Graph change notifications + delta API |
| GitHub | planned | repository webhooks |
| Slack | planned | Events API (`message`, `file_shared`, ...) |
| Disk | planned | inotify / FSEvents / filesystem watcher |
| RAM, Redis, others | — | no external writers to observe |
| Resource | Pull (`deltaHook`) | Fingerprint | Push signal (provider-side) |
| --------------------- | ----------------------------- | ----------------- | ------------------------------------------ |
| Nextcloud | ✓ recursive PROPFIND | WebDAV ETag | `webhook_listeners` app (Nextcloud 30+) |
| S3 / S3-compatible | ✓ recursive `ListObjectsV2` | object ETag | S3 Event Notifications (SQS/EventBridge) |
| GitHub | ✓ recursive git tree | blob sha | repository webhooks |
| Dropbox | ✓ recursive `list_folder` | `content_hash` | Dropbox webhooks + cursor delta |
| GridFS | ✓ `fs.files` prefix query | revision ObjectId | MongoDB change streams |
| Hugging Face | ✓ recursive tree listing | Hub ETag | |
| Disk | ✓ recursive directory walk | mtime\|size | inotify / FSEvents / filesystem watcher |
| SSH / SFTP | ✓ per-directory descent | mtime\|size | — |
| Google Drive | ✓ per-folder descent | `modifiedTime` | `changes.watch` push channels |
| OneDrive / SharePoint | ✓ per-folder descent | `cTag` | Graph change notifications + delta API |
| Box | ✓ per-folder descent | content `sha1` | Box `/events` long poll |
| Slack | planned | — | Events API (`message`, `file_shared`, ...) |
| RAM, Redis, others | — | — | no external writers to observe |
The **Fingerprint** column is what decides UPDATE. A content-addressed one
(GitHub's blob sha, S3's single-part ETag, Dropbox's `content_hash`, Box's
`sha1`) reports
nothing when a write stores identical bytes. `mtime|size` cannot: rewriting a
file with its own contents moves the mtime, so disk and SSH report an UPDATE.
That is the filesystem's own resolution, not a Mirage choice.
### Cost Per Pull
Not every hook costs the same, and the poll cadence should follow the shape:
- **One request per pull**, whatever the tree: GitHub (one recursive tree
call), Nextcloud (one recursive PROPFIND), Hugging Face, and GridFS (one
prefix query). S3 is one request per 1000 keys.
- **One request per directory**: Google Drive, OneDrive/SharePoint and Box key
their trees by opaque id and offer no whole-subtree listing, so the walk
descends folder by folder. SFTP does too, for the same reason.
Where a provider offers a native cursor (Dropbox `list_folder/continue`, Graph
`/delta`, Drive `changes.list`), that is a **faster** pull, not a more correct
one, and it does not replace the walk: a server may invalidate a cursor at any
time, and the only answer to that is a full listing. Those fast paths belong
behind `pull()` with the walk as their reset.
## Delivery Without a Hook
@@ -56,6 +84,17 @@ the iterator drops the event.
The guarantee is the same on every backend: caches for the changed path and
its ancestor listings are invalidated before delivery.
One thing every backend has to agree on for that to hold: the index is
keyed by the mount-absolute path (`/m/data/x`), which is what
`CacheManager` builds when it evicts. GitHub was the one exception, keying
its by the repo-relative path because its index is a materialized git
tree, and the mismatch was silent: evicting a key an index never held
succeeds, so a GitHub mount kept serving pre-change bytes with nothing
failing anywhere a caller could see. It now keys like the rest, and the
repo-relative path logic that wanted the other spelling (`find`, `du`,
grep's scope counter) reads the git tree on the accessor instead, which is
where a repo-relative path belongs.
## Adding Pull Detection
`ListingDeltaHook` provides a generic checkpointed diff. A backend supplies an
@@ -83,6 +122,31 @@ async function* walk(root: PathSpec): AsyncGenerator<WalkEntry> {
const hook = new ListingDeltaHook(walk);
```
Two shared helpers cover the shapes that recur, so a new backend usually
writes neither loop itself:
- **`synthDirs`** builds the directory rows a prefix store implies but does not
store. An object store has no directories, so a walk that reported only keys
would show a file appearing inside a directory that never appeared. S3 and
GridFS use it, and it takes explicitly stored markers too, so an empty
directory made by `mkdir` is still reported.
- **`ReaddirWalk`** covers a backend with no recursive listing at all. It
descends through the backend's own `readdir` and `stat`, exactly as `find`
does, and gives each pull a **fresh private index**. That is what keeps the
DeltaHook contract: the index is not Mirage's read cache, so the walk cannot
compare the cache to itself, and it starts empty every pull. It has to exist
at all because Drive, Box and Graph resolve a path's id through the index its
parent's readdir populated; a null index makes every path below the root read
as absent.
Backends with a native cursor API can implement `pull(root, checkpoint)`
directly. The opaque checkpoint can be a Dropbox cursor, a Microsoft Graph
delta link, or any other serialized provider state.
delta link, or any other serialized provider state. Keep the walk as the reset
path, though. A cursor is a promise the server keeps, and when it breaks
(`path/reset`, `resyncRequired`) a full listing is the only answer.
A walk that knows it saw only part of the tree must throw
`IncompleteWalkError` rather than return what it has. A snapshot diff reads
every unlisted path as a DELETE, so a partial listing does not degrade into
fewer events, it invents wrong ones. GitHub does this when the API truncates a
large repository's tree.
+11 -4
View File
@@ -112,8 +112,10 @@ HTTP framework, authentication, retries, and deployment model.
## Pull Mode
Backends that implement `deltaHook()` answer one question: what changed under
this root since the last checkpoint? Nextcloud ships this capability today. A
baseline pull (`checkpoint === null`) establishes state and emits no changes.
this root since the last checkpoint? Eleven resource families ship one today,
listed in the [Watch Matrix](/typescript/watch-matrix) with the fingerprint
each one compares on. A baseline pull (`checkpoint === null`) establishes
state and emits no changes.
```ts
import { PathSpec } from "@struktoai/mirage-node";
@@ -135,6 +137,11 @@ for (;;) {
}
```
A runnable version needing no credentials is
[`examples/typescript/disk/watch.ts`](https://github.com/strukto-ai/mirage/blob/main/examples/typescript/disk/watch.ts):
it writes to the mount's directory behind mirage's back, pulls the diff, and
reads the changed file back through the workspace.
Pull is self-healing because current backend state is compared with the saved
checkpoint. Events missed while your service was down appear on the next pull.
A common production shape is push-first with a startup pull for recovery, or a
@@ -178,7 +185,7 @@ watch loops cleanly without closing the workspace. The next `watch()` or
the scope.
- One watch may span mounts, including nested mounts. Each event is invalidated
on the longest-prefix mount that owns its path.
- Delivery works on every backend. Nextcloud is currently the first backend
with a shipped `deltaHook()`; see the [Watch Matrix](/typescript/watch-matrix).
- Delivery works on every backend. Pull detection ships for eleven resource
families; see the [Watch Matrix](/typescript/watch-matrix).
- Events are in-memory and at-most-once per subscriber unless you provide a
durable queue implementation.
+88
View File
@@ -0,0 +1,88 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import shutil
import tempfile
from pathlib import Path
from mirage import MountMode, Workspace
from mirage.resource.disk import DiskResource
from mirage.types import PathSpec
MOUNT = "/data"
def seed(root: Path) -> None:
"""Lay down the files the baseline pull will record.
Args:
root (Path): Directory the mount is rooted at.
"""
(root / "reports").mkdir()
(root / "reports" / "q1.txt").write_text("first quarter\n")
(root / "reports" / "q2.txt").write_text("second quarter\n")
def write_behind_mirage(root: Path) -> None:
"""Change the directory the way an outside writer would.
Nothing here goes through the workspace: this stands in for the
teammate, cron job or pipeline that mirage has to detect rather
than be told about.
Args:
root (Path): Directory the mount is rooted at.
"""
(root / "reports" / "q3.txt").write_text("third quarter\n")
(root / "reports" / "q1.txt").write_text("first quarter, revised\n")
(root / "reports" / "q2.txt").unlink()
async def main() -> None:
tmp = Path(tempfile.mkdtemp())
seed(tmp)
resource = DiskResource(root=str(tmp))
ws = Workspace({MOUNT: resource}, mode=MountMode.READ)
hook = resource.delta_hook()
root = PathSpec.from_str_path(MOUNT, resource_path="")
# A baseline pull records state and reports nothing. Hand the
# checkpoint back on the next call and it diffs against it.
delta = await hook.pull(root, None)
checkpoint = delta.checkpoint
print(f"baseline: {len(delta.changes)} changes")
write_behind_mirage(tmp)
delta = await hook.pull(root, checkpoint)
print(f"\npull: {len(delta.changes)} changes")
for change in sorted(delta.changes, key=lambda c: c.path.virtual):
print(f" {change.kind.value:6} {change.path.virtual}")
await ws.notify(change)
# notify invalidates the caches for the changed path and its
# ancestor listings before delivering, so a read after an event
# can never serve pre-change bytes.
result = await ws.execute(f"cat {MOUNT}/reports/q1.txt")
print(f"\nread after notify: {(await result.stdout_str()).strip()!r}")
result = await ws.execute(f"ls {MOUNT}/reports")
print(f"listing: {' '.join((await result.stdout_str()).split())}")
await ws.close()
shutil.rmtree(tmp)
asyncio.run(main())
+75
View File
@@ -0,0 +1,75 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { DiskResource, MountMode, PathSpec, Workspace } from '@struktoai/mirage-node'
const MOUNT = '/data'
// Lay down the files the baseline pull will record.
function seed(root: string): void {
mkdirSync(join(root, 'reports'))
writeFileSync(join(root, 'reports', 'q1.txt'), 'first quarter\n')
writeFileSync(join(root, 'reports', 'q2.txt'), 'second quarter\n')
}
// Change the directory the way an outside writer would. Nothing here
// goes through the workspace: this stands in for the teammate, cron job
// or pipeline that mirage has to detect rather than be told about.
function writeBehindMirage(root: string): void {
writeFileSync(join(root, 'reports', 'q3.txt'), 'third quarter\n')
writeFileSync(join(root, 'reports', 'q1.txt'), 'first quarter, revised\n')
unlinkSync(join(root, 'reports', 'q2.txt'))
}
async function main(): Promise<void> {
const tmp = mkdtempSync(join(tmpdir(), 'mirage-disk-watch-'))
seed(tmp)
const resource = new DiskResource({ root: tmp })
const ws = new Workspace({ [MOUNT]: resource }, { mode: MountMode.READ })
const hook = resource.deltaHook()
const root = PathSpec.fromStrPath(MOUNT, '')
// A baseline pull records state and reports nothing. Hand the
// checkpoint back on the next call and it diffs against it.
let delta = await hook.pull(root, null)
const checkpoint = delta.checkpoint
console.log(`baseline: ${delta.changes.length} changes`)
writeBehindMirage(tmp)
delta = await hook.pull(root, checkpoint)
console.log(`\npull: ${delta.changes.length} changes`)
const changes = [...delta.changes].sort((a, b) => a.path.virtual.localeCompare(b.path.virtual))
for (const change of changes) {
console.log(` ${change.kind.padEnd(6)} ${change.path.virtual}`)
await ws.notify(change)
}
// notify invalidates the caches for the changed path and its ancestor
// listings before delivering, so a read after an event can never serve
// pre-change bytes.
let result = await ws.execute(`cat ${MOUNT}/reports/q1.txt`)
console.log(`\nread after notify: '${result.stdoutText.trim()}'`)
result = await ws.execute(`ls ${MOUNT}/reports`)
console.log(`listing: ${result.stdoutText.split(/\s+/).filter(Boolean).join(' ')}`)
await ws.close()
rmSync(tmp, { recursive: true, force: true })
}
await main()
+59 -9
View File
@@ -12,6 +12,7 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { createHash } from 'node:crypto'
import {
createServer,
type IncomingMessage,
@@ -38,6 +39,8 @@ interface DropboxEntryJson {
path_display: string
size?: number
server_modified?: string
rev?: string
content_hash?: string
}
interface SearchMatchJson {
@@ -96,6 +99,7 @@ class Account {
readonly folders = new Set<string>()
readonly files = new Map<string, StoredFile>()
readonly searchCursors = new Map<string, { matches: SearchMatchJson[]; start: number; limit: number }>()
readonly listCursors = new Map<string, DropboxEntryJson[]>()
addAncestors(path: string): void {
const parts = path.split('/').slice(1, -1)
@@ -109,6 +113,12 @@ class Account {
entryFor(path: string): DropboxEntryJson | null {
const stored = this.files.get(path)
if (stored !== undefined) {
// Real Dropbox derives content_hash from a SHA-256 over 4 MiB
// block digests; the fake hashes the whole content instead, which
// is opaque to a client that only ever compares it for equality,
// and keeps the property that matters: identical bytes hash
// identically.
const digest = createHash('sha256').update(stored.data).digest('hex')
return {
'.tag': 'file',
id: `id:${path}`,
@@ -117,6 +127,8 @@ class Account {
path_display: path,
size: stored.data.length,
server_modified: stored.modified,
rev: digest.slice(0, 16),
content_hash: digest,
}
}
if (this.folders.has(path)) {
@@ -131,18 +143,37 @@ class Account {
return null
}
listChildren(path: string): DropboxEntryJson[] | null {
listChildren(path: string, recursive = false): DropboxEntryJson[] | null {
if (path !== '' && !this.folders.has(path)) return null
const prefix = `${path}/`
const under = (candidate: string): boolean =>
recursive ? candidate.startsWith(prefix) : candidate.slice(0, candidate.lastIndexOf('/')) === path
const out: DropboxEntryJson[] = []
for (const folder of this.folders) {
if (folder.slice(0, folder.lastIndexOf('/')) !== path) continue
out.push(this.entryFor(folder) as DropboxEntryJson)
if (under(folder)) out.push(this.entryFor(folder) as DropboxEntryJson)
}
for (const file of this.files.keys()) {
if (file.slice(0, file.lastIndexOf('/')) !== path) continue
out.push(this.entryFor(file) as DropboxEntryJson)
if (under(file)) out.push(this.entryFor(file) as DropboxEntryJson)
}
return out.sort((a, b) => (a.name < b.name ? -1 : 1))
// A recursive listing is ordered parent-before-child, which is what
// real Dropbox guarantees and what a consumer building a tree from
// the stream relies on.
const key = recursive ? 'path_display' : 'name'
return out.sort((a, b) => ((a[key] ?? '') < (b[key] ?? '') ? -1 : 1))
}
// Real list_folder always returns a cursor and pages on `limit`, so
// the fake does too: a client that ignores `has_more` sees a short
// listing, which is the bug the pagination loop exists to avoid.
listPage(
entries: DropboxEntryJson[],
limit: number,
): { entries: DropboxEntryJson[]; cursor: string; has_more: boolean } {
const head = entries.slice(0, limit)
const tail = entries.slice(limit)
const token = `cursor-${this.listCursors.size}`
this.listCursors.set(token, tail)
return { entries: head, cursor: token, has_more: tail.length > 0 }
}
// Removes a file, or a folder plus its subtree (delete_v2 semantics).
@@ -223,13 +254,32 @@ function handle(
return
}
if (url === '/2/files/list_folder') {
const { path = '' } = JSON.parse(body.toString('utf8') || '{}') as { path?: string }
const entries = account.listChildren(path)
const {
path = '',
recursive = false,
limit = 2000,
} = JSON.parse(body.toString('utf8') || '{}') as {
path?: string
recursive?: boolean
limit?: number
}
const entries = account.listChildren(path, recursive)
if (entries === null) {
jsonError(res, 'path/not_found/...')
return
}
json(res, { entries, cursor: 'cursor-0', has_more: false })
json(res, account.listPage(entries, limit))
return
}
if (url === '/2/files/list_folder/continue') {
const { cursor = '' } = JSON.parse(body.toString('utf8') || '{}') as { cursor?: string }
const rest = account.listCursors.get(cursor)
if (rest === undefined) {
jsonError(res, 'reset/...')
return
}
account.listCursors.delete(cursor)
json(res, account.listPage(rest, 2000))
return
}
if (url === '/2/files/get_metadata') {
+64 -10
View File
@@ -12,6 +12,7 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import hashlib
import json
import re
import time
@@ -68,6 +69,7 @@ class FakeDropbox:
self.folders: set[str] = set()
self.files: dict[str, tuple[bytes, str]] = {}
self.search_cursors: dict[str, tuple[list, int, int]] = {}
self.list_cursors: dict[str, list[dict]] = {}
self.endpoint = ""
def _add_ancestors(self, path: str) -> None:
@@ -80,6 +82,12 @@ class FakeDropbox:
def _entry_for(self, path: str) -> dict | None:
stored = self.files.get(path)
if stored is not None:
# Real Dropbox derives content_hash from a SHA-256 over
# 4 MiB block digests; the fake hashes the whole content
# instead, which is opaque to a client that only ever
# compares it for equality, and keeps the property that
# matters: identical bytes hash identically.
digest = hashlib.sha256(stored[0]).hexdigest()
return {
".tag": "file",
"id": f"id:{path}",
@@ -88,6 +96,8 @@ class FakeDropbox:
"path_display": path,
"size": len(stored[0]),
"server_modified": stored[1],
"rev": digest[:16],
"content_hash": digest,
}
if path in self.folders:
return {
@@ -99,17 +109,30 @@ class FakeDropbox:
}
return None
def _list_children(self, path: str) -> list[dict] | None:
def _list_children(self,
path: str,
recursive: bool = False) -> list[dict] | None:
if path and path not in self.folders:
return None
prefix = f"{path}/"
def under(candidate: str) -> bool:
if recursive:
return candidate.startswith(prefix)
return candidate.rsplit("/", 1)[0] == path
out: list[dict] = []
for folder in self.folders:
if folder.rsplit("/", 1)[0] == path:
if under(folder):
out.append(self._entry_for(folder))
for file in self.files:
if file.rsplit("/", 1)[0] == path:
if under(file):
out.append(self._entry_for(file))
return sorted(out, key=lambda e: e["name"])
# A recursive listing is ordered parent-before-child, which is
# what real Dropbox guarantees and what a consumer building a
# tree from the stream relies on.
key = "path_display" if recursive else "name"
return sorted(out, key=lambda e: e[key])
def _remove(self, path: str) -> bool:
# Removes a file, or a folder plus its subtree (delete_v2).
@@ -151,17 +174,46 @@ class FakeDropbox:
"expires_in": 14400,
})
def _list_page(self, entries: list[dict], limit: int) -> dict:
"""Cut one page off a listing and park the rest under a cursor.
Real list_folder always returns a cursor and pages on ``limit``,
so the fake does too: a client that ignores ``has_more`` sees a
short listing, which is the bug the pagination loop exists to
avoid.
Args:
entries (list[dict]): full listing to page.
limit (int): maximum entries per page.
"""
head, tail = entries[:limit], entries[limit:]
token = f"cursor-{len(self.list_cursors)}"
self.list_cursors[token] = tail
return {
"entries": head,
"cursor": token,
"has_more": bool(tail),
}
async def handle_list_folder(self, request: web.Request) -> web.Response:
body = await request.json()
entries = self._list_children(body.get("path") or "")
entries = self._list_children(body.get("path") or "",
recursive=bool(body.get("recursive")))
if entries is None:
return web.json_response({"error_summary": "path/not_found/..."},
status=409)
return web.json_response({
"entries": entries,
"cursor": "cursor-0",
"has_more": False,
})
return web.json_response(
self._list_page(entries, int(body.get("limit") or 2000)))
async def handle_list_folder_continue(
self, request: web.Request) -> web.Response:
body = await request.json()
token = body.get("cursor") or ""
rest = self.list_cursors.pop(token, None)
if rest is None:
return web.json_response({"error_summary": "reset/..."},
status=409)
return web.json_response(self._list_page(rest, 2000))
async def handle_get_metadata(self, request: web.Request) -> web.Response:
body = await request.json()
@@ -348,6 +400,8 @@ async def start_fake_dropbox() -> tuple[FakeDropbox, web.AppRunner]:
app = web.Application(client_max_size=8 * 1024 * 1024)
app.router.add_post("/oauth2/token", fake.handle_token)
app.router.add_post("/2/files/list_folder", fake.handle_list_folder)
app.router.add_post("/2/files/list_folder/continue",
fake.handle_list_folder_continue)
app.router.add_post("/2/files/get_metadata", fake.handle_get_metadata)
app.router.add_post("/2/files/download", fake.handle_download)
app.router.add_post("/2/files/upload", fake.handle_upload)
+10
View File
@@ -0,0 +1,10 @@
# Shared by both languages: a delta hook reports the same events from the
# same directory whichever implementation pulled it, so one truth file
# gates examples/python/disk/watch.py and examples/typescript/disk/watch.ts.
baseline: 0 changes
pull: 3 changes
update /data/reports/q1.txt
delete /data/reports/q2.txt
create /data/reports/q3.txt
read after notify: 'first quarter, revised'
listing: q1.txt q3.txt
+591
View File
@@ -0,0 +1,591 @@
{
"resources": [
"disk",
"ssh",
"dropbox",
"s3",
"gridfs",
"onedrive",
"box",
"hf_buckets",
"gdrive",
"github"
],
"mount": "/m",
"watch_dir": "/m/data",
"modes": [
"pull"
],
"seed": [
"keep.txt"
],
"cases": [
{
"id": "create_file",
"warm": [
"ls /m/data"
],
"mutate": {
"op": "write",
"path": "data/report.txt",
"body": "alpha line\nbeta line\n"
},
"expect": {
"kind": "create",
"path": "/m/data/report.txt"
},
"checks": [
{
"cmd": "cat /m/data/report.txt",
"contains": "alpha line"
},
{
"cmd": "head -n 1 /m/data/report.txt",
"contains": "alpha line"
},
{
"cmd": "ls /m/data",
"contains": "report.txt"
},
{
"cmd": "grep -rl beta /m/data",
"contains": "report.txt"
}
]
},
{
"id": "update_etag_change",
"warm": [
"cat /m/data/report.txt",
"grep -rl alpha /m/data"
],
"mutate": {
"op": "write",
"path": "data/report.txt",
"body": "gamma line\ndelta line\n"
},
"expect": {
"kind": "update",
"path": "/m/data/report.txt"
},
"checks": [
{
"cmd": "cat /m/data/report.txt",
"contains": "gamma line"
},
{
"cmd": "cat /m/data/report.txt",
"absent": "alpha"
},
{
"cmd": "grep -rl gamma /m/data",
"contains": "report.txt"
},
{
"cmd": "grep -r alpha /m/data",
"absent": "report.txt"
}
]
},
{
"id": "create_nested",
"warm": [
"ls /m/data"
],
"mutate": {
"op": "write",
"path": "data/sub/deep.txt",
"body": "deep marker\n"
},
"expect": {
"kind": "create",
"path": "/m/data/sub/deep.txt"
},
"checks": [
{
"cmd": "cat /m/data/sub/deep.txt",
"contains": "deep marker"
},
{
"cmd": "grep -rl 'deep marker' /m/data",
"contains": "deep.txt"
}
]
},
{
"id": "delete_file",
"warm": [
"cat /m/data/report.txt",
"ls /m/data"
],
"mutate": {
"op": "delete",
"path": "data/report.txt"
},
"expect": {
"kind": "delete",
"path": "/m/data/report.txt"
},
"checks": [
{
"cmd": "ls /m/data",
"absent": "report.txt"
},
{
"cmd": "grep -rl gamma /m/data",
"absent": "report.txt"
}
]
}
],
"scopes": [
{
"id": "folder",
"watch": [
"/m/data/sub"
],
"cases": [
{
"id": "inside_delivered",
"mutate": {
"op": "write",
"path": "data/sub/in.txt",
"body": "inside scope\n"
},
"expect": {
"kind": "create",
"path": "/m/data/sub/in.txt"
},
"checks": [
{
"cmd": "cat /m/data/sub/in.txt",
"contains": "inside scope"
}
]
},
{
"id": "outside_skipped",
"mutate": {
"op": "write",
"path": "data/out.txt",
"body": "outside scope\n"
},
"expect": {
"kind": "create",
"path": "/m/data/out.txt",
"delivered": false
}
}
]
},
{
"id": "pattern",
"watch": [
"/m/data/*.txt"
],
"cases": [
{
"id": "txt_delivered",
"mutate": {
"op": "write",
"path": "data/notes.txt",
"body": "txt matches\n"
},
"expect": {
"kind": "create",
"path": "/m/data/notes.txt"
},
"checks": [
{
"cmd": "cat /m/data/notes.txt",
"contains": "txt matches"
}
]
},
{
"id": "md_skipped",
"mutate": {
"op": "write",
"path": "data/notes.md",
"body": "md does not match\n"
},
"expect": {
"kind": "create",
"path": "/m/data/notes.md",
"delivered": false
}
},
{
"id": "nested_txt_skipped",
"mutate": {
"op": "write",
"path": "data/sub/deep.txt",
"body": "glob does not cross slash\n"
},
"expect": {
"kind": "create",
"path": "/m/data/sub/deep.txt",
"delivered": false
}
}
]
},
{
"id": "shallow_glob",
"watch": [
"/m/data/*"
],
"cases": [
{
"id": "top_level_delivered",
"mutate": {
"op": "write",
"path": "data/top.txt",
"body": "shallow scope\n"
},
"expect": {
"kind": "create",
"path": "/m/data/top.txt"
},
"checks": [
{
"cmd": "cat /m/data/top.txt",
"contains": "shallow scope"
}
]
},
{
"id": "nested_skipped",
"mutate": {
"op": "write",
"path": "data/sub/deep.txt",
"body": "below shallow scope\n"
},
"expect": {
"kind": "create",
"path": "/m/data/sub/deep.txt",
"delivered": false
}
}
]
},
{
"id": "dir_glob",
"watch": [
"/m/data/*/"
],
"cases": [
{
"id": "inside_dir_delivered",
"mutate": {
"op": "write",
"path": "data/sub/in.txt",
"body": "inside a child dir\n"
},
"expect": {
"kind": "create",
"path": "/m/data/sub/in.txt"
},
"checks": [
{
"cmd": "cat /m/data/sub/in.txt",
"contains": "inside a child dir"
}
]
},
{
"id": "top_level_skipped",
"mutate": {
"op": "write",
"path": "data/top.txt",
"body": "not inside a dir\n"
},
"expect": {
"kind": "create",
"path": "/m/data/top.txt",
"delivered": false
}
}
]
},
{
"id": "mid_wildcard",
"watch": [
"/m/data/*/reports/"
],
"cases": [
{
"id": "inside_reports_delivered",
"mutate": {
"op": "write",
"path": "data/proj1/reports/spec.md",
"body": "quarterly spec\n"
},
"expect": {
"kind": "create",
"path": "/m/data/proj1/reports/spec.md"
},
"checks": [
{
"cmd": "cat /m/data/proj1/reports/spec.md",
"contains": "quarterly spec"
}
]
},
{
"id": "project_sibling_skipped",
"mutate": {
"op": "write",
"path": "data/proj1/other.txt",
"body": "outside reports\n"
},
"expect": {
"kind": "create",
"path": "/m/data/proj1/other.txt",
"delivered": false
}
}
]
},
{
"id": "multi_roots",
"watch": [
"/m/data/sub",
"/m/data/*.yaml"
],
"cases": [
{
"id": "literal_root_delivered",
"mutate": {
"op": "write",
"path": "data/sub/in.txt",
"body": "literal subtree\n"
},
"expect": {
"kind": "create",
"path": "/m/data/sub/in.txt"
},
"checks": [
{
"cmd": "cat /m/data/sub/in.txt",
"contains": "literal subtree"
}
]
},
{
"id": "glob_root_delivered",
"mutate": {
"op": "write",
"path": "data/app.yaml",
"body": "yaml root\n"
},
"expect": {
"kind": "create",
"path": "/m/data/app.yaml"
},
"checks": [
{
"cmd": "cat /m/data/app.yaml",
"contains": "yaml root"
}
]
},
{
"id": "neither_root_skipped",
"mutate": {
"op": "write",
"path": "data/note.md",
"body": "matches no root\n"
},
"expect": {
"kind": "create",
"path": "/m/data/note.md",
"delivered": false
}
}
]
},
{
"id": "rename",
"watch": [
"/m/data"
],
"cases": [
{
"id": "seed_orig",
"mutate": {
"op": "write",
"path": "data/orig.txt",
"body": "original content\n"
},
"expect": {
"kind": "create",
"path": "/m/data/orig.txt"
},
"checks": [
{
"cmd": "cat /m/data/orig.txt",
"contains": "original content"
}
]
},
{
"id": "rename_as_move",
"modes": [
"push"
],
"warm": [
"cat /m/data/orig.txt",
"ls /m/data"
],
"mutate": {
"op": "rename",
"path": "data/orig.txt",
"to": "data/renamed.txt"
},
"expect": {
"kind": "move",
"path": "/m/data/renamed.txt",
"previous": "/m/data/orig.txt"
},
"checks": [
{
"cmd": "cat /m/data/renamed.txt",
"contains": "original content"
},
{
"cmd": "ls /m/data",
"absent": "orig.txt"
},
{
"cmd": "cat /m/data/orig.txt",
"absent": "original content"
}
]
},
{
"id": "rename_as_delete_create",
"modes": [
"pull"
],
"warm": [
"cat /m/data/orig.txt",
"ls /m/data"
],
"mutate": {
"op": "rename",
"path": "data/orig.txt",
"to": "data/renamed.txt"
},
"expect": {
"kind": "create",
"path": "/m/data/renamed.txt"
},
"checks": [
{
"cmd": "cat /m/data/renamed.txt",
"contains": "original content"
},
{
"cmd": "ls /m/data",
"absent": "orig.txt"
},
{
"cmd": "cat /m/data/orig.txt",
"absent": "original content"
}
]
}
],
"skip_resources": [
"hf_buckets"
]
},
{
"id": "file",
"watch": [
"/m/data/keep.txt"
],
"cases": [
{
"id": "update_delivered",
"warm": [
"cat /m/data/keep.txt"
],
"mutate": {
"op": "write",
"path": "data/keep.txt",
"body": "fresh keep\n"
},
"expect": {
"kind": "update",
"path": "/m/data/keep.txt"
},
"checks": [
{
"cmd": "cat /m/data/keep.txt",
"contains": "fresh keep"
},
{
"cmd": "cat /m/data/keep.txt",
"absent": "seed"
}
]
},
{
"id": "sibling_skipped",
"mutate": {
"op": "write",
"path": "data/other.txt",
"body": "sibling file\n"
},
"expect": {
"kind": "create",
"path": "/m/data/other.txt",
"delivered": false
}
},
{
"id": "delete_delivered",
"warm": [
"ls /m/data"
],
"mutate": {
"op": "delete",
"path": "data/keep.txt"
},
"expect": {
"kind": "delete",
"path": "/m/data/keep.txt"
},
"checks": [
{
"cmd": "ls /m/data",
"absent": "keep.txt"
}
]
}
]
}
],
"overflow": {
"max_pending": 2,
"paths": [
"data/f1.txt",
"data/f2.txt",
"data/f3.txt",
"data/f4.txt",
"data/f5.txt"
],
"checks": [
{
"cmd": "ls /m/data",
"contains": "f5.txt"
},
{
"cmd": "cat /m/data/f3.txt",
"contains": "burst"
}
]
}
}
+478
View File
@@ -0,0 +1,478 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import base64
import importlib.util
import os
import tempfile
import uuid
from pathlib import Path
from types import ModuleType
from typing import Any, Awaitable, Callable
import aiohttp
from pydantic import SecretStr
from mirage import MountMode, Workspace
from mirage.core.github._client import github_request
from mirage.core.github.read import read_bytes
from mirage.core.github.tree import fetch_tree
from mirage.core.github.tree_entry import TreeEntry
from mirage.resource.box import BoxConfig, BoxResource
from mirage.resource.disk import DiskResource
from mirage.resource.dropbox import DropboxConfig, DropboxResource
from mirage.resource.gdrive import GoogleDriveResource
from mirage.resource.gdrive.config import GoogleDriveConfig
from mirage.resource.github import GitHubConfig, GitHubResource
from mirage.resource.gridfs import GridFSConfig, GridFSResource
from mirage.resource.hf_buckets import HfBucketsConfig, HfBucketsResource
from mirage.resource.onedrive import OneDriveConfig, OneDriveResource
from mirage.resource.s3 import S3Config, S3Resource
from mirage.resource.ssh import SSHConfig, SSHResource
SERVER_DIR = Path(__file__).resolve().parents[1] / "server"
GITHUB_OWNER = "integ"
GITHUB_REPO = "watch"
GITHUB_REF = "main"
Pair = tuple[Workspace, "WorkspaceWriter | GitHubWriter"]
ResourceFactory = Callable[[], Any]
class WorkspaceWriter:
"""External writer backed by a second workspace over one backend.
The batteries need a mutation the watched workspace did not make, so
its caches are genuinely stale when the event lands. A second
workspace over the same backend is exactly that: its own cache
manager, its own index, the same bytes underneath. It also gives
every backend one writer instead of one adapter per API, which is
what lets the same cases run against all of them.
The surface mirrors the opendal operator the Nextcloud battery
writes through, so ``_mutate`` and ``_seed`` need no branch.
"""
def __init__(self, ws: Workspace, mount: str) -> None:
"""Args:
ws (Workspace): Writer workspace, distinct from the watched
one.
mount (str): Mount prefix both workspaces share.
"""
self._ws = ws
self._mount = mount.rstrip("/")
def _virtual(self, path: str) -> str:
return f"{self._mount}/{path.strip('/')}"
async def create_dir(self, path: str) -> None:
"""Args:
path (str): Mount-relative directory, trailing slash
optional.
"""
await self._ws.execute(f"mkdir -p {self._virtual(path)}")
async def write(self, path: str, data: bytes) -> None:
"""Args:
path (str): Mount-relative file path.
data (bytes): Content to store.
"""
key = path.strip("/")
parent = key.rsplit("/", 1)[0]
if parent != key:
await self.create_dir(parent)
await self._ws.ops.write(self._virtual(key), data)
async def delete(self, path: str) -> None:
"""Args:
path (str): Mount-relative path; a directory goes with its
subtree, matching opendal's delete.
"""
await self._ws.execute(f"rm -rf {self._virtual(path)}")
async def remove_all(self, path: str) -> None:
"""Args:
path (str): Mount-relative subtree to empty.
"""
await self.delete(path)
async def rename(self, path: str, to: str) -> None:
"""Args:
path (str): Mount-relative source.
to (str): Mount-relative destination.
"""
await self._ws.ops.rename(self._virtual(path), self._virtual(to))
async def close(self) -> None:
await self._ws.close()
class GitHubWriter:
"""External writer over GitHub's own contents API.
Every other backend here is written through a second workspace,
which needs the resource to have write ops. GitHub's has none, and
should not: a mount is a read view of one ref, and a ref changes by
being committed to. So this speaks what a committer speaks, ``PUT``
and ``DELETE`` on ``/contents/{path}``, each carrying the blob sha
it replaces, which is also the strongest fingerprint the differ
ever compares.
Git stores no directory object, so ``create_dir`` is a no-op: a
directory exists exactly as long as a path runs through it. That is
also why ``remove_all`` enumerates the tree rather than deleting
one marker, and why ``rename`` is a write plus a delete.
"""
def __init__(self, config: GitHubConfig, owner: str, repo: str,
ref: str) -> None:
"""Args:
config (GitHubConfig): Token and API base of the fake.
owner (str): Repository owner.
repo (str): Repository name.
ref (str): Branch the mount is pinned to.
"""
self._config = config
self._owner = owner
self._repo = repo
self._ref = ref
async def _tree(self) -> dict[str, TreeEntry]:
"""The ref's recursive tree, which names every blob's sha."""
tree, _truncated = await fetch_tree(self._config, self._owner,
self._repo, self._ref)
return tree
async def _commit(self, method: str, path: str, body: dict[str,
str]) -> None:
"""Send one contents-API call against the pinned branch.
Args:
method (str): "PUT" or "DELETE".
path (str): Repo-relative path.
body (dict[str, str]): Call-specific fields.
"""
await github_request(
self._config.token,
method,
f"/repos/{self._owner}/{self._repo}/contents/{path}", {
"branch": self._ref,
"message": f"integ watch {path}",
**body,
},
base_url=self._config.base_url)
async def create_dir(self, path: str) -> None:
"""No-op: git has no directory object to create.
Args:
path (str): Mount-relative directory, ignored.
"""
async def write(self, path: str, data: bytes) -> None:
"""Args:
path (str): Mount-relative file path.
data (bytes): Content to commit.
"""
key = path.strip("/")
entry = (await self._tree()).get(key)
body = {"content": base64.b64encode(data).decode("ascii")}
if entry is not None:
body["sha"] = entry.sha
await self._commit("PUT", key, body)
async def delete(self, path: str) -> None:
"""Args:
path (str): Mount-relative file path.
"""
entry = (await self._tree()).get(path.strip("/"))
if entry is not None:
await self._commit("DELETE", entry.path, {"sha": entry.sha})
async def remove_all(self, path: str) -> None:
"""Args:
path (str): Mount-relative subtree to empty.
"""
stem = path.strip("/")
base = f"{stem}/" if stem else ""
for entry in (await self._tree()).values():
if entry.type == "tree" or not entry.path.startswith(base):
continue
await self._commit("DELETE", entry.path, {"sha": entry.sha})
async def rename(self, path: str, to: str) -> None:
"""Args:
path (str): Mount-relative source.
to (str): Mount-relative destination.
"""
entry = (await self._tree())[path.strip("/")]
data = await read_bytes(self._config, self._owner, self._repo,
entry.sha)
await self.write(to, data)
await self._commit("DELETE", entry.path, {"sha": entry.sha})
def _load(path: Path, name: str) -> ModuleType:
"""Import one integ server module by path.
Args:
path (Path): Module file.
name (str): Name to register it under.
"""
spec_obj = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec_obj)
spec_obj.loader.exec_module(module)
return module
def _pair(spec: dict, make: ResourceFactory) -> Pair:
"""Build the watched workspace and its external writer.
Each side gets its own resource instance over the same backend, so
nothing is shared but the bytes.
Args:
spec (dict): Parsed case file.
make (ResourceFactory): Builds one fresh resource.
"""
mount = spec["mount"]
watched = Workspace({mount: make()}, mode=MountMode.WRITE)
writer = Workspace({mount: make()}, mode=MountMode.WRITE)
return watched, WorkspaceWriter(writer, mount)
async def build_disk(spec: dict) -> Pair | None:
"""Disk battery: a throwaway directory, no service at all.
Args:
spec (dict): Parsed case file.
"""
root = tempfile.mkdtemp(prefix="mirage-watch-disk-")
return _pair(spec, lambda: DiskResource(root))
async def build_ssh(spec: dict) -> Pair | None:
"""SSH battery against the in-process asyncssh SFTP server.
A real server on a real socket, chrooted to a throwaway directory,
so the per-directory SFTP descent is exercised end to end.
Args:
spec (dict): Parsed case file.
"""
module = _load(SERVER_DIR / "ssh_server.py", "integ_watch_ssh")
root = tempfile.mkdtemp(prefix="mirage-watch-ssh-")
server = await module.start_server(root)
port = server.get_port()
return _pair(
spec, lambda: SSHResource(
SSHConfig(host="127.0.0.1",
port=port,
username="integ",
known_hosts=None,
root="/")))
async def build_dropbox(spec: dict) -> Pair | None:
"""Dropbox battery against the in-process fake.
Exercises the recursive ``list_folder`` and its cursor pagination,
and fingerprints on the fake's ``content_hash``.
Args:
spec (dict): Parsed case file.
"""
module = _load(SERVER_DIR / "dropbox_server.py", "integ_watch_dropbox")
fake, _runner = await module.start_fake_dropbox()
return _pair(
spec, lambda: DropboxResource(
DropboxConfig(client_id="integ-client",
client_secret="integ-secret",
refresh_token="integ-refresh",
endpoint=fake.endpoint,
root_path="/")))
async def build_s3(spec: dict) -> Pair | None:
"""S3 battery against whatever ``S3_ENDPOINT`` names (MinIO in CI).
Each run takes its own key prefix so repeat runs cannot see each
other's objects.
Args:
spec (dict): Parsed case file.
"""
endpoint = os.environ.get("S3_ENDPOINT")
bucket = os.environ.get("S3_BUCKET")
if not endpoint or not bucket:
return None
prefix = f"watch-{uuid.uuid4().hex[:8]}/"
return _pair(
spec, lambda: S3Resource(
S3Config(bucket=bucket,
region=os.environ.get("S3_REGION", "us-east-1"),
endpoint_url=endpoint,
aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=os.environ.get(
"AWS_SECRET_ACCESS_KEY"),
path_style=True,
key_prefix=prefix)))
async def build_gridfs(spec: dict) -> Pair | None:
"""GridFS battery against ``MONGODB_URI``.
Each run takes its own database so repeat runs stay isolated.
Args:
spec (dict): Parsed case file.
"""
uri = os.environ.get("MONGODB_URI")
if not uri:
return None
database = f"watch_{uuid.uuid4().hex[:8]}"
return _pair(
spec, lambda: GridFSResource(
GridFSConfig(uri=uri, database=database, bucket="fs")))
async def build_onedrive(spec: dict) -> Pair | None:
"""OneDrive battery against the in-process Graph fake.
This is the ReaddirWalk path: Graph keys its tree by item id and has
no whole-subtree listing, so the walk descends one
``/children`` request per directory against a fresh private index.
Args:
spec (dict): Parsed case file.
"""
module = _load(SERVER_DIR / "onedrive_server.py", "integ_watch_onedrive")
state, _server, _runner = await module.start_fake_graph()
return _pair(
spec, lambda: OneDriveResource(
OneDriveConfig(access_token="integ-token",
graph_base_url=state.base)))
async def build_box(spec: dict) -> Pair | None:
"""Box battery against the in-process fake.
The second ReaddirWalk target, and the one that proves the walk is
not Graph-shaped: Box addresses folders by its own ids and answers a
different listing endpoint.
Args:
spec (dict): Parsed case file.
"""
module = _load(SERVER_DIR / "box_server.py", "integ_watch_box")
state, _server, _runner = await module.start_fake_box()
folder = state.add_folder("0", "watch")
return _pair(
spec, lambda: BoxResource(
BoxConfig(access_token="integ-box-token",
endpoint=state.base,
root_folder_id=folder["id"])))
async def build_hf(spec: dict) -> Pair | None:
"""Hugging Face battery against the in-process Hub fake.
Covers the shared opendal walk on a lister that omits per-entry
metadata, which is the stat-backfill branch Nextcloud never reaches.
The fake freezes ``modified``, so only the ETag can move: a case that
passes here passes on the ETag alone.
Args:
spec (dict): Parsed case file.
"""
module = _load(SERVER_DIR / "hf_server.py", "integ_watch_hf")
_hub, server, _runner = await module.start_fake_hub()
bucket = f"integ/watch-{uuid.uuid4().hex[:8]}"
return _pair(
spec, lambda: HfBucketsResource(
HfBucketsConfig(
bucket=bucket, token="integ-token", endpoint=server.endpoint)))
async def build_gdrive(spec: dict) -> Pair | None:
"""Google Drive battery against the external gws fake.
The gws server is TypeScript and shared across runs, so this needs
``GWS_URL`` and each run resets it and takes its own folder.
Args:
spec (dict): Parsed case file.
"""
url = os.environ.get("GWS_URL")
if not url:
return None
url = url.rstrip("/")
async with aiohttp.ClientSession() as session:
async with session.post(f"{url}/reset", json={}) as resp:
resp.raise_for_status()
folder = f"watch-{uuid.uuid4().hex[:8]}"
async with session.post(f"{url}/drive/v3/files",
json={
"name":
folder,
"mimeType":
"application/vnd.google-apps.folder",
}) as resp:
resp.raise_for_status()
folder_id = (await resp.json())["id"]
return _pair(
spec, lambda: GoogleDriveResource(
GoogleDriveConfig(client_id="integ-client",
client_secret="integ-secret",
refresh_token="integ-refresh",
api_base=url,
folder_id=folder_id)))
async def build_github(spec: dict) -> Pair | None:
"""GitHub battery against the in-process fake.
The one target whose writer is not a workspace, because a git ref
has no write ops to lend one. It is also the only backend whose
fingerprint is a git blob sha, so a rewrite of identical bytes is
correctly reported as nothing at all.
Args:
spec (dict): Parsed case file.
"""
module = _load(SERVER_DIR / "github_server.py", "integ_watch_github")
state, _server, _runner = await module.start_fake_github()
state.repo(GITHUB_OWNER, GITHUB_REPO)
config = GitHubConfig(token=SecretStr("integ-github-token"),
owner=GITHUB_OWNER,
repo=GITHUB_REPO,
ref=GITHUB_REF,
base_url=state.base)
resource = await GitHubResource.build(config)
ws = Workspace({spec["mount"]: resource}, mode=MountMode.WRITE)
return ws, GitHubWriter(config, GITHUB_OWNER, GITHUB_REPO, GITHUB_REF)
BUILDERS: dict[str, Callable[[dict], Awaitable[Pair | None]]] = {
"disk": build_disk,
"ssh": build_ssh,
"dropbox": build_dropbox,
"s3": build_s3,
"gridfs": build_gridfs,
"onedrive": build_onedrive,
"box": build_box,
"hf_buckets": build_hf,
"gdrive": build_gdrive,
"github": build_github,
}
+69 -20
View File
@@ -21,6 +21,7 @@ from pathlib import Path
import aiohttp
from aiohttp import web
from backends import BUILDERS as BACKEND_BUILDERS
from webhook_server import make_app
from mirage import MountMode, Workspace
@@ -53,7 +54,7 @@ def _nextcloud_config(url: str) -> NextcloudConfig:
)
def _build_nextcloud(spec: dict) -> tuple[Workspace, object] | None:
async def _build_nextcloud(spec: dict) -> tuple[Workspace, object] | None:
"""Build the watched workspace and a separate external writer.
Returns None when the deployment env is absent, so a local run
@@ -72,7 +73,8 @@ def _build_nextcloud(spec: dict) -> tuple[Workspace, object] | None:
return ws, external
def _build_nextcloud_nested(spec: dict) -> tuple[Workspace, object] | None:
async def _build_nextcloud_nested(
spec: dict) -> tuple[Workspace, object] | None:
"""Build the nested-mount battery's workspace: the outer mount at
the account root plus a second mount, rooted at a subfolder of the
same account, nested inside the outer mount's subtree.
@@ -97,7 +99,7 @@ def _build_nextcloud_nested(spec: dict) -> tuple[Workspace, object] | None:
return ws, external
BUILDERS = {"nextcloud": _build_nextcloud}
BUILDERS = {"nextcloud": _build_nextcloud, **BACKEND_BUILDERS}
def _files_prefix() -> str:
@@ -105,14 +107,23 @@ def _files_prefix() -> str:
return f"/{os.environ.get('NEXTCLOUD_USERNAME', 'admin')}/files"
def _watch_rel(spec: dict) -> str:
"""The watch dir as the external writer spells it, mount-relative.
Args:
spec (dict): Parsed case file.
"""
return spec["watch_dir"][len(spec["mount"].rstrip("/")):].strip("/")
def _framed_root(spec: dict) -> PathSpec:
"""Build the mount-framed watch_dir root the delta hook pulls over.
Args:
spec (dict): Parsed case file.
"""
rel = spec["watch_dir"][len(spec["mount"].rstrip("/")):].strip("/")
return PathSpec.from_str_path(spec["watch_dir"], resource_path=rel)
return PathSpec.from_str_path(spec["watch_dir"],
resource_path=_watch_rel(spec))
async def _mutate(op: object, mutate: dict) -> None:
@@ -362,11 +373,22 @@ async def _run_case(ws: Workspace, op: object, trigger, stream: EventStream,
async def _seed(ws: Workspace, op: object, spec: dict) -> None:
"""Reset the watch dir and lay down the seed files.
Both halves of the reset are load-bearing. The external writer
empties the directory, because a mount that is a read view of its
backend cannot ``rm -rf`` its own watch dir (github's is one: a ref
changes by being committed to), and a battery whose reset silently
did nothing carries one scope's files into the next, where their
CREATEs arrive as UPDATEs. The watched workspace then runs its own
``rm -rf``, which is what drops the listing it had cached.
Args:
ws (Workspace): Watched workspace.
op (object): External writer operator.
spec (dict): Parsed case file.
"""
root = _watch_rel(spec) + "/"
await op.create_dir(root)
await op.remove_all(root)
await ws.execute(f"rm -rf {spec['watch_dir']}")
await ws.execute(f"mkdir -p {spec['watch_dir']}")
for name in spec["seed"]:
@@ -450,7 +472,7 @@ async def _overflow_core(spec: dict, ws: Workspace, op: object, trigger,
await stream.close()
def _overflow_workspace(spec: dict) -> tuple[Workspace, object]:
async def _overflow_workspace(spec: dict) -> tuple[Workspace, object]:
"""Build the overflow battery's own workspace with a tiny queue.
A dedicated workspace is required because the custom queue factory
@@ -459,7 +481,7 @@ def _overflow_workspace(spec: dict) -> tuple[Workspace, object]:
Args:
spec (dict): Parsed case file.
"""
ws, op = BUILDERS[spec["resource"]](spec)
ws, op = await BUILDERS[spec["resource"]](spec)
ws.attach_watch_runtime(
Watcher(ws.registry,
queue_factory=partial(
@@ -477,7 +499,7 @@ async def _run_overflow_pull(spec: dict, results: list) -> None:
"""
if "overflow" not in spec:
return
ws, op = _overflow_workspace(spec)
ws, op = await _overflow_workspace(spec)
try:
await _seed(ws, op, spec)
resource = ws.registry.mount_for(spec["mount"]).resource
@@ -499,7 +521,7 @@ async def _run_overflow_push(spec: dict, results: list) -> None:
"""
if "overflow" not in spec:
return
ws, op = _overflow_workspace(spec)
ws, op = await _overflow_workspace(spec)
try:
await _seed(ws, op, spec)
runner = web.AppRunner(make_app(ws, _files_prefix(), spec["mount"]))
@@ -577,7 +599,7 @@ async def _run_nested_pull(spec: dict, results: list) -> None:
"""
if "nested" not in spec:
return
built = _build_nextcloud_nested(spec)
built = await _build_nextcloud_nested(spec)
if built is None:
return
ws, op = built
@@ -605,7 +627,7 @@ async def _run_nested_push(spec: dict, results: list) -> None:
"""
if "nested" not in spec:
return
built = _build_nextcloud_nested(spec)
built = await _build_nextcloud_nested(spec)
if built is None:
return
ws, op = built
@@ -653,6 +675,10 @@ async def _run_pull(spec: dict, ws: Workspace,
spec["cases"], "pull", "pull"))
for scope in spec.get("scopes", []):
# A scope whose mutation the backend has no op for (hf has no
# rename) is declared inapplicable rather than run and excused.
if spec["resource"] in scope.get("skip_resources", []):
continue
await _seed(ws, op, spec)
agen = ws.watch(scope["watch"])
poller = ConsumerPoller(resource.delta_hook(), ws, hook_root)
@@ -718,31 +744,54 @@ async def _run_file(spec: dict) -> list[tuple[str, bool, str]]:
builder = BUILDERS.get(spec["resource"])
if builder is None:
return [(spec["resource"], False, "no builder")]
modes = {"pull": _run_pull, "push": _run_push}
# Push mode needs a provider that can send a webhook, which only the
# Nextcloud deployment has; every other backend declares pull only.
wanted = spec.get("modes", ["pull", "push"])
results: list[tuple[str, bool, str]] = []
for mode in (_run_pull, _run_push):
built = builder(spec)
for name in wanted:
built = await builder(spec)
if built is None:
print(f"skip [{spec['resource']}]: deployment env absent",
file=sys.stderr)
return []
ws, op = built
try:
results.extend(await mode(spec, ws, op))
results.extend(await modes[name](spec, ws, op))
finally:
await ws.close()
closer = getattr(op, "close", None)
if closer is not None:
await closer()
return results
def _expand(spec: dict) -> list[dict]:
"""Fan one case file out over the resources it names.
A file that declares ``resources`` runs its whole body once per
backend, so the shared batteries are written down once rather than
copied per target.
Args:
spec (dict): Parsed case file.
"""
names = spec.get("resources")
if not names:
return [spec]
return [{**spec, "resource": name} for name in names]
async def main() -> None:
files = sorted(p for p in CASE_DIR.glob("*.json"))
failed = 0
for path in files:
spec = json.loads(path.read_text())
for case_id, ok, detail in await _run_file(spec):
status = "PASS" if ok else "FAIL"
print(f"{status} [{spec['resource']}] {case_id}: {detail}")
if not ok:
failed += 1
for spec in _expand(json.loads(path.read_text())):
for case_id, ok, detail in await _run_file(spec):
status = "PASS" if ok else "FAIL"
print(f"{status} [{spec['resource']}] {case_id}: {detail}")
if not ok:
failed += 1
if failed:
print(f"FAIL: {failed} watch case(s) failed", file=sys.stderr)
sys.exit(1)
+9
View File
@@ -13,6 +13,7 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.base import Accessor
from mirage.core.github.tree_entry import TreeEntry
class GitHubAccessor(Accessor):
@@ -23,10 +24,18 @@ class GitHubAccessor(Accessor):
repo,
ref,
default_branch,
tree: dict[str, TreeEntry] | None = None,
truncated=False):
self.config = config
self.owner = owner
self.repo = repo
self.ref = ref
self.default_branch = default_branch
# The recursive git tree, keyed repo-relative with no leading
# slash, which is this mount's whole listing. find, du and grep's
# scope counter read it straight, the way TypeScript's always
# have: repo-relative path logic belongs on a git tree, not on an
# index whose keys are the mount's business. Reseated by every
# refill, so it is as fresh as the last one.
self.tree: dict[str, TreeEntry] = tree if tree is not None else {}
self.truncated = truncated
+19 -9
View File
@@ -36,8 +36,9 @@ class CacheManager:
"""Args:
file_cache (FileCacheMixin | None): Workspace file cache
store; entries are keyed by mount-absolute path.
index (IndexCacheStore): The mount resource's index
cache; listings are keyed by mount-absolute path.
index (IndexCacheStore): The mount resource's index cache;
listings are keyed by mount-absolute path, which every
backend agrees on.
prefix (str): Mount prefix (e.g. "/data/").
caches_reads (bool): Whether the resource caches reads; the
file cache only holds paths for read-caching backends.
@@ -47,6 +48,19 @@ class CacheManager:
self._prefix = prefix.rstrip("/")
self._caches_reads = caches_reads
async def _evict_dir(self, virtual: str) -> None:
"""Drop one directory's cached listing.
Both spellings of the directory go, because a backend may have
keyed it with or without its trailing slash and an eviction that
hits no key is silent.
Args:
virtual (str): Mount-absolute path of the directory.
"""
await self._index.invalidate_dir(virtual)
await self._index.invalidate_dir(virtual + "/")
def _virtual(self, path: PathSpec) -> str:
mount_path = path.mount_path
if not mount_path.startswith("/"):
@@ -94,8 +108,7 @@ class CacheManager:
virtual = self._virtual(path)
if self._caches_reads and self._file_cache is not None:
await self._file_cache.remove(virtual)
await self._index.invalidate_dir(virtual)
await self._index.invalidate_dir(virtual + "/")
await self._evict_dir(virtual)
await self._invalidate_parent(virtual)
async def invalidate_ancestors(self, path: PathSpec) -> None:
@@ -114,8 +127,7 @@ class CacheManager:
parent = self._virtual(path).rsplit("/", 1)[0]
while parent and parent != self._prefix:
parent = parent.rsplit("/", 1)[0]
await self._index.invalidate_dir(parent or "/")
await self._index.invalidate_dir(parent + "/")
await self._evict_dir(parent or "/")
async def drop_prefix(self) -> None:
"""Drop every cached body under this mount, path unspecified.
@@ -135,6 +147,4 @@ class CacheManager:
await self._file_cache.evict_prefix(self._prefix + "/")
async def _invalidate_parent(self, virtual: str) -> None:
parent = virtual.rsplit("/", 1)[0] or "/"
await self._index.invalidate_dir(parent)
await self._index.invalidate_dir(parent + "/")
await self._evict_dir(virtual.rsplit("/", 1)[0] or "/")
+22 -13
View File
@@ -27,24 +27,33 @@ from mirage.provision.types import ProvisionResult
from mirage.types import PathSpec
async def _subtree(index: IndexCacheStore,
path: PathSpec) -> list[tuple[str, int]]:
key = "/" + path.resource_path if path.resource_path else "/"
prefix = key.rstrip("/") + "/"
found = [(ep, entry.size) for ep, entry in (await index.entries()).items()
if (ep == key or ep.startswith(prefix)) and entry.size is not None
]
def _subtree(accessor: GitHubAccessor,
path: PathSpec) -> list[tuple[str, int]]:
"""Every sized entry at or under ``path``, in mount-relative space.
Read off the git tree rather than the index, mirroring TypeScript's
du: the tree is keyed repo-relative, which is the space these
comparisons are in.
Args:
accessor (GitHubAccessor): backend handle holding the tree.
path (PathSpec): subtree root.
"""
key = path.resource_path.strip("/")
prefix = key + "/" if key else ""
found = [("/" + p, entry.size) for p, entry in accessor.tree.items()
if (p == key or p.startswith(prefix)) and entry.size is not None]
found.sort()
return found
async def _du_size(index: IndexCacheStore, path: PathSpec) -> int:
return sum(size for _, size in await _subtree(index, path))
async def _du_size(accessor: GitHubAccessor, path: PathSpec) -> int:
return sum(size for _, size in _subtree(accessor, path))
async def _du_entries(index: IndexCacheStore,
async def _du_entries(accessor: GitHubAccessor,
path: PathSpec) -> tuple[list[tuple[str, int]], int]:
found = await _subtree(index, path)
found = _subtree(accessor, path)
return found, sum(size for _, size in found)
@@ -70,5 +79,5 @@ async def du(accessor: GitHubAccessor, paths: list[PathSpec], texts: list[str],
return await du_generic(paths, list(texts), opts,
partial(_resolve, accessor, opts.index),
partial(_stat, accessor, opts.index),
partial(_du_size, opts.index),
partial(_du_entries, opts.index))
partial(_du_size, accessor),
partial(_du_entries, accessor))
@@ -51,9 +51,7 @@ async def find(
return await find_generic(paths,
texts,
opts,
find_core=partial(find_core,
accessor,
index=opts.index),
find_core=partial(find_core, accessor),
stat=partial(stat_core,
accessor,
index=opts.index))
@@ -67,7 +67,7 @@ async def narrow_scope(
search actually narrowed the set.
"""
key = scope_relative_key(paths[0])
file_count = count_scope_files(await index.entries(), key)
file_count = count_scope_files(accessor.tree, key)
query = search_query(pattern,
fixed_string) if pattern is not None else None
literal = (pattern is not None
+2
View File
@@ -75,6 +75,7 @@ async def readdir(
continue
is_dir = it.get("type") == "folder"
filename = it["name"]
sha1 = it.get("sha1")
entry = IndexEntry(
id=it["id"],
name=filename,
@@ -82,6 +83,7 @@ async def readdir(
remote_time=it.get("modified_at") or "",
vfs_name=filename,
size=None if is_dir else it.get("size"),
extra={"sha1": sha1} if sha1 else {},
)
entries.append((filename, entry, is_dir))
+14 -2
View File
@@ -41,15 +41,23 @@ def _stat_from_item(item: dict[str, Any]) -> FileStat:
extra={"box_id": item["id"]},
)
remote_time = item.get("modified_at") or ""
# Box returns the content sha1 in the same listing, so prefer it:
# it is content-addressed, where modified_at cannot tell two writes
# in the same second apart and does not move at all on a re-upload
# of identical bytes.
sha1 = item.get("sha1") or None
return FileStat(
name=vfs_name,
size=item.get("size"),
type=guess_type(vfs_name),
modified=remote_time,
fingerprint=remote_time or None,
fingerprint=sha1 or remote_time or None,
extra={
"box_id": item["id"],
"resource_type": rt,
**({
"sha1": sha1
} if sha1 else {}),
},
)
@@ -106,14 +114,18 @@ async def stat(
modified=result.entry.remote_time,
extra={"box_id": result.entry.id},
)
sha1 = result.entry.extra.get("sha1")
return FileStat(
name=result.entry.vfs_name or result.entry.name,
size=result.entry.size,
type=guess_type(result.entry.vfs_name),
modified=result.entry.remote_time,
fingerprint=result.entry.remote_time or None,
fingerprint=sha1 or result.entry.remote_time or None,
extra={
"box_id": result.entry.id,
"resource_type": result.entry.resource_type,
**({
"sha1": sha1
} if sha1 else {}),
},
)
+39
View File
@@ -0,0 +1,39 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from functools import partial
from mirage.accessor.box import BoxAccessor
from mirage.core.box.readdir import readdir
from mirage.core.box.stat import stat
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.walk import ReaddirWalk
def build_delta_hook(accessor: BoxAccessor) -> DeltaHook:
"""Build the Box delta hook.
Box keys its tree by folder id and has no recursive listing, so the
pull is one ``/folders/{id}/items`` request per directory. Box does
offer an account-wide ``/events`` feed, which is the cheaper signal
and belongs in a push receiver, not here.
Fingerprints on ``modified_at``, which is what Box stat reports.
Args:
accessor (BoxAccessor): Backend handle.
"""
return ListingDeltaHook(
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
+132
View File
@@ -0,0 +1,132 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import os
from collections.abc import AsyncIterator
from pathlib import Path
from mirage.accessor.disk import DiskAccessor
from mirage.core.timeutil import epoch_to_iso
from mirage.types import PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
def _resolve(root: Path, path: str) -> Path:
relative = path.lstrip("/")
resolved = (root / relative).resolve()
resolved.relative_to(root)
return resolved
def _reraise(error: OSError) -> None:
"""Fail the walk on a directory it could not read.
``os.walk`` swallows every listing error by default, which for a
snapshot differ means an unreadable subtree is indistinguishable
from an empty one: it diffs into a DELETE for every child, then a
CREATE for each when access comes back. Absence is the one error
that is genuinely a DELETE, and the caller drops it.
Args:
error (OSError): The failure ``os.walk`` was about to discard.
"""
raise error
def _walk_sync(root: Path,
path: str) -> list[tuple[str, bool, str | None, int | None]]:
"""Collect (mount-relative path, is_dir, mtime, size) under a path.
Runs in a worker thread; ``os.walk`` and ``stat`` are blocking.
Symlinks are not followed, matching every other disk walk in the
repo and keeping a link loop from hanging the poll.
Args:
root (Path): Mount root on the local filesystem.
path (str): Mount-relative directory to walk.
"""
start = _resolve(root, path)
out: list[tuple[str, bool, str | None, int | None]] = []
for dirpath, dirnames, filenames in os.walk(start, onerror=_reraise):
current = Path(dirpath)
for name in dirnames:
relative = (current / name).relative_to(root).as_posix()
out.append(("/" + relative, True, None, None))
for name in filenames:
full = current / name
relative = full.relative_to(root).as_posix()
try:
info = full.lstat()
except FileNotFoundError:
# Same rule one entry down: a file that vanished between
# the listing and the stat is a DELETE the next pull
# reports, an unreadable one is not.
continue
out.append(("/" + relative, False, epoch_to_iso(info.st_mtime),
info.st_size))
return out
class DiskWalk:
"""Recursive ``os.walk`` feeding the generic listing differ.
Reads the filesystem directly, never through mirage's caches, as
the DeltaHook contract requires. Fingerprints on mtime, the same
value ``disk`` stat reports, so an editor that rewrites identical
bytes still registers as an UPDATE. That is the local filesystem's
own resolution, not a mirage choice: nothing cheaper than hashing
every file can tell those two apart.
"""
def __init__(self, accessor: DiskAccessor) -> None:
"""Args:
accessor (DiskAccessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
prefix = mount_prefix_of(root.virtual, root.resource_path)
try:
found = await asyncio.to_thread(_walk_sync, self._accessor.root,
root.mount_path)
except FileNotFoundError:
return
for relative, is_dir, modified, size in found:
virtual = (prefix.rstrip("/") + relative if prefix else relative)
if is_dir:
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
continue
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=stat_fingerprint(None, modified, size),
size=size,
modified=modified)
def build_delta_hook(accessor: DiskAccessor) -> DeltaHook:
"""Build the disk delta hook.
Args:
accessor (DiskAccessor): Backend handle.
"""
return ListingDeltaHook(DiskWalk(accessor))
+113
View File
@@ -0,0 +1,113 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
from mirage.accessor.dropbox import DropboxAccessor
from mirage.core.dropbox._client import DropboxApiError
from mirage.core.dropbox.api import list_folder
from mirage.core.dropbox.paths import dropbox_path_of
from mirage.types import PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
class DropboxWalk:
"""One recursive ``list_folder`` feeding the generic listing differ.
Reads the account directly, never through mirage's caches, as the
DeltaHook contract requires.
Fingerprints on ``content_hash``, Dropbox's own content digest, so
an upload of identical bytes is correctly reported as no change;
``rev`` is the fallback, and it moves on any write.
Dropbox also offers a cursor: the same endpoint returns one, and
``list_folder/continue`` replays only what changed since. That is a
faster pull, not a more correct one, and it cannot replace this
walk, because the server may invalidate a cursor at any time and
the only answer to that is a full listing. When the fast path is
added it belongs behind ``pull``, with this walk as its reset path.
"""
def __init__(self, accessor: DropboxAccessor) -> None:
"""Args:
accessor (DropboxAccessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
accessor = self._accessor
prefix = mount_prefix_of(root.virtual, root.resource_path)
api_root = dropbox_path_of(accessor, root)
try:
found = await list_folder(accessor.token_manager,
api_root,
recursive=True)
except DropboxApiError as exc:
# list_folder 409s on a missing path and on a file operand;
# either way there is nothing under this root to report.
if exc.status == 409:
return
raise
# Dropbox paths are case-insensitive: `path_display` carries the
# server's casing while `root_path` carries the user's, so a
# configured `/team` whose displayed path is `/Team` matched
# nothing and every event landed outside the watch scope. The
# comparison folds case; the slice keeps the server's casing for
# everything below the root, and is safe because `path_lower` is
# `path_display` lowercased, same length.
base = accessor.root_path
folded = base.lower()
for entry in found:
display = entry.get("path_display") or entry.get("path_lower")
if not display:
continue
relative = display[len(base):] if base and display.lower(
).startswith(folded) else display
relative = relative.strip("/")
if not relative:
continue
virtual = (prefix.rstrip("/") + "/" + relative if prefix else "/" +
relative)
if entry.get(".tag") == "folder":
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
continue
modified = entry.get("server_modified") or entry.get(
"client_modified") or None
size = entry.get("size")
size = size if isinstance(size, int) else None
version = entry.get("content_hash") or entry.get("rev")
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=stat_fingerprint(
version, modified, size),
size=size,
modified=modified)
def build_delta_hook(accessor: DropboxAccessor) -> DeltaHook:
"""Build the Dropbox delta hook.
Args:
accessor (DropboxAccessor): Backend handle.
"""
return ListingDeltaHook(DropboxWalk(accessor))
+43
View File
@@ -0,0 +1,43 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from functools import partial
from mirage.accessor.gdrive import GDriveAccessor
from mirage.core.gdrive.readdir import readdir
from mirage.core.gdrive.stat import stat
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.walk import ReaddirWalk
def build_delta_hook(accessor: GDriveAccessor) -> DeltaHook:
"""Build the Google Drive delta hook.
Drive addresses files by id and returns ``parents`` rather than
paths, so a whole-corpus ``files.list`` would still have to rebuild
the tree before it could name anything; the walk descends per folder
instead, which is the same shape ``find`` already uses here.
Fingerprints on ``modifiedTime``, which is what Drive stat reports.
Drive also has ``changes.list`` with a page token, an account-wide
feed that is cheaper than any walk and would have to be filtered
back down to the watch root; that belongs behind ``pull`` as a fast
path, with this walk as its reset.
Args:
accessor (GDriveAccessor): Backend handle.
"""
return ListingDeltaHook(
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
+19 -17
View File
@@ -13,7 +13,6 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.github import GitHubAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
emit_start_path, keep,
start_basename, tree_has_empty)
@@ -38,8 +37,6 @@ async def find(
path_pattern: str | None = None,
empty: bool = False,
tree: PredNode | None = None,
*,
index: IndexCacheStore = NULL_INDEX,
) -> list[str]:
base = path.mount_path.strip("/")
base_depth = 0 if base == "" else base.count("/") + 1
@@ -54,30 +51,33 @@ async def find(
empty=empty)
start_kind = "d" if base == "" else None
start_size = 0
start_remote_time = ""
has_child = False
entries = await index.entries()
# The git tree, not the index: it is keyed repo-relative, which is
# the space this walk compares in, and it stays right however the
# mount keys its index. Mirrors TypeScript's find.
entries = accessor.tree
non_empty_dirs: set[str] = set()
if tree_has_empty(tree):
# Every intermediate folder is itself an entry, so marking direct
# parents is enough to classify all non-empty directories.
# Tree keys carry no leading slash, so a top-level entry has no
# parent segment at all; rsplit would hand back the entry itself
# and mark every top-level directory non-empty.
non_empty_dirs = {
entry_path.rsplit("/", 1)[0] or "/"
entry_path.rsplit("/", 1)[0] if "/" in entry_path else ""
for entry_path in entries
}
for entry_path in sorted(entries):
p = entry_path.lstrip("/")
for p in sorted(entries):
if p == base:
meta = entries[entry_path]
start_kind = "d" if meta.resource_type == "folder" else "f"
meta = entries[p]
start_kind = "d" if meta.type == "tree" else "f"
start_size = meta.size or 0
start_remote_time = meta.remote_time
continue
if base and not p.startswith(base + "/"):
continue
has_child = True
entry_meta = entries[entry_path]
is_dir = entry_meta.resource_type == "folder"
entry_meta = entries[p]
is_dir = entry_meta.type == "tree"
full_path = "/" + p
depth = p.count("/") + 1 - base_depth
if maxdepth is not None and depth > maxdepth:
@@ -86,8 +86,7 @@ async def find(
size = 0 if is_dir else (entry_meta.size or 0)
is_empty = None
if tree_has_empty(tree):
is_empty = (entry_path.rstrip("/") not in non_empty_dirs
if is_dir else size == 0)
is_empty = (p not in non_empty_dirs if is_dir else size == 0)
entry = FindEntry(key=full_path,
name=p.rsplit("/", 1)[-1],
kind="d" if is_dir else "f",
@@ -99,11 +98,14 @@ async def find(
continue
if max_size is not None and size > max_size:
continue
if not matches_mtime(entry_meta.remote_time, mtime_min, mtime_max):
# A git tree carries no timestamp, so every entry's mtime is
# unknown, which -mtime excludes. That is what the index answered
# too: nothing here ever set IndexEntry.remote_time.
if not matches_mtime("", mtime_min, mtime_max):
continue
results.append(full_path)
if ((start_kind is not None or has_child)
and matches_mtime(start_remote_time, mtime_min, mtime_max)):
and matches_mtime("", mtime_min, mtime_max)):
root_kind = start_kind or "d"
emit_start_path(
results,
+7 -5
View File
@@ -18,9 +18,10 @@ from mirage.accessor.github import GitHubAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore, LookupStatus
from mirage.core.github._client import github_get
from mirage.core.github.config import GitHubConfig
from mirage.core.github.tree import refill_index
from mirage.core.github.tree import ensure_live_index, refill_index
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_prefix_of
async def read_bytes(config: GitHubConfig, owner: str, repo: str,
@@ -42,9 +43,9 @@ async def read(
index: IndexCacheStore = NULL_INDEX,
) -> bytes:
virtual = path_spec.virtual
path = path_spec.mount_path
key = "/" + path.strip("/")
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
key = path_spec.mount_path.strip("/")
key = prefix + "/" + key if key else prefix or "/"
# Freshness is tracked per directory, never per entry, so a blob's row
# is exactly as fresh as its parent's listing and `get` can never
# report staleness of its own. The parent is therefore the probe:
@@ -53,11 +54,12 @@ async def read(
# miss is not a probe either -- against a live index it is a real
# absence, and refetching the whole tree on every ENOENT costs a
# recursive-tree call per miss.
await ensure_live_index(accessor, index, prefix)
if not accessor.truncated:
cut = key.rfind("/")
parent = key[:cut] if cut > 0 else "/"
if (await index.list_dir(parent)).status == LookupStatus.EXPIRED:
await refill_index(accessor, index)
await refill_index(accessor, index, prefix)
result = await index.get(key)
if result.status == LookupStatus.NOT_FOUND or result.entry is None:
raise enoent(virtual)
+42 -28
View File
@@ -16,7 +16,8 @@ import logging
from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
LookupStatus)
from mirage.core.github.tree import fetch_dir_tree, refill_index
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
refill_index)
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_prefix_of
@@ -29,47 +30,51 @@ async def readdir(accessor,
index: IndexCacheStore = NULL_INDEX) -> list[str]:
virtual = path_spec.virtual
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
path = path_spec.directory if path_spec.pattern else path_spec.virtual
if prefix and path.startswith(prefix):
rest = path[len(prefix):]
if prefix.endswith("/") or rest == "" or rest.startswith("/"):
path = rest or "/"
path = path.rstrip("/") or "/"
listing = await index.list_dir(path)
path = (path_spec.dir if path_spec.pattern else path_spec).mount_path
key = path.strip("/")
virtual_key = prefix + "/" + key if key else prefix or "/"
await ensure_live_index(accessor, index, prefix)
listing = await index.list_dir(virtual_key)
# The index is the whole listing here, not a cache in front of one, so
# an *expired* answer means the tree aged out, not that the path is
# gone. Refetch once and ask again. A NOT_FOUND against a live index
# is a real absence and must not cost a tree fetch.
if listing.status == LookupStatus.EXPIRED and not accessor.truncated:
if await refill_index(accessor, index):
listing = await index.list_dir(path)
if await refill_index(accessor, index, prefix):
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
if prefix and listing.entries and not listing.entries[0].startswith(
prefix):
return [prefix + e for e in listing.entries]
return listing.entries
if listing.status == LookupStatus.NOT_FOUND:
if accessor.truncated:
return await _fallback_readdir(accessor, path, index, virtual,
prefix)
return await _fallback_readdir(accessor, virtual_key, index,
virtual, prefix)
raise enoent(virtual)
return []
async def _fallback_readdir(
accessor,
path: str,
virtual_key: str,
index: IndexCacheStore,
virtual: str,
prefix: str = "",
prefix: str,
) -> list[str]:
"""Per-directory tree fetch when recursive tree was truncated."""
parent_sha = await _resolve_dir_sha(accessor, path, index)
"""Per-directory tree fetch when recursive tree was truncated.
Args:
accessor (GitHubAccessor): backend handle.
virtual_key (str): Mount-absolute directory key.
index (IndexCacheStore): the mount's index.
virtual (str): Virtual path, for the error message.
prefix (str): the mount prefix, which the descent has to strip to
walk the repository's own path segments.
"""
parent_sha = await _resolve_dir_sha(accessor, virtual_key, index, prefix)
if parent_sha is None:
raise enoent(virtual)
entries = await fetch_dir_tree(accessor.config, accessor.owner,
accessor.repo, parent_sha)
norm = "/" + path.strip("/")
norm = virtual_key.rstrip("/")
child_keys: list[str] = []
dir_entries: list[tuple[str, IndexEntry]] = []
for entry in entries:
@@ -85,24 +90,33 @@ async def _fallback_readdir(
child_keys.append(child_path)
await index.set_dir(norm, dir_entries)
log.debug("fallback readdir populated %d entries for %s", len(entries),
path)
virtual_keys = sorted((prefix + k if prefix else k) for k in child_keys)
return virtual_keys
virtual_key)
return sorted(child_keys)
async def _resolve_dir_sha(accessor, path: str,
index: IndexCacheStore) -> str | None:
async def _resolve_dir_sha(accessor,
virtual_key: str,
index: IndexCacheStore,
prefix: str = "") -> str | None:
"""Get the tree SHA for a directory path.
Walks from root if needed, fetching per-directory trees.
Args:
accessor (GitHubAccessor): backend handle.
virtual_key (str): Mount-absolute directory key.
index (IndexCacheStore): the mount's index.
prefix (str): the mount prefix the keys are built against.
"""
norm = "/" + path.strip("/")
norm = virtual_key.rstrip("/") or "/"
result = await index.get(norm)
if result.entry is not None:
return result.entry.id
parts = norm.strip("/").split("/")
stem = prefix.rstrip("/")
rest = norm[len(stem):] if stem and norm.startswith(stem) else norm
parts = [p for p in rest.strip("/").split("/") if p]
current_sha = accessor.ref
current_path = ""
current_path = stem
for part in parts:
entries = await fetch_dir_tree(accessor.config, accessor.owner,
accessor.repo, current_sha)
+12 -10
View File
@@ -12,7 +12,7 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.cache.index import IndexEntry
from mirage.core.github.tree_entry import TreeEntry
from mirage.types import PathSpec
from mirage.utils.key_prefix import mount_prefix_of
@@ -45,24 +45,26 @@ def is_repo_root(key: str) -> bool:
return key in ("", "/")
def count_scope_files(entries: dict[str, IndexEntry], key: str) -> int:
"""Count indexed files under a repo-relative scope key.
def count_scope_files(tree: dict[str, TreeEntry], key: str) -> int:
"""Count files under a repo-relative scope key.
Counted off the git tree rather than the index, mirroring
TypeScript's: the tree keys are repo-relative with no leading slash,
which is the space ``key`` is already in.
Args:
entries (dict[str, IndexEntry]): Index entries keyed by repo-relative
path with a leading slash.
tree (dict[str, TreeEntry]): The recursive git tree.
key (str): Repo-relative scope key from :func:`scope_relative_key`.
Returns:
int: Number of file entries at or below the scope.
"""
if is_repo_root(key):
return sum(1 for e in entries.values() if e.resource_type == "file")
norm = "/" + key.strip("/")
return sum(1 for e in tree.values() if e.type == "blob")
norm = key.strip("/")
prefix = norm + "/"
return sum(
1 for p, e in entries.items()
if e.resource_type == "file" and (p == norm or p.startswith(prefix)))
return sum(1 for p, e in tree.items()
if e.type == "blob" and (p == norm or p.startswith(prefix)))
def should_use_search(
+4 -10
View File
@@ -29,19 +29,13 @@ async def stat(accessor,
index: IndexCacheStore = NULL_INDEX) -> FileStat:
virtual = path_spec.virtual
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
path = path_spec.virtual
if prefix and path.startswith(prefix):
rest = path[len(prefix):]
if prefix.endswith("/") or rest == "" or rest.startswith("/"):
path = rest or "/"
if path == "/" or path == "":
rel = path_spec.mount_path.strip("/")
if not rel:
return FileStat(name="/", type=FileType.DIRECTORY)
key = "/" + path.strip("/") if path.strip("/") else "/"
key = prefix + "/" + rel if prefix else "/" + rel
result = await index.get(key)
if result.entry is None:
parent_idx = key.rsplit("/", 1)[0] or "/"
parent_path = (prefix + parent_idx) if prefix else parent_idx
parent_path = key.rsplit("/", 1)[0] or "/"
try:
await _readdir(
accessor,
+89 -16
View File
@@ -17,7 +17,8 @@ from collections import defaultdict
from datetime import datetime, timedelta, timezone
from typing import Any
from mirage.cache.index import NULL_INDEX, IndexCacheStore, IndexEntry
from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
LookupStatus)
from mirage.core.github._client import github_get
from mirage.core.github.config import GitHubConfig
from mirage.core.github.tree_entry import TreeEntry
@@ -101,28 +102,40 @@ async def fetch_dir_tree(
def index_rows(
tree: dict[str, TreeEntry]
) -> tuple[dict[str, IndexEntry], dict[str, list[str]]]:
tree: dict[str, TreeEntry],
prefix: str) -> tuple[dict[str, IndexEntry], dict[str, list[str]]]:
"""Turn a git tree into the index's entry and children tables.
Shared so the mount's initial seed and a later refill build the same
rows; TypeScript keeps its twin in this module too (`populateIndex`).
Keyed by mount-absolute path, the way every other backend keys its
index, so the shared cache machinery can spell an eviction without
knowing which backend it is talking to. The tree itself stays
repo-relative; ``prefix`` is what lifts it.
Shared so the mount's seed and a later refill build the same rows;
TypeScript keeps its twin in this module too (`populateIndex`).
Args:
tree (dict[str, TreeEntry]): the recursive tree, keyed by
repo-relative path.
prefix (str): the mount prefix ("/gh"), or "" for a root mount.
Returns:
tuple[dict[str, IndexEntry], dict[str, list[str]]]: entries keyed
by absolute path, and each directory's sorted children.
by mount-absolute path, and each directory's sorted children.
"""
stem = prefix.rstrip("/")
dirs: dict[str, list[tuple[str, IndexEntry]]] = defaultdict(list)
# The repository root always exists, so it gets a row even when the
# tree is empty. Without it an empty repository is byte for byte a
# dropped index, and `ensure_live_index` would refetch on every read
# of one; `ls` on it also read as ENOENT rather than as empty.
dirs[stem or "/"] = []
for path, entry in tree.items():
parts = path.rsplit("/", 1)
if len(parts) == 2:
parent, name = "/" + parts[0], parts[1]
parent, name = stem + "/" + parts[0], parts[1]
else:
parent, name = "/", parts[0]
parent, name = stem or "/", parts[0]
dirs[parent].append(
(name,
IndexEntry(
@@ -132,20 +145,31 @@ def index_rows(
size=entry.size,
)))
entries = {
("/" + parent.strip("/") + "/" + name).replace("//", "/"): entry
(parent.rstrip("/") + "/" + name): entry
for parent, rows in dirs.items()
for name, entry in rows
}
children = {
parent:
sorted(("/" + parent.strip("/") + "/" + name).replace("//", "/")
for name, _ in rows)
parent: sorted(parent.rstrip("/") + "/" + name for name, _ in rows)
for parent, rows in dirs.items()
}
return entries, children
async def refill_index(accessor, index: IndexCacheStore) -> bool:
def seed_index(accessor, index: IndexCacheStore, prefix: str) -> None:
"""Write the accessor's tree into ``index`` under ``prefix``.
Args:
accessor (GitHubAccessor): the mount's accessor, holding the tree.
index (IndexCacheStore): the index to seed.
prefix (str): the mount prefix the keys are built against.
"""
entries, children = index_rows(accessor.tree, prefix)
index.seed(entries, children,
datetime.now(timezone.utc) + timedelta(days=365))
async def refill_index(accessor, index: IndexCacheStore, prefix: str) -> bool:
"""Refetch the recursive tree and re-seed the index from it.
The mount fetches the whole tree once and seeds the index with it, so
@@ -160,6 +184,7 @@ async def refill_index(accessor, index: IndexCacheStore) -> bool:
accessor (GitHubAccessor): the mount's accessor, holding the
config and the ref to refetch.
index (IndexCacheStore): the index to re-seed.
prefix (str): the mount prefix the index keys are built against.
Returns:
bool: whether a refill happened; False when there is no index to
@@ -170,7 +195,55 @@ async def refill_index(accessor, index: IndexCacheStore) -> bool:
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
accessor.repo, accessor.ref)
accessor.truncated = truncated
entries, children = index_rows(tree)
index.seed(entries, children,
datetime.now(timezone.utc) + timedelta(days=365))
accessor.tree = tree
seed_index(accessor, index, prefix)
return True
async def ensure_live_index(accessor, index: IndexCacheStore,
prefix: str) -> bool:
"""Refetch when the index holds no listing at all.
Every reader here treats a missing listing as a real absence, which
is right against a *live* index and wrong against one that was never
filled or has been dropped, and invalidation drops rather than
expires: `invalidate_dir` removes the directory's row outright, so
the EXPIRED probe each reader already runs never fires. An external
change (a watch event is the only thing that invalidates a mount
with no write ops) therefore left the whole mount answering ENOENT
permanently, since the seeded expiry is a year out.
The root listing is what tells live from not, in one lookup and no
request: the tree is written whole, so while the index is live every
directory has a row and the mount root always does. One refill makes
it live again, so this cannot cost a fetch per miss, which is what
kept the readers from probing on absence in the first place.
Not live always **refetches**, and never re-seeds the tree the mount
was built with. That tree is only true at build time: the first read
of a mount can come long after it, and reusing it then served an
index built from a repository five external writes ago. It is still
what ``accessor.tree`` starts as, so find and du have something to
read before any listing happens, and every refill reseats it.
Args:
accessor (GitHubAccessor): the mount's accessor.
index (IndexCacheStore): the index to check and fill.
prefix (str): the mount prefix the index keys are built against.
Returns:
bool: whether the index was filled.
"""
if index is NULL_INDEX:
return False
# The liveness probe comes before anything on the accessor, so a live
# index still answers every read without one.
if (await index.list_dir(prefix.rstrip("/") or "/")).status \
!= LookupStatus.NOT_FOUND:
return False
# A truncated tree is not the whole listing, so the invariant this
# rests on does not hold and readdir's per-directory fallback owns
# the miss instead.
if accessor.truncated:
return False
return await refill_index(accessor, index, prefix)
+96
View File
@@ -0,0 +1,96 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
from mirage.accessor.github import GitHubAccessor
from mirage.core.github.tree import fetch_tree
from mirage.types import PathSpec, WalkEntry
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.errors import IncompleteWalkError
class GitHubWalk:
"""One recursive git tree fetch feeding the generic listing differ.
``GET /git/trees/{ref}?recursive=1`` returns every path in the
repository with its object sha, so a pull is one request whatever
the repository's shape, and the fingerprint is the sha itself.
That is the strongest fingerprint any mirage backend has: git is
content-addressed, so identical bytes have an identical sha and a
rewrite that changes nothing correctly reports nothing.
A mount is pinned to one ref, so what this detects is that ref
moving. Nothing is reported while the branch sits still, however
much is pushed elsewhere in the repository.
"""
def __init__(self, accessor: GitHubAccessor) -> None:
"""Args:
accessor (GitHubAccessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
Raises:
IncompleteWalkError: The repository is large enough that
GitHub truncated the recursive tree, so the listing is
not a complete picture of the ref and diffing it would
report every unlisted path as deleted.
"""
accessor = self._accessor
prefix = mount_prefix_of(root.virtual, root.resource_path)
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
accessor.repo, accessor.ref)
if truncated:
raise IncompleteWalkError(
f"github tree for {accessor.owner}/{accessor.repo}"
f"@{accessor.ref} was truncated; cannot diff a partial tree")
# A complete tree for the ref is exactly what the accessor holds,
# and find/du/grep's scope counter read it directly. Discarding it
# here left them answering from the tree the mount was built with
# until an unrelated read happened to refill the index, so a pull
# that reported a CREATE was followed by a find that could not see
# the file.
accessor.tree = tree
stem = root.mount_path.strip("/")
base = (stem + "/") if stem else ""
for entry in tree.values():
if base and not entry.path.startswith(base):
continue
virtual = (prefix.rstrip("/") + "/" +
entry.path if prefix else "/" + entry.path)
if entry.type == "tree":
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
continue
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=entry.sha,
size=entry.size)
def build_delta_hook(accessor: GitHubAccessor) -> DeltaHook:
"""Build the GitHub delta hook.
Args:
accessor (GitHubAccessor): Backend handle.
"""
return ListingDeltaHook(GitHubWalk(accessor))
+89
View File
@@ -0,0 +1,89 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
from mirage.accessor.gridfs import GridFSAccessor
from mirage.core.gridfs._client import (_prefix, _strip_prefix, iter_latest,
prefix_query)
from mirage.core.timeutil import to_iso_z
from mirage.types import PathSpec, WalkEntry
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.walk import synth_dirs
class GridFSWalk:
"""One flat ``fs.files`` aggregation feeding the generic differ.
GridFS stores a flat filename space, so the whole subtree comes back
from a single prefix query rather than one round trip per directory,
and the aggregation already reduces each filename to its newest
revision. Reads the collection directly, never through mirage's
caches, as the DeltaHook contract requires.
Fingerprints on the revision's ObjectId, which is exactly what
``gridfs`` stat reports. That is an exact version: every upload
mints a new document, so a rewrite always moves it and an untouched
file never does.
"""
def __init__(self, accessor: GridFSAccessor) -> None:
"""Args:
accessor (GridFSAccessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
config = self._accessor.config
prefix = mount_prefix_of(root.virtual, root.resource_path)
pfx = _prefix(root.mount_path, config)
files: list[str] = []
markers: list[str] = []
async for doc in iter_latest(self._accessor, prefix_query(pfx)):
filename = doc["filename"]
relative = _strip_prefix(filename, config)
virtual = (prefix.rstrip("/") + "/" +
relative.lstrip("/") if prefix else "/" +
relative.lstrip("/"))
if filename.endswith("/"):
# A directory marker, the same convention readdir reads
# as an immediate child directory.
markers.append(virtual.rstrip("/"))
continue
files.append(virtual)
upload = doc.get("uploadDate")
modified = to_iso_z(upload) if upload else None
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=str(doc["_id"]),
size=doc["length"],
modified=modified)
for entry in synth_dirs(root.virtual, files, markers):
yield entry
def build_delta_hook(accessor: GridFSAccessor) -> DeltaHook:
"""Build the GridFS delta hook.
Args:
accessor (GridFSAccessor): Backend handle.
"""
return ListingDeltaHook(GridFSWalk(accessor))
+33
View File
@@ -0,0 +1,33 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor._hf import _HfAccessor
from mirage.core.opendal.watch import OpendalWalk
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
def build_delta_hook(accessor: _HfAccessor) -> DeltaHook:
"""Build the delta hook shared by every Hugging Face resource.
One recursive tree listing per pull, fingerprinted on the Hub's
ETag. A mount pinned to an immutable ``revision`` cannot report a
change, because the revision it reads is frozen by definition; the
hook is only meaningful against a moving ref such as ``main``.
Args:
accessor (_HfAccessor): Backend handle for any of the four hf
resources.
"""
return ListingDeltaHook(OpendalWalk(accessor))
+5 -60
View File
@@ -12,75 +12,20 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
from opendal.exceptions import NotFound
from mirage.accessor.nextcloud import NextcloudAccessor
from mirage.types import PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_prefix_of
from mirage.core.opendal.watch import OpendalWalk
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
class NextcloudWalk:
"""Recursive WebDAV walk feeding the generic listing differ.
Reads through the opendal operator directly (a single recursive
PROPFIND), never through mirage's caches, as the DeltaHook contract
requires. Fingerprints use mirage's default (native ETag when the
listing carries one, mtime|size otherwise).
"""
def __init__(self, accessor: NextcloudAccessor) -> None:
"""Args:
accessor (NextcloudAccessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
prefix = mount_prefix_of(root.virtual, root.resource_path)
base = root.resource_path.strip("/")
list_path = base + "/" if base else "/"
op = self._accessor.operator()
try:
entries = await op.list(list_path, recursive=True)
except NotFound:
return
async for entry in entries:
relative = entry.path
if not relative or relative == list_path:
continue
is_dir = relative.endswith("/")
resource_rel = relative.rstrip("/")
virtual = (prefix.rstrip("/") + "/" +
resource_rel if prefix else "/" + resource_rel)
meta = entry.metadata
if is_dir:
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
continue
modified = meta.last_modified.isoformat() \
if meta and meta.last_modified else None
size = meta.content_length if meta else None
fingerprint = stat_fingerprint(meta.etag if meta else None,
modified, size)
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=fingerprint,
size=size,
modified=modified)
NextcloudWalk = OpendalWalk
def build_delta_hook(accessor: NextcloudAccessor) -> DeltaHook:
"""Build the Nextcloud delta hook.
One recursive PROPFIND per pull, fingerprinted on the WebDAV ETag.
Args:
accessor (NextcloudAccessor): Backend handle.
"""
return ListingDeltaHook(NextcloudWalk(accessor))
return ListingDeltaHook(OpendalWalk(accessor))
+42
View File
@@ -0,0 +1,42 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from functools import partial
from mirage.accessor.onedrive import OneDriveAccessor
from mirage.core.onedrive.readdir import readdir
from mirage.core.onedrive.stat import stat
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.walk import ReaddirWalk
def build_delta_hook(accessor: OneDriveAccessor) -> DeltaHook:
"""Build the OneDrive delta hook.
Fingerprints on the item's ``cTag``, which Graph moves only when the
content changes (``eTag`` also moves on a metadata edit), so a
rename does not read as a content change.
Graph has a native ``/delta`` feed with a resumable token, which is
cheaper than this walk and reports deletes directly. It is a fast
path rather than a replacement: Graph can answer ``resyncRequired``
at any time, and the only response to that is a full listing. When
it is added it belongs behind ``pull``, with this walk as its reset.
Args:
accessor (OneDriveAccessor): Backend handle.
"""
return ListingDeltaHook(
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
+18
View File
@@ -0,0 +1,18 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.core.opendal.types import OperatorAccessor
from mirage.core.opendal.watch import OpendalWalk, build_delta_hook
__all__ = ["OperatorAccessor", "OpendalWalk", "build_delta_hook"]
+28
View File
@@ -0,0 +1,28 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Protocol
import opendal
class OperatorAccessor(Protocol):
"""An accessor that reaches its backend through an opendal operator.
``NextcloudAccessor`` and the four ``_HfAccessor`` subclasses satisfy
this structurally, which is what lets one walk serve both.
"""
def operator(self) -> opendal.AsyncOperator:
...
+116
View File
@@ -0,0 +1,116 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
import opendal
from opendal.exceptions import NotFound
from opendal.types import Metadata
from mirage.core.opendal.types import OperatorAccessor
from mirage.types import PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
class OpendalWalk:
"""Recursive opendal list feeding the generic listing differ.
Reads through the operator directly (one recursive LIST), never
through mirage's caches, as the DeltaHook contract requires.
Fingerprints use mirage's default: the native ETag when the listing
carries one, ``mtime|size`` otherwise.
Some opendal services answer LIST without per-entry metadata (the hf
lister does; WebDAV's PROPFIND does not), which would leave every
file unfingerprinted and reduce detection to create/delete. When a
listed file carries no metadata at all, one ``stat`` per affected
file fills the gap, mirroring what the hf readdir already does for
sizes. A backend whose lister is complete never pays for it.
"""
def __init__(self, accessor: OperatorAccessor) -> None:
"""Args:
accessor (OperatorAccessor): Backend handle exposing an
opendal operator.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
prefix = mount_prefix_of(root.virtual, root.resource_path)
base = root.resource_path.strip("/")
list_path = base + "/" if base else "/"
op = self._accessor.operator()
try:
entries = await op.list(list_path, recursive=True)
except NotFound:
return
async for entry in entries:
relative = entry.path
if not relative or relative == list_path:
continue
is_dir = relative.endswith("/")
resource_rel = relative.rstrip("/")
virtual = (prefix.rstrip("/") + "/" +
resource_rel if prefix else "/" + resource_rel)
if is_dir:
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
continue
meta = entry.metadata
if meta is None or (meta.etag is None
and meta.last_modified is None
and meta.content_length is None):
meta = await self._stat(op, resource_rel)
modified = meta.last_modified.isoformat() \
if meta and meta.last_modified else None
size = meta.content_length if meta else None
fingerprint = stat_fingerprint(meta.etag if meta else None,
modified, size)
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=fingerprint,
size=size,
modified=modified)
async def _stat(self, op: opendal.AsyncOperator,
key: str) -> Metadata | None:
"""Fetch one entry's metadata when the listing omitted it.
Args:
op (opendal.AsyncOperator): Open operator.
key (str): Operator-relative key of the entry.
"""
try:
return await op.stat(key)
except NotFound:
# Deleted between the listing and the stat; the next pull
# reports the DELETE from the snapshot diff.
return None
def build_delta_hook(accessor: OperatorAccessor) -> DeltaHook:
"""Build a delta hook for any opendal-backed accessor.
Args:
accessor (OperatorAccessor): Backend handle exposing an opendal
operator.
"""
return ListingDeltaHook(OpendalWalk(accessor))
+101
View File
@@ -0,0 +1,101 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
from mirage.accessor.s3 import S3Accessor
from mirage.core.s3._client import (_client_kwargs, _key, _strip_prefix,
async_session)
from mirage.core.timeutil import to_iso_z
from mirage.types import PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.walk import synth_dirs
class S3Walk:
"""Recursive ``list_objects_v2`` feeding the generic listing differ.
One paginated LIST with no Delimiter covers the whole subtree, so a
pull costs one request per 1000 keys rather than one per directory.
Reads the bucket directly, never through mirage's caches, as the
DeltaHook contract requires.
Fingerprints on the object's ETag, which for a single-part upload is
the MD5 of the content, so an overwrite with identical bytes is
correctly reported as no change. Multipart ETags are a digest of the
part digests, which still changes with the content.
"""
def __init__(self, accessor: S3Accessor) -> None:
"""Args:
accessor (S3Accessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
config = self._accessor.config
prefix = mount_prefix_of(root.virtual, root.resource_path)
stem = _key(root.mount_path, config).rstrip("/")
base = (stem + "/") if stem else ""
files: list[str] = []
markers: list[str] = []
session = async_session(config)
async with session.client(**_client_kwargs(config)) as client:
paginator = client.get_paginator("list_objects_v2")
async for page in paginator.paginate(Bucket=config.bucket,
Prefix=stem):
for obj in page.get("Contents") or []:
okey = obj["Key"]
if not (okey == stem or okey.startswith(base)):
continue
relative = _strip_prefix(okey, config)
virtual = (prefix.rstrip("/") + "/" +
relative.lstrip("/") if prefix else "/" +
relative.lstrip("/"))
if okey.endswith("/"):
# A directory marker: mirage's own mkdir writes
# one. It carries an ETag and a size, but it is
# not a file, so synth_dirs reports it instead.
markers.append(virtual.rstrip("/"))
continue
files.append(virtual)
last_mod = obj.get("LastModified")
modified = to_iso_z(last_mod) if last_mod else None
size = obj.get("Size")
etag = (obj.get("ETag") or "").strip('"') or None
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=stat_fingerprint(
etag, modified, size),
size=size,
modified=modified)
for entry in synth_dirs(root.virtual, files, markers):
yield entry
def build_delta_hook(accessor: S3Accessor) -> DeltaHook:
"""Build the delta hook shared by S3 and every S3-compatible alias.
Args:
accessor (S3Accessor): Backend handle.
"""
return ListingDeltaHook(S3Walk(accessor))
+36
View File
@@ -0,0 +1,36 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from functools import partial
from mirage.accessor.sharepoint import SharePointAccessor
from mirage.core.sharepoint.readdir import readdir
from mirage.core.sharepoint.stat import stat
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
from mirage.watch.walk import ReaddirWalk
def build_delta_hook(accessor: SharePointAccessor) -> DeltaHook:
"""Build the SharePoint delta hook.
Same Graph drive surface as OneDrive, so the same ``cTag``
fingerprint and the same standing offer of a native ``/delta`` fast
path; see ``mirage.core.onedrive.watch``.
Args:
accessor (SharePointAccessor): Backend handle.
"""
return ListingDeltaHook(
ReaddirWalk(partial(readdir, accessor), partial(stat, accessor)))
+111
View File
@@ -0,0 +1,111 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import AsyncIterator
import asyncssh
from mirage.accessor.ssh import SSHAccessor
from mirage.core.ssh._client import _abs
from mirage.core.ssh.config import SSHConfig
from mirage.core.timeutil import epoch_to_iso
from mirage.types import PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_prefix_of
from mirage.watch.base import DeltaHook
from mirage.watch.delta import ListingDeltaHook
async def _descend(
sftp: asyncssh.SFTPClient, config: SSHConfig,
path: str) -> AsyncIterator[tuple[str, bool, str | None, int | None]]:
"""Yield (mount-relative path, is_dir, mtime, size) under a path.
One ``readdir`` per directory, which is one round trip per
directory; SFTP has no recursive listing. Each entry already
carries its attributes, so no extra stat is needed.
Args:
sftp (asyncssh.SFTPClient): Open SFTP channel.
config (SSHConfig): Backend config, for the remote root.
path (str): Mount-relative directory to descend into.
"""
try:
listing = await sftp.readdir(_abs(config, path))
except asyncssh.SFTPNoSuchFile:
return
for entry in listing:
filename = (entry.filename.decode("utf-8") if isinstance(
entry.filename, bytes) else entry.filename)
if filename in (".", ".."):
continue
child = f"{path.rstrip('/')}/{filename}"
attrs = entry.attrs
if attrs.type == asyncssh.FILEXFER_TYPE_DIRECTORY:
yield child, True, None, None
async for row in _descend(sftp, config, child):
yield row
continue
modified = epoch_to_iso(
attrs.mtime) if attrs.mtime is not None else None
yield child, False, modified, attrs.size
class SSHWalk:
"""Recursive SFTP descent feeding the generic listing differ.
Reads the remote host directly, never through mirage's caches, as
the DeltaHook contract requires. Fingerprints on mtime, the same
value ``ssh`` stat reports.
Unlike the object stores, this costs one round trip per directory,
because SFTP has no recursive listing to ask for. Poll cadence
should account for the shape of the tree.
"""
def __init__(self, accessor: SSHAccessor) -> None:
"""Args:
accessor (SSHAccessor): Backend handle.
"""
self._accessor = accessor
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
accessor = self._accessor
prefix = mount_prefix_of(root.virtual, root.resource_path)
sftp = await accessor.sftp()
async for relative, is_dir, modified, size in _descend(
sftp, accessor.config, root.mount_path):
virtual = (prefix.rstrip("/") + relative if prefix else relative)
if is_dir:
yield WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
continue
yield WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=stat_fingerprint(None, modified, size),
size=size,
modified=modified)
def build_delta_hook(accessor: SSHAccessor) -> DeltaHook:
"""Build the SSH delta hook.
Args:
accessor (SSHAccessor): Backend handle.
"""
return ListingDeltaHook(SSHWalk(accessor))
+5
View File
@@ -20,12 +20,14 @@ from mirage.commands.builtin.box import COMMANDS as BOX_COMMANDS
from mirage.core.box._client import BoxTokenManager
from mirage.core.box.config import BoxConfig
from mirage.core.box.readdir import readdir
from mirage.core.box.watch import build_delta_hook
from mirage.ops.box import OPS as BOX_OPS
from mirage.resource.base import BaseResource
from mirage.resource.box.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir)
@@ -51,6 +53,9 @@ class BoxResource(BaseResource):
for op in BOX_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
+5
View File
@@ -33,6 +33,7 @@ from mirage.core.disk.stat import stat as disk_stat
from mirage.core.disk.stream import read_stream
from mirage.core.disk.truncate import truncate
from mirage.core.disk.unlink import unlink
from mirage.core.disk.watch import build_delta_hook
from mirage.core.disk.write import write_bytes
from mirage.ops.disk import OPS as DISK_OPS
from mirage.resource.base import BaseResource
@@ -40,6 +41,7 @@ from mirage.resource.disk.prompt import PROMPT
from mirage.types import CapacityResult, CapacityState, PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -91,6 +93,9 @@ class DiskResource(BaseResource):
# same directory are one store, however they were spelled.
return f"{self.name}:{self.root}"
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -24,12 +24,14 @@ from mirage.core.dropbox.rename import rename
from mirage.core.dropbox.rmdir import rmdir
from mirage.core.dropbox.stat import stat
from mirage.core.dropbox.unlink import unlink
from mirage.core.dropbox.watch import build_delta_hook
from mirage.core.dropbox.write import write_bytes
from mirage.resource.base import BaseResource
from mirage.resource.dropbox.config import DropboxConfig
from mirage.resource.dropbox.prompt import PROMPT
from mirage.types import ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir)
@@ -72,6 +74,9 @@ class DropboxResource(BaseResource):
for op in DROPBOX_VFS_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
return await _resolve_glob(self.accessor, paths, self._index)
+5
View File
@@ -16,12 +16,14 @@ from typing import Any
from mirage.accessor.gdrive import GDriveAccessor
from mirage.core.gdrive.readdir import readdir
from mirage.core.gdrive.watch import build_delta_hook
from mirage.core.google._client import TokenManager
from mirage.resource.base import BaseResource
from mirage.resource.gdrive.config import GoogleDriveConfig
from mirage.resource.gdrive.prompt import PROMPT
from mirage.types import ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir)
@@ -51,6 +53,9 @@ class GoogleDriveResource(BaseResource):
for op in GDRIVE_VFS_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
return await _resolve_glob(self.accessor, paths, index=self._index)
+7 -22
View File
@@ -12,20 +12,20 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from datetime import datetime, timedelta, timezone
from typing import Any
from mirage.accessor.github import GitHubAccessor
from mirage.cache.index import IndexConfig
from mirage.core.github.config import GitHubConfig
from mirage.core.github.readdir import readdir
from mirage.core.github.repo import fetch_default_branch
from mirage.core.github.tree import fetch_tree, index_rows
from mirage.core.github.tree import fetch_tree
from mirage.core.github.tree_entry import TreeEntry
from mirage.core.github.watch import build_delta_hook
from mirage.resource.base import BaseResource
from mirage.resource.github.prompt import PROMPT
from mirage.types import ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir)
@@ -73,14 +73,14 @@ class GitHubResource(BaseResource):
truncated (bool): whether GitHub truncated that tree, in
which case readdir falls back to per-directory fetches.
"""
super().__init__()
self.accessor = GitHubAccessor(config,
owner,
repo,
ref,
default_branch,
tree=tree,
truncated=truncated)
self._populate_index(tree)
super().__init__()
from mirage.commands.builtin.github import COMMANDS as _github_cmds
from mirage.ops.github import OPS as _github_vfs_ops
@@ -138,23 +138,8 @@ class GitHubResource(BaseResource):
tree,
truncated=truncated)
def _populate_index(self, tree: dict[str, TreeEntry]) -> None:
entries, children = index_rows(tree)
self._github_index_entries = entries
self._github_index_children = children
self._github_index_expiry = (datetime.now(timezone.utc) +
timedelta(days=365))
self._seed_github_index()
def _seed_github_index(self) -> None:
self._index.seed(self._github_index_entries,
self._github_index_children,
self._github_index_expiry)
def set_index(self, config: IndexConfig | None = None) -> None:
super().set_index(config)
if hasattr(self, "_github_index_entries"):
self._seed_github_index()
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
return await _resolve_glob(self.accessor, paths, self._index)
+5
View File
@@ -34,6 +34,7 @@ from mirage.core.gridfs.stat import stat as gridfs_stat
from mirage.core.gridfs.stream import range_read, read_stream
from mirage.core.gridfs.truncate import truncate
from mirage.core.gridfs.unlink import unlink
from mirage.core.gridfs.watch import build_delta_hook
from mirage.core.gridfs.write import write_bytes
from mirage.ops.gridfs import OPS as GRIDFS_OPS
from mirage.resource.base import BaseResource
@@ -41,6 +42,7 @@ from mirage.resource.gridfs.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -86,6 +88,9 @@ class GridFSResource(BaseResource):
for op in GRIDFS_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
from mirage.core.hf_buckets.stat import stat as hf_stat
from mirage.core.hf_buckets.stream import range_read, read_stream
from mirage.core.hf_buckets.unlink import unlink
from mirage.core.hf_buckets.watch import build_delta_hook
from mirage.core.hf_buckets.write import write_bytes
from mirage.ops.hf_buckets import OPS as HF_OPS
from mirage.resource.base import BaseResource
@@ -36,6 +37,7 @@ from mirage.resource.hf_buckets.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -78,6 +80,9 @@ class HfBucketsResource(BaseResource):
for op in HF_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
from mirage.core.hf_buckets.stat import stat as hf_stat
from mirage.core.hf_buckets.stream import range_read, read_stream
from mirage.core.hf_buckets.unlink import unlink
from mirage.core.hf_buckets.watch import build_delta_hook
from mirage.core.hf_buckets.write import write_bytes
from mirage.ops.hf_buckets import OPS as HF_OPS
from mirage.resource.base import BaseResource
@@ -36,6 +37,7 @@ from mirage.resource.hf_datasets.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -77,6 +79,9 @@ class HfDatasetsResource(BaseResource):
for op in HF_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
from mirage.core.hf_buckets.stat import stat as hf_stat
from mirage.core.hf_buckets.stream import range_read, read_stream
from mirage.core.hf_buckets.unlink import unlink
from mirage.core.hf_buckets.watch import build_delta_hook
from mirage.core.hf_buckets.write import write_bytes
from mirage.ops.hf_buckets import OPS as HF_OPS
from mirage.resource.base import BaseResource
@@ -36,6 +37,7 @@ from mirage.resource.hf_models.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -77,6 +79,9 @@ class HfModelsResource(BaseResource):
for op in HF_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -29,6 +29,7 @@ from mirage.core.hf_buckets.readdir import readdir
from mirage.core.hf_buckets.stat import stat as hf_stat
from mirage.core.hf_buckets.stream import range_read, read_stream
from mirage.core.hf_buckets.unlink import unlink
from mirage.core.hf_buckets.watch import build_delta_hook
from mirage.core.hf_buckets.write import write_bytes
from mirage.ops.hf_buckets import OPS as HF_OPS
from mirage.resource.base import BaseResource
@@ -36,6 +37,7 @@ from mirage.resource.hf_spaces.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -77,6 +79,9 @@ class HfSpacesResource(BaseResource):
for op in HF_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -33,6 +33,7 @@ from mirage.core.onedrive.stat import stat as onedrive_stat
from mirage.core.onedrive.stream import range_read, read_stream
from mirage.core.onedrive.truncate import truncate
from mirage.core.onedrive.unlink import unlink
from mirage.core.onedrive.watch import build_delta_hook
from mirage.core.onedrive.write import write_bytes
from mirage.ops.onedrive import OPS as ONEDRIVE_OPS
from mirage.resource.base import BaseResource
@@ -40,6 +41,7 @@ from mirage.resource.onedrive.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir)
@@ -91,6 +93,9 @@ class OneDriveResource(BaseResource):
for op in ONEDRIVE_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
+5
View File
@@ -34,6 +34,7 @@ from mirage.core.s3.stat import stat as s3_stat
from mirage.core.s3.stream import range_read, read_stream
from mirage.core.s3.truncate import truncate
from mirage.core.s3.unlink import unlink
from mirage.core.s3.watch import build_delta_hook
from mirage.core.s3.write import write_bytes
from mirage.ops.s3 import OPS as S3_OPS
from mirage.resource.base import BaseResource
@@ -41,6 +42,7 @@ from mirage.resource.s3.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -97,6 +99,9 @@ class S3Resource(BaseResource):
base = f"{self.name}:{cfg.endpoint_url or 'aws'}:{cfg.bucket}"
return f"{base}/{prefix}" if prefix else base
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
@@ -19,6 +19,7 @@ from mirage.core.sharepoint.stat import stat as sharepoint_stat
from mirage.core.sharepoint.stream import range_read, read_stream
from mirage.core.sharepoint.truncate import truncate
from mirage.core.sharepoint.unlink import unlink
from mirage.core.sharepoint.watch import build_delta_hook
from mirage.core.sharepoint.write import write_bytes
from mirage.ops.sharepoint import OPS as SHAREPOINT_OPS
from mirage.resource.base import BaseResource
@@ -26,6 +27,7 @@ from mirage.resource.sharepoint.prompt import PROMPT
from mirage.types import PathSpec, ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.utils.key_prefix import mount_key
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir)
@@ -77,6 +79,9 @@ class SharePointResource(BaseResource):
for op in SHAREPOINT_OPS:
self.register_op(op)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
if prefix:
paths = [
+5
View File
@@ -35,12 +35,14 @@ from mirage.core.ssh.stat import stat as ssh_stat
from mirage.core.ssh.stream import range_read, read_stream
from mirage.core.ssh.truncate import truncate
from mirage.core.ssh.unlink import unlink
from mirage.core.ssh.watch import build_delta_hook
from mirage.core.ssh.write import write_bytes
from mirage.ops.ssh import OPS as SSH_OPS
from mirage.resource.base import BaseResource
from mirage.resource.ssh.prompt import PROMPT
from mirage.types import ResourceName
from mirage.utils.glob_walk import make_resolve_glob
from mirage.watch.base import DeltaHook
_resolve_glob = make_resolve_glob(readdir, SCOPE_ERROR)
@@ -90,6 +92,9 @@ class SSHResource(BaseResource):
for ro in SSH_OPS:
self.register_op(ro)
def delta_hook(self) -> DeltaHook:
return build_delta_hook(self.accessor)
async def resolve_glob(self, paths, prefix: str = ""):
return await _resolve_glob(self.accessor, paths, self._index)
+11
View File
@@ -22,6 +22,17 @@ class QueueOverflowError(Exception):
"""
class IncompleteWalkError(Exception):
"""Raised when a delta walk could not see the whole watch root.
A snapshot diff reads every path the walk did not report as a
DELETE, so a partial listing does not degrade into fewer events, it
invents wrong ones. A hook that knows its listing was truncated
raises this instead, leaving the caller's checkpoint untouched so
the next pull can still succeed.
"""
class QueueClosed(Exception):
"""Terminal signal from ``WatchQueue.pop`` after ``close``.
+194
View File
@@ -0,0 +1,194 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import (AsyncIterator, Awaitable, Callable, Iterable,
Iterator)
from mirage.cache.index import IndexCacheStore
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.types import FileStat, FileType, PathSpec, WalkEntry
from mirage.utils.fingerprint import stat_fingerprint
from mirage.utils.key_prefix import mount_key, mount_prefix_of
ReaddirFn = Callable[[PathSpec, IndexCacheStore], Awaitable[list[str]]]
StatFn = Callable[[PathSpec, IndexCacheStore], Awaitable[FileStat]]
def _ancestors(stem: str, start: str, seen: set[str]) -> Iterator[WalkEntry]:
"""Emit ``start`` and each ancestor up to but excluding ``stem``.
Args:
stem (str): Watch root, the exclusive upper bound.
start (str): Deepest directory to emit.
seen (set[str]): Paths already emitted; mutated in place so a
shared prefix is reported once across calls.
"""
parent = start
while parent and parent != stem and parent not in seen:
seen.add(parent)
yield WalkEntry(virtual=parent, is_dir=True, fingerprint=None)
parent = parent.rsplit("/", 1)[0]
def synth_dirs(root: str, files: Iterable[str],
dirs: Iterable[str]) -> Iterator[WalkEntry]:
"""Directory rows a prefix store implies but does not store.
An object store has no directories: ``ls`` shows them because
readdir synthesizes them from the common prefixes of the keys, and a
walk feeding change detection has to synthesize the same ones, or a
consumer would see a file appear inside a directory that never
appeared.
``dirs`` carries prefixes the store does name explicitly (a
zero-byte marker key, which mirage's own ``mkdir`` writes), so an
empty directory is still reported. A prefix backed by both a marker
and children is reported once.
``root`` itself is never emitted; ``find``'s start-point rule
applies here too, the generic owns that row.
Args:
root (str): Watch root's virtual path.
files (Iterable[str]): Virtual paths of every file under the
root; each contributes its ancestor chain.
dirs (Iterable[str]): Virtual paths of explicitly stored
directories; each contributes itself and its ancestors.
"""
stem = root.rstrip("/")
seen: set[str] = set()
for path in dirs:
yield from _ancestors(stem, path.rstrip("/"), seen)
for path in files:
yield from _ancestors(stem, path.rsplit("/", 1)[0], seen)
def entry_of(virtual: str, stat: FileStat) -> WalkEntry:
"""One walk row built from a backend's own stat.
Args:
virtual (str): Entry's virtual path.
stat (FileStat): What the backend reported for it.
"""
if stat.type == FileType.DIRECTORY:
return WalkEntry(virtual=virtual, is_dir=True, fingerprint=None)
return WalkEntry(virtual=virtual,
is_dir=False,
fingerprint=stat_fingerprint(stat.fingerprint,
stat.modified, stat.size),
size=stat.size,
modified=stat.modified)
async def _stat_at(stat: StatFn, virtual: str, prefix: str,
index: IndexCacheStore) -> FileStat | None:
"""Stat one virtual path, or None when it has vanished.
Args:
stat (StatFn): Backend stat.
virtual (str): Absolute virtual path.
prefix (str): Mount prefix.
index (IndexCacheStore): Index the walk is populating.
"""
spec = PathSpec(virtual=virtual,
directory=virtual,
resolved=False,
resource_path=mount_key(virtual, prefix))
try:
return await stat(spec, index)
except FileNotFoundError:
# Removed between the readdir and the stat; the next pull
# reports the DELETE from the snapshot diff. Only absence is
# swallowed, an API error still propagates.
return None
async def _descend(readdir: ReaddirFn, stat: StatFn, spec: PathSpec,
index: IndexCacheStore,
prefix: str) -> AsyncIterator[WalkEntry]:
"""Yield every entry under one directory, depth first.
Args:
readdir (ReaddirFn): Backend readdir.
stat (StatFn): Backend stat.
spec (PathSpec): Directory to descend into.
index (IndexCacheStore): Index the walk is populating.
prefix (str): Mount prefix.
"""
try:
children = await readdir(spec, index)
except FileNotFoundError:
return
for child in children:
# Classification is stat's job, the same rule find's walk
# follows: the one in-band proof is a trailing slash on a cold
# listing, and the stat behind it is an index lookup against
# the readdir that just populated it, not another request.
trimmed = child.rstrip("/")
if child.endswith("/"):
yield WalkEntry(virtual=trimmed, is_dir=True, fingerprint=None)
is_dir = True
else:
found = await _stat_at(stat, trimmed, prefix, index)
if found is None:
continue
yield entry_of(trimmed, found)
is_dir = found.type == FileType.DIRECTORY
if is_dir:
child_spec = PathSpec(virtual=trimmed,
directory=trimmed,
resolved=False,
resource_path=mount_key(trimmed, prefix))
async for row in _descend(readdir, stat, child_spec, index,
prefix):
yield row
class ReaddirWalk:
"""Recursive readdir descent for a backend with no recursive listing.
Box, Google Drive and Microsoft Graph key their trees by opaque id,
so a child cannot be addressed without having listed its parent, and
none of them offers a whole-subtree listing. This walks them the way
``find`` does, one readdir per directory.
Each pull builds its **own** index and throws it away afterwards.
That is what keeps the DeltaHook contract: the index is not mirage's
read cache, so the walk cannot compare the cache to itself, and it
starts empty every time, so nothing carries over between pulls. It
still has to exist, because these backends resolve a path's id
through the index their parent's readdir populated; handing them
``NULL_INDEX`` makes every path below the root read as absent.
"""
def __init__(self, readdir: ReaddirFn, stat: StatFn) -> None:
"""Args:
readdir (ReaddirFn): Backend readdir, already bound to its
accessor.
stat (StatFn): Backend stat, already bound to its accessor.
"""
self._readdir = readdir
self._stat = stat
async def __call__(self, root: PathSpec) -> AsyncIterator[WalkEntry]:
"""Yield every entry under ``root``.
Args:
root (PathSpec): Watch root (mount-virtual path).
"""
prefix = mount_prefix_of(root.virtual, root.resource_path)
index = RAMIndexCacheStore()
async for entry in _descend(self._readdir, self._stat, root, index,
prefix):
yield entry
@@ -14,12 +14,13 @@
import pytest
from mirage.cache.index import IndexCacheStore, IndexEntry
from mirage.accessor.github import GitHubAccessor
from mirage.commands.builtin.github.du import _du_size
from mirage.commands.builtin.github.grep import grep
from mirage.commands.builtin.github.narrow import narrow_scope
from mirage.commands.builtin.github.rg import rg
from mirage.commands.config import CommandOpts
from mirage.core.github.tree_entry import TreeEntry
from mirage.io.stream import materialize
from mirage.types import PathSpec
from tests.fixtures.github_mock import MOCK_BLOBS
@@ -27,15 +28,6 @@ from tests.fixtures.github_mock import MOCK_BLOBS
_NGLOBALS = narrow_scope.__globals__
class EntryOnlyIndex(IndexCacheStore):
async def entries(self) -> dict[str, IndexEntry]:
return {
"/src/main.py":
IndexEntry(id="main", name="main.py", resource_type="file", size=7)
}
@pytest.fixture
def counting_read(monkeypatch):
reads: list[str] = []
@@ -63,8 +55,23 @@ def _subdir() -> PathSpec:
@pytest.mark.asyncio
async def test_du_uses_store_interface_not_ram_implementation():
assert await _du_size(EntryOnlyIndex(), _subdir()) == 7
async def test_du_sizes_from_the_git_tree():
# du reads the tree, not the index: the tree is keyed repo-relative,
# which is the space this comparison is in, and it stays right
# however the mount keys its index.
accessor = GitHubAccessor(None,
"acme",
"proj",
"main",
"main",
tree={
"src/main.py":
TreeEntry(path="src/main.py",
type="blob",
sha="main",
size=7)
})
assert await _du_size(accessor, _subdir()) == 7
@pytest.mark.asyncio
@@ -47,12 +47,23 @@ async def test_read_missing_path(github_env):
@pytest.mark.asyncio
async def test_read_empty_index(github_env):
async def test_read_empty_index_refills(github_env):
# An empty index is no knowledge, not knowledge of absence: the index
# here is the whole listing, and invalidation drops rows rather than
# expiring them, so reading a miss as ENOENT made an invalidated mount
# answer ENOENT forever.
accessor, _ = github_env
data = await github_read(accessor, PathSpec.from_str_path("/README.md"),
RAMIndexCacheStore())
assert b"Mock Repo" in data
@pytest.mark.asyncio
async def test_read_empty_index_still_enoent_off_tree(github_env):
accessor, _ = github_env
empty_index = RAMIndexCacheStore()
with pytest.raises(FileNotFoundError):
await github_read(accessor, PathSpec.from_str_path("/README.md"),
empty_index)
await github_read(accessor, PathSpec.from_str_path("/nonexistent.txt"),
RAMIndexCacheStore())
@pytest.mark.asyncio
+134
View File
@@ -0,0 +1,134 @@
import asyncio
import os
from pathlib import Path
import pytest
from mirage.accessor.disk import DiskAccessor
from mirage.core.disk.watch import DiskWalk, build_delta_hook
from mirage.types import FileChangeKind, PathSpec
def _accessor(root: Path) -> DiskAccessor:
return DiskAccessor(root)
def _root(virtual: str, resource_path: str) -> PathSpec:
return PathSpec(virtual=virtual,
directory=virtual,
resource_path=resource_path)
def _touch(root: Path, relative: str, body: bytes, mtime: float) -> None:
target = root / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(body)
os.utime(target, (mtime, mtime))
async def _collect(walk, root):
return [entry async for entry in walk(root)]
def test_walk_reports_files_and_directories(tmp_path):
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
_touch(tmp_path, "data/sub/deep.txt", b"deep", 1_700_000_000)
entries = asyncio.run(
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/data", "data")))
files = {e.virtual for e in entries if not e.is_dir}
dirs = {e.virtual for e in entries if e.is_dir}
assert files == {"/d/data/a.txt", "/d/data/sub/deep.txt"}
assert dirs == {"/d/data/sub"}
def test_walk_carries_size_and_mtime(tmp_path):
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
entries = asyncio.run(
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/data", "data")))
entry = next(e for e in entries if not e.is_dir)
assert entry.size == 5
assert entry.modified is not None
assert entry.fingerprint == f"{entry.modified}|5"
def test_missing_root_walks_empty(tmp_path):
entries = asyncio.run(
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/gone", "gone")))
assert entries == []
def test_baseline_then_create_update_delete(tmp_path):
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
_touch(tmp_path, "data/b.txt", b"beta", 1_700_000_000)
hook = build_delta_hook(_accessor(tmp_path))
root = _root("/d/data", "data")
first = asyncio.run(hook.pull(root, None))
assert first.changes == ()
_touch(tmp_path, "data/a.txt", b"gamma", 1_700_000_500)
_touch(tmp_path, "data/c.txt", b"new", 1_700_000_500)
(tmp_path / "data/b.txt").unlink()
second = asyncio.run(hook.pull(root, first.checkpoint))
by_path = {c.path.virtual: c.kind for c in second.changes}
assert by_path == {
"/d/data/a.txt": FileChangeKind.UPDATE,
"/d/data/c.txt": FileChangeKind.CREATE,
"/d/data/b.txt": FileChangeKind.DELETE,
}
def test_untouched_tree_reports_nothing(tmp_path):
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
hook = build_delta_hook(_accessor(tmp_path))
root = _root("/d/data", "data")
first = asyncio.run(hook.pull(root, None))
second = asyncio.run(hook.pull(root, first.checkpoint))
assert second.changes == ()
def test_new_directory_is_reported(tmp_path):
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
hook = build_delta_hook(_accessor(tmp_path))
root = _root("/d/data", "data")
first = asyncio.run(hook.pull(root, None))
(tmp_path / "data" / "fresh").mkdir()
second = asyncio.run(hook.pull(root, first.checkpoint))
assert [(c.kind, c.path.virtual) for c in second.changes
] == [(FileChangeKind.CREATE, "/d/data/fresh")]
def test_changed_path_carries_the_mount_framing(tmp_path):
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
hook = build_delta_hook(_accessor(tmp_path))
root = _root("/d/data", "data")
first = asyncio.run(hook.pull(root, None))
_touch(tmp_path, "data/a.txt", b"gamma", 1_700_000_500)
second = asyncio.run(hook.pull(root, first.checkpoint))
changed = second.changes[0].path
assert changed.virtual == "/d/data/a.txt"
assert changed.resource_path == "data/a.txt"
def test_missing_root_reports_nothing(tmp_path):
entries = asyncio.run(
_collect(DiskWalk(_accessor(tmp_path)), _root("/d/gone", "gone")))
assert entries == []
def test_unreadable_directory_aborts_rather_than_reporting_empty(tmp_path):
# An unreadable subtree is not an empty one. Swallowing the error
# diffs into a DELETE for every child, then a CREATE for each once
# access returns, so the walk fails and the checkpoint stands.
_touch(tmp_path, "data/a.txt", b"alpha", 1_700_000_000)
locked = tmp_path / "data" / "locked"
locked.mkdir()
(locked / "inner.txt").write_bytes(b"inner")
locked.chmod(0o000)
try:
with pytest.raises(PermissionError):
asyncio.run(
_collect(DiskWalk(_accessor(tmp_path)),
_root("/d/data", "data")))
finally:
locked.chmod(0o755)
+71
View File
@@ -0,0 +1,71 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import patch
import pytest
from mirage.accessor.dropbox import DropboxAccessor
from mirage.core.dropbox._client import DropboxTokenManager
from mirage.core.dropbox.watch import DropboxWalk
from mirage.resource.dropbox.config import DropboxConfig
from mirage.types import PathSpec
def _accessor(root_path: str) -> DropboxAccessor:
config = DropboxConfig(client_id="c",
client_secret="s",
refresh_token="r",
root_path=root_path)
return DropboxAccessor(config, DropboxTokenManager(config))
def _root() -> PathSpec:
return PathSpec(virtual="/m", directory="/m", resource_path="")
async def _collect(walk, root):
return [entry async for entry in walk(root)]
@pytest.mark.asyncio
async def test_server_casing_of_the_root_is_still_stripped() -> None:
# Dropbox paths are case-insensitive: path_display carries the
# server's casing and root_path the user's. Comparing them exactly
# left the root on the front of every virtual path, which put every
# event outside the watch scope and silently disabled delivery.
listing = [{
".tag": "file",
"path_display": "/Team/notes.txt",
"path_lower": "/team/notes.txt",
"size": 4,
"rev": "r1",
}]
with patch("mirage.core.dropbox.watch.list_folder", return_value=listing):
entries = await _collect(DropboxWalk(_accessor("/team")), _root())
assert [e.virtual for e in entries] == ["/m/notes.txt"]
@pytest.mark.asyncio
async def test_casing_below_the_root_is_preserved() -> None:
listing = [{
".tag": "file",
"path_display": "/Team/Notes/Report.TXT",
"path_lower": "/team/notes/report.txt",
"size": 4,
"rev": "r1",
}]
with patch("mirage.core.dropbox.watch.list_folder", return_value=listing):
entries = await _collect(DropboxWalk(_accessor("/team")), _root())
assert [e.virtual for e in entries] == ["/m/Notes/Report.TXT"]
+48 -64
View File
@@ -16,34 +16,32 @@ from datetime import datetime, timezone
import pytest
from mirage.cache.index import IndexCacheStore, IndexEntry
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.accessor.github import GitHubAccessor
from mirage.core.github.find import find
from mirage.core.github.tree_entry import TreeEntry
from mirage.types import PathSpec
from mirage.utils.key_prefix import mount_key
def _index() -> RAMIndexCacheStore:
index = RAMIndexCacheStore()
index._entries.update({
"/src":
IndexEntry(id="a", name="src", resource_type="folder", size=None),
"/src/main.py":
IndexEntry(id="b", name="main.py", resource_type="file", size=120),
"/src/utils":
IndexEntry(id="c", name="utils", resource_type="folder", size=None),
"/src/utils/helpers.py":
IndexEntry(id="d", name="helpers.py", resource_type="file", size=80),
"/README.md":
IndexEntry(id="e", name="README.md", resource_type="file", size=50),
})
return index
def _accessor() -> GitHubAccessor:
"""A mount whose git tree holds the fixture repository.
class EntryOnlyIndex(IndexCacheStore):
async def entries(self) -> dict[str, IndexEntry]:
return await _index().entries()
find reads the tree, not the index: the tree keys are repo-relative,
which is the space find compares in.
"""
tree = {
"src":
TreeEntry(path="src", type="tree", sha="a", size=None),
"src/main.py":
TreeEntry(path="src/main.py", type="blob", sha="b", size=120),
"src/utils":
TreeEntry(path="src/utils", type="tree", sha="c", size=None),
"src/utils/helpers.py":
TreeEntry(path="src/utils/helpers.py", type="blob", sha="d", size=80),
"README.md":
TreeEntry(path="README.md", type="blob", sha="e", size=50),
}
return GitHubAccessor(None, "acme", "proj", "main", "main", tree=tree)
def _spec(path: str, prefix: str = "") -> PathSpec:
@@ -54,55 +52,48 @@ def _spec(path: str, prefix: str = "") -> PathSpec:
@pytest.mark.asyncio
async def test_find_all_from_root():
results = await find(None, _spec("/"), index=_index())
results = await find(_accessor(), _spec("/"))
assert results == [
"/", "/README.md", "/src", "/src/main.py", "/src/utils",
"/src/utils/helpers.py"
]
@pytest.mark.asyncio
async def test_find_uses_store_interface_not_ram_implementation():
results = await find(None, _spec("/"), index=EntryOnlyIndex())
assert "/src/main.py" in results
@pytest.mark.asyncio
async def test_find_name_pattern():
results = await find(None, _spec("/"), name="*.py", index=_index())
results = await find(_accessor(), _spec("/"), name="*.py")
assert results == ["/src/main.py", "/src/utils/helpers.py"]
@pytest.mark.asyncio
async def test_find_type_directory():
results = await find(None, _spec("/src"), type="d", index=_index())
results = await find(_accessor(), _spec("/src"), type="d")
assert results == ["/src", "/src/utils"]
@pytest.mark.asyncio
async def test_find_type_file_under_subdir():
results = await find(None, _spec("/src"), type="f", index=_index())
results = await find(_accessor(), _spec("/src"), type="f")
assert results == ["/src/main.py", "/src/utils/helpers.py"]
@pytest.mark.asyncio
async def test_find_maxdepth():
results = await find(None, _spec("/src"), maxdepth=1, index=_index())
results = await find(_accessor(), _spec("/src"), maxdepth=1)
assert results == ["/src", "/src/main.py", "/src/utils"]
@pytest.mark.asyncio
async def test_find_mindepth():
results = await find(None, _spec("/src"), mindepth=2, index=_index())
results = await find(_accessor(), _spec("/src"), mindepth=2)
assert results == ["/src/utils/helpers.py"]
@pytest.mark.asyncio
async def test_find_strips_mount_prefix():
results = await find(None,
results = await find(_accessor(),
_spec("/github/src", prefix="/github"),
type="f",
index=_index())
type="f")
assert results == ["/src/main.py", "/src/utils/helpers.py"]
@@ -110,58 +101,51 @@ async def test_find_strips_mount_prefix():
async def test_find_size_filters():
# Directories contribute size 0 to -size, so the root is excluded
# under a positive minimum (#318).
results = await find(None, _spec("/"), min_size=100, index=_index())
results = await find(_accessor(), _spec("/"), min_size=100)
assert results == ["/src/main.py"]
@pytest.mark.asyncio
async def test_find_file_start_path():
results = await find(None, _spec("/src/main.py"), index=_index())
results = await find(_accessor(), _spec("/src/main.py"))
assert results == ["/src/main.py"]
@pytest.mark.asyncio
async def test_find_size_filters_file_start():
too_big = await find(None,
_spec("/src/main.py"),
max_size=50,
index=_index())
too_big = await find(_accessor(), _spec("/src/main.py"), max_size=50)
assert too_big == []
big_enough = await find(None,
_spec("/src/main.py"),
min_size=100,
index=_index())
big_enough = await find(_accessor(), _spec("/src/main.py"), min_size=100)
assert big_enough == ["/src/main.py"]
@pytest.mark.asyncio
async def test_find_mtime_filters_unknown_and_out_of_window_entries():
index = _index()
index._entries["/src/main.py"] = index._entries["/src/main.py"].model_copy(
update={"remote_time": "2026-07-15T12:00:00+00:00"})
async def test_find_mtime_excludes_every_entry_a_git_tree_has_no_times():
# A git tree carries no timestamps, so every entry's mtime is unknown
# and -mtime excludes it. This was already true through the index,
# which nothing ever gave a remote_time.
results = await find(
None,
_accessor(),
_spec("/"),
mtime_min=datetime(2026, 7, 15, tzinfo=timezone.utc).timestamp(),
mtime_max=datetime(2026, 7, 16, tzinfo=timezone.utc).timestamp(),
index=index,
)
assert results == ["/src/main.py"]
assert results == []
@pytest.mark.asyncio
async def test_find_empty_matches_empty_files_and_directories():
index = _index()
index._entries["/empty.txt"] = IndexEntry(id="empty-file",
name="empty.txt",
resource_type="file",
size=0)
index._entries["/empty-dir"] = IndexEntry(id="empty-dir",
name="empty-dir",
resource_type="folder")
accessor = _accessor()
accessor.tree["empty.txt"] = TreeEntry(path="empty.txt",
type="blob",
sha="empty-file",
size=0)
accessor.tree["empty-dir"] = TreeEntry(path="empty-dir",
type="tree",
sha="empty-dir",
size=None)
results = await find(None, _spec("/"), empty=True, index=index)
results = await find(accessor, _spec("/"), empty=True)
assert results == ["/empty-dir", "/empty.txt"]
+8
View File
@@ -58,6 +58,14 @@ def _index() -> RAMIndexCacheStore:
index._entries["/src/main.py"] = entry
index._children["/src"] = ["/src/main.py"]
index._expiry["/src"] = datetime.now(timezone.utc) + timedelta(days=365)
# The root row is what makes this a live index rather than a dropped
# one; without it every read here would be a refill, which is the
# distinction ensure_live_index draws.
index._entries["/src"] = IndexEntry(id="aaa",
name="src",
resource_type="folder")
index._children["/"] = ["/src"]
index._expiry["/"] = datetime.now(timezone.utc) + timedelta(days=365)
return index
+9 -8
View File
@@ -24,20 +24,21 @@ from mirage.utils.key_prefix import mount_key
@pytest.fixture
def entries():
"""A git tree, keyed repo-relative with no leading slash."""
def _f():
return SimpleNamespace(resource_type="file")
return SimpleNamespace(type="blob")
def _d():
return SimpleNamespace(resource_type="folder")
return SimpleNamespace(type="tree")
return {
"/README.md": _f(),
"/src": _d(),
"/src/main.py": _f(),
"/src/utils.py": _f(),
"/src/models": _d(),
"/src/models/user.py": _f(),
"README.md": _f(),
"src": _d(),
"src/main.py": _f(),
"src/utils.py": _f(),
"src/models": _d(),
"src/models/user.py": _f(),
}
+119 -1
View File
@@ -17,8 +17,12 @@ from unittest.mock import patch
import pytest
from mirage.accessor.github import GitHubAccessor
from mirage.cache.index import NULL_INDEX
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.core.github.config import GitHubConfig
from mirage.core.github.tree import fetch_dir_tree, fetch_tree
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
fetch_tree, index_rows)
from mirage.core.github.tree_entry import TreeEntry
@@ -133,3 +137,117 @@ async def test_fetch_tree_passes_params(mock_get, config):
repo="proj",
ref="v1",
)
def _tree_payload() -> dict:
return {
"truncated":
False,
"tree": [
{
"path": "data",
"type": "tree",
"sha": "t1",
"size": None
},
{
"path": "data/keep.txt",
"type": "blob",
"sha": "b1",
"size": 4
},
],
}
def _accessor(config):
tree = {
"data":
TreeEntry(path="data", type="tree", sha="t1", size=None),
"data/keep.txt":
TreeEntry(path="data/keep.txt", type="blob", sha="b1", size=4),
}
return GitHubAccessor(config, "acme", "proj", "main", "main", tree=tree)
@pytest.mark.asyncio
@patch("mirage.core.github.tree.github_get")
async def test_ensure_live_index_refetches_the_build_tree(mock_get, config):
# The build tree is only true at build time: a mount's first read can
# come long after it, so reusing it would key an index built from a
# repository several external writes ago.
mock_get.return_value = _tree_payload()
index = RAMIndexCacheStore(ttl=600)
accessor = _accessor(config)
assert await ensure_live_index(accessor, index, "/gh") is True
mock_get.assert_awaited_once()
assert (await index.list_dir("/gh/data")).entries == ["/gh/data/keep.txt"]
@pytest.mark.asyncio
@patch("mirage.core.github.tree.github_get")
async def test_ensure_live_index_refetches_a_dropped_listing(mock_get, config):
mock_get.return_value = _tree_payload()
index = RAMIndexCacheStore(ttl=600)
accessor = _accessor(config)
await ensure_live_index(accessor, index, "/gh")
# What invalidation does: drop the row rather than expire it, which
# is why the readers' EXPIRED probe never fires.
await index.invalidate_dir("/gh")
await index.invalidate_dir("/gh/data")
assert await ensure_live_index(accessor, index, "/gh") is True
assert mock_get.await_count == 2
assert (await index.list_dir("/gh/data")).entries == ["/gh/data/keep.txt"]
@pytest.mark.asyncio
@patch("mirage.core.github.tree.github_get")
async def test_ensure_live_index_leaves_a_live_index_alone(mock_get, config):
mock_get.return_value = _tree_payload()
index = RAMIndexCacheStore(ttl=600)
accessor = _accessor(config)
await ensure_live_index(accessor, index, "/gh")
mock_get.reset_mock()
assert await ensure_live_index(accessor, index, "/gh") is False
mock_get.assert_not_awaited()
@pytest.mark.asyncio
@patch("mirage.core.github.tree.github_get")
async def test_ensure_live_index_skips_a_truncated_tree(mock_get, config):
index = RAMIndexCacheStore(ttl=600)
accessor = _accessor(config)
accessor.truncated = True
assert await ensure_live_index(accessor, index, "/gh") is False
mock_get.assert_not_awaited()
@pytest.mark.asyncio
async def test_ensure_live_index_skips_the_null_index(config):
assert await ensure_live_index(_accessor(config), NULL_INDEX, "") is False
def test_index_rows_key_by_mount_absolute_path():
# Every other backend keys its index this way, which is what lets the
# shared CacheManager spell an eviction without knowing the backend.
tree = {
"data":
TreeEntry(path="data", type="tree", sha="t1", size=None),
"data/keep.txt":
TreeEntry(path="data/keep.txt", type="blob", sha="b1", size=4),
}
entries, children = index_rows(tree, "/gh")
assert sorted(entries) == ["/gh/data", "/gh/data/keep.txt"]
assert sorted(children) == ["/gh", "/gh/data"]
def test_index_rows_root_mount_keeps_bare_paths():
entries, children = index_rows(
{"a.txt": TreeEntry(path="a.txt", type="blob", sha="b", size=1)}, "")
assert sorted(entries) == ["/a.txt"]
assert sorted(children) == ["/"]
def test_index_rows_gives_an_empty_repo_a_root_row():
_entries, children = index_rows({}, "/gh")
assert children == {"/gh": []}
+86
View File
@@ -0,0 +1,86 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import patch
import pytest
from mirage.accessor.github import GitHubAccessor
from mirage.core.github.config import GitHubConfig
from mirage.core.github.tree_entry import TreeEntry
from mirage.core.github.watch import GitHubWalk
from mirage.types import PathSpec
from mirage.watch.errors import IncompleteWalkError
CONFIG = GitHubConfig(token="t")
def _accessor(tree: dict[str, TreeEntry]) -> GitHubAccessor:
return GitHubAccessor(config=CONFIG,
owner="acme",
repo="proj",
ref="main",
default_branch="main",
tree=tree)
def _root() -> PathSpec:
return PathSpec(virtual="/gh", directory="/gh", resource_path="")
def _entry(path: str, sha: str) -> TreeEntry:
return TreeEntry(path=path, type="blob", sha=sha, size=3)
async def _collect(walk, root):
return [entry async for entry in walk(root)]
@pytest.mark.asyncio
async def test_pull_refreshes_the_accessor_tree() -> None:
# find, du and grep's scope counter read accessor.tree directly, so a
# pull that fetched a newer tree and dropped it left them reporting the
# repository as it stood when the mount was built.
stale = {"a.txt": _entry("a.txt", "sha-a")}
fresh = {
"a.txt": _entry("a.txt", "sha-a"),
"b.txt": _entry("b.txt", "sha-b"),
}
accessor = _accessor(stale)
with patch("mirage.core.github.watch.fetch_tree",
return_value=(fresh, False)):
await _collect(GitHubWalk(accessor), _root())
assert accessor.tree == fresh
@pytest.mark.asyncio
async def test_truncated_tree_is_not_adopted() -> None:
# A partial tree would make find report the missing half as deleted.
stale = {"a.txt": _entry("a.txt", "sha-a")}
accessor = _accessor(stale)
with patch("mirage.core.github.watch.fetch_tree", return_value=({}, True)):
with pytest.raises(IncompleteWalkError):
await _collect(GitHubWalk(accessor), _root())
assert accessor.tree == stale
@pytest.mark.asyncio
async def test_walk_reports_blobs_with_their_sha() -> None:
tree = {"a.txt": _entry("a.txt", "sha-a")}
accessor = _accessor(tree)
with patch("mirage.core.github.watch.fetch_tree",
return_value=(tree, False)):
entries = await _collect(GitHubWalk(accessor), _root())
assert [(e.virtual, e.fingerprint)
for e in entries] == [("/gh/a.txt", "sha-a")]
+122
View File
@@ -0,0 +1,122 @@
import asyncio
from mirage.accessor.s3 import S3Accessor
from mirage.core.s3.watch import S3Walk, build_delta_hook
from mirage.resource.s3 import S3Config
from mirage.types import FileChangeKind, PathSpec
from tests.e2e.s3_mock import patch_s3_multi
BUCKET = "watch-bucket"
def _accessor(key_prefix: str | None = None) -> S3Accessor:
return S3Accessor(
S3Config(bucket=BUCKET,
region="us-east-1",
aws_access_key_id="fake",
aws_secret_access_key="fake",
key_prefix=key_prefix))
def _root(virtual: str, resource_path: str) -> PathSpec:
return PathSpec(virtual=virtual,
directory=virtual,
resource_path=resource_path)
async def _collect(walk, root):
return [entry async for entry in walk(root)]
def test_walk_yields_files_with_etag_fingerprints():
store = {BUCKET: {"data/a.txt": b"alpha", "data/b.txt": b"beta"}}
with patch_s3_multi(store):
entries = asyncio.run(
_collect(S3Walk(_accessor()), _root("/s3/data", "data")))
files = {e.virtual: e for e in entries if not e.is_dir}
assert set(files) == {"/s3/data/a.txt", "/s3/data/b.txt"}
assert files["/s3/data/a.txt"].size == 5
# ETag, not the mtime|size composite: the mock's LastModified is a
# constant, so a composite would collide across files of equal size.
assert "|" not in (files["/s3/data/a.txt"].fingerprint or "")
assert (files["/s3/data/a.txt"].fingerprint
!= files["/s3/data/b.txt"].fingerprint)
def test_walk_synthesizes_intermediate_directories():
store = {BUCKET: {"data/sub/deep/x.txt": b"x"}}
with patch_s3_multi(store):
entries = asyncio.run(
_collect(S3Walk(_accessor()), _root("/s3/data", "data")))
dirs = {e.virtual for e in entries if e.is_dir}
assert dirs == {"/s3/data/sub", "/s3/data/sub/deep"}
def test_walk_reports_an_explicit_marker_as_its_own_directory():
store = {BUCKET: {"data/empty/": b""}}
with patch_s3_multi(store):
entries = asyncio.run(
_collect(S3Walk(_accessor()), _root("/s3/data", "data")))
assert [e.virtual for e in entries if e.is_dir] == ["/s3/data/empty"]
assert not [e for e in entries if not e.is_dir]
def test_walk_strips_the_key_prefix():
store = {BUCKET: {"team/x/data/a.txt": b"alpha"}}
with patch_s3_multi(store):
entries = asyncio.run(
_collect(S3Walk(_accessor("team/x/")), _root("/s3/data", "data")))
assert [e.virtual for e in entries if not e.is_dir] == ["/s3/data/a.txt"]
def test_baseline_pull_reports_nothing_then_detects_a_write():
store = {BUCKET: {"data/a.txt": b"alpha"}}
hook = build_delta_hook(_accessor())
root = _root("/s3/data", "data")
with patch_s3_multi(store):
first = asyncio.run(hook.pull(root, None))
assert first.changes == ()
store[BUCKET]["data/a.txt"] = b"gamma"
store[BUCKET]["data/new.txt"] = b"new"
second = asyncio.run(hook.pull(root, first.checkpoint))
by_path = {c.path.virtual: c.kind for c in second.changes}
assert by_path == {
"/s3/data/a.txt": FileChangeKind.UPDATE,
"/s3/data/new.txt": FileChangeKind.CREATE,
}
def test_delete_is_detected():
store = {BUCKET: {"data/a.txt": b"alpha", "data/b.txt": b"beta"}}
hook = build_delta_hook(_accessor())
root = _root("/s3/data", "data")
with patch_s3_multi(store):
first = asyncio.run(hook.pull(root, None))
del store[BUCKET]["data/b.txt"]
second = asyncio.run(hook.pull(root, first.checkpoint))
assert [(c.kind, c.path.virtual) for c in second.changes
] == [(FileChangeKind.DELETE, "/s3/data/b.txt")]
def test_same_bytes_rewritten_is_not_a_change():
store = {BUCKET: {"data/a.txt": b"alpha"}}
hook = build_delta_hook(_accessor())
root = _root("/s3/data", "data")
with patch_s3_multi(store):
first = asyncio.run(hook.pull(root, None))
store[BUCKET]["data/a.txt"] = b"alpha"
second = asyncio.run(hook.pull(root, first.checkpoint))
assert second.changes == ()
def test_changed_path_carries_the_mount_framing():
store = {BUCKET: {"data/a.txt": b"alpha"}}
hook = build_delta_hook(_accessor())
root = _root("/s3/data", "data")
with patch_s3_multi(store):
first = asyncio.run(hook.pull(root, None))
store[BUCKET]["data/a.txt"] = b"gamma"
second = asyncio.run(hook.pull(root, first.checkpoint))
changed = second.changes[0].path
assert changed.virtual == "/s3/data/a.txt"
assert changed.resource_path == "data/a.txt"
+25 -15
View File
@@ -37,6 +37,7 @@ _CORE_MODULES = [
"mirage.core.s3.mkdir",
"mirage.core.s3.create",
"mirage.core.s3.truncate",
"mirage.core.s3.watch",
]
@@ -59,6 +60,24 @@ def _mock_s3_error(code: str) -> Exception:
return exc
def _content_entry(key: str, data: bytes) -> dict[str, object]:
"""One Contents row, shaped like the real list_objects_v2.
Real S3 carries an ETag on every listed object, which is what the
watch walk fingerprints on, so the mock has to carry one too.
Args:
key (str): Object key.
data (bytes): Object content.
"""
return {
"Key": key,
"Size": len(data),
"LastModified": LAST_MODIFIED,
"ETag": f'"{hashlib.md5(data).hexdigest()}"',
}
def _paginate_directory(objects, prefix):
common_prefixes: set[str] = set()
contents: list[dict[str, object]] = []
@@ -67,21 +86,13 @@ def _paginate_directory(objects, prefix):
continue
relative = key[len(prefix):]
if not relative:
contents.append({
"Key": key,
"Size": len(data),
"LastModified": LAST_MODIFIED
})
contents.append(_content_entry(key, data))
continue
if "/" in relative:
child = relative.split("/", 1)[0]
common_prefixes.add(prefix + child + "/")
continue
contents.append({
"Key": key,
"Size": len(data),
"LastModified": LAST_MODIFIED
})
contents.append(_content_entry(key, data))
return {
"CommonPrefixes": [{
"Prefix": v
@@ -92,11 +103,10 @@ def _paginate_directory(objects, prefix):
def _paginate_flat(objects, prefix):
return {
"Contents": [{
"Key": k,
"Size": len(v),
"LastModified": LAST_MODIFIED
} for k, v in sorted(objects.items()) if k.startswith(prefix)]
"Contents": [
_content_entry(k, v) for k, v in sorted(objects.items())
if k.startswith(prefix)
]
}
+4
View File
@@ -131,5 +131,9 @@ def mock_github_api(monkeypatch):
_fetch_default_branch)
monkeypatch.setattr("mirage.resource.github.github.fetch_tree",
_fetch_tree)
# refill_index reads the name in its own module, and an empty index is
# a refill trigger now, so leaving this one real let a test reach the
# live API.
monkeypatch.setattr("mirage.core.github.tree.fetch_tree", _fetch_tree)
monkeypatch.setattr("mirage.core.github.read.read_bytes", _read_bytes)
monkeypatch.setattr("mirage.core.github.search.search_code", _search_code)
+42 -12
View File
@@ -28,16 +28,36 @@ OWNER = "test-owner"
REPO = "test-repo"
def _offline(tree: dict,
truncated: bool = False,
default_branch: str = "main"):
"""Patch every path that would reach the network.
``mirage.core.github.tree.fetch_tree`` is the second one: a read fills
the index by refetching, because the tree a mount was built with is
only true at build time.
Args:
tree (dict): The recursive tree to answer with.
truncated (bool): Whether to report it truncated.
default_branch (str): Branch the repo endpoint reports.
"""
return (patch("mirage.resource.github.github.fetch_default_branch",
return_value=default_branch),
patch("mirage.resource.github.github.fetch_tree",
return_value=(tree, truncated)),
patch("mirage.core.github.tree.fetch_tree",
return_value=(tree, truncated)))
async def _make_resource(ref: str = "main",
default_branch: str = "main",
tree: dict | None = None,
truncated: bool = False) -> GitHubResource:
if tree is None:
tree = {}
with patch("mirage.resource.github.github.fetch_default_branch",
return_value=default_branch), \
patch("mirage.resource.github.github.fetch_tree",
return_value=(tree, truncated)):
branch_p, build_p, core_p = _offline(tree, truncated, default_branch)
with branch_p, build_p, core_p:
return await GitHubResource.build(
config=CONFIG,
owner=OWNER,
@@ -137,13 +157,15 @@ async def test_stat_returns_sha_fingerprint() -> None:
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
}
resource = await _make_resource(tree=tree)
result = await stat(resource.accessor,
PathSpec.from_str_path("/src/main.py"), resource.index)
with _offline(tree)[2]:
result = await stat(resource.accessor,
PathSpec.from_str_path("/src/main.py"),
resource.index)
assert result.fingerprint == "abc123"
@pytest.mark.asyncio
async def test_replacing_index_preserves_preloaded_tree() -> None:
async def test_replacing_index_still_serves_the_tree() -> None:
tree = {
"src/main.py":
TreeEntry(path="src/main.py", type="blob", sha="abc123", size=100),
@@ -151,15 +173,19 @@ async def test_replacing_index_preserves_preloaded_tree() -> None:
resource = await _make_resource(tree=tree)
resource.set_index(IndexConfig())
result = await stat(resource.accessor,
PathSpec.from_str_path("/src/main.py"), resource.index)
# The fresh store is empty, which reads as not-live, so the next read
# fills it by refetching rather than reporting the path gone.
with _offline(tree)[2]:
result = await stat(resource.accessor,
PathSpec.from_str_path("/src/main.py"),
resource.index)
assert result.fingerprint == "abc123"
@pytest.mark.asyncio
async def test_stat_raises_when_path_not_in_tree() -> None:
resource = await _make_resource()
with pytest.raises(FileNotFoundError):
with _offline({})[2], pytest.raises(FileNotFoundError):
await stat(resource.accessor,
PathSpec.from_str_path("/nonexistent.py"), resource.index)
@@ -194,6 +220,10 @@ async def test_the_constructor_reaches_no_network() -> None:
branch.assert_not_called()
fetch.assert_not_called()
assert resource.accessor.default_branch == "main"
result = await stat(resource.accessor,
PathSpec.from_str_path("/src/main.py"), resource.index)
# A read is where the network comes in: the index is keyed by mount
# prefix, so filling it waits for a PathSpec and refetches then.
with _offline(tree)[2]:
result = await stat(resource.accessor,
PathSpec.from_str_path("/src/main.py"),
resource.index)
assert result.fingerprint == "abc123"
+182
View File
@@ -0,0 +1,182 @@
import asyncio
import pytest
from mirage.cache.index import IndexCacheStore
from mirage.types import FileStat, FileType, PathSpec
from mirage.watch.walk import ReaddirWalk, entry_of, synth_dirs
def _root(virtual: str, resource_path: str) -> PathSpec:
return PathSpec(virtual=virtual,
directory=virtual,
resource_path=resource_path)
def test_synth_dirs_emits_every_ancestor_excluding_the_root():
dirs = [
e.virtual for e in synth_dirs("/m/data", ["/m/data/a/b/c.txt"], [])
]
assert dirs == ["/m/data/a/b", "/m/data/a"]
def test_synth_dirs_reports_a_shared_prefix_once():
dirs = list(
synth_dirs("/m/data", ["/m/data/a/x.txt", "/m/data/a/y.txt"], []))
assert [e.virtual for e in dirs] == ["/m/data/a"]
def test_synth_dirs_emits_a_stored_directory_with_no_children():
dirs = list(synth_dirs("/m/data", [], ["/m/data/empty"]))
assert [e.virtual for e in dirs] == ["/m/data/empty"]
def test_synth_dirs_does_not_double_report_a_marker_with_children():
dirs = list(synth_dirs("/m/data", ["/m/data/a/x.txt"], ["/m/data/a"]))
assert [e.virtual for e in dirs] == ["/m/data/a"]
def test_synth_dirs_rows_are_directories_without_fingerprints():
dirs = list(synth_dirs("/m/data", ["/m/data/a/x.txt"], []))
assert all(e.is_dir and e.fingerprint is None for e in dirs)
def test_synth_dirs_emits_nothing_for_a_file_at_the_root():
assert list(synth_dirs("/m/data", ["/m/data/x.txt"], [])) == []
def test_entry_of_reports_a_directory_without_a_fingerprint():
entry = entry_of("/m/d", FileStat(name="d", type=FileType.DIRECTORY))
assert entry.is_dir is True
assert entry.fingerprint is None
def test_entry_of_prefers_the_backend_fingerprint():
stat = FileStat(name="f.txt", size=3, modified="T", fingerprint="etag-1")
assert entry_of("/m/f.txt", stat).fingerprint == "etag-1"
def test_entry_of_falls_back_to_the_composite():
stat = FileStat(name="f.txt", size=3, modified="T")
assert entry_of("/m/f.txt", stat).fingerprint == "T|3"
def _backend(tree: dict[str, dict]) -> ReaddirWalk:
async def readdir(spec: PathSpec, index: IndexCacheStore) -> list[str]:
node = tree.get(spec.virtual)
if node is None or "children" not in node:
raise FileNotFoundError(spec.virtual)
return list(node["children"])
async def stat(spec: PathSpec, index: IndexCacheStore) -> FileStat:
node = tree.get(spec.virtual)
if node is None:
raise FileNotFoundError(spec.virtual)
return node["stat"]
return ReaddirWalk(readdir, stat)
async def _collect(walk: ReaddirWalk, spec: PathSpec) -> list:
return [entry async for entry in walk(spec)]
def test_readdir_walk_descends_and_reports_leaves():
walk = _backend({
"/m/data": {
"children": ["/m/data/a.txt", "/m/data/sub"],
"stat": FileStat(name="data", type=FileType.DIRECTORY),
},
"/m/data/a.txt": {
"stat":
FileStat(name="a.txt", size=5, modified="T1", fingerprint="fp-a")
},
"/m/data/sub": {
"children": ["/m/data/sub/deep.txt"],
"stat": FileStat(name="sub", type=FileType.DIRECTORY),
},
"/m/data/sub/deep.txt": {
"stat":
FileStat(name="deep.txt",
size=4,
modified="T2",
fingerprint="fp-d")
},
})
entries = asyncio.run(_collect(walk, _root("/m/data", "data")))
assert [e.virtual for e in entries] == [
"/m/data/a.txt",
"/m/data/sub",
"/m/data/sub/deep.txt",
]
assert [e.virtual for e in entries if e.is_dir] == ["/m/data/sub"]
def test_readdir_walk_trusts_a_trailing_slash_without_a_stat():
# No stat entry for the child at all: the slash is the proof, so a
# stat would raise and the walk would lose the subtree.
walk = _backend({
"/m/data": {
"children": ["/m/data/sub/"],
"stat": FileStat(name="data", type=FileType.DIRECTORY),
},
"/m/data/sub": {
"children": [],
"stat": FileStat(name="sub", type=FileType.DIRECTORY),
},
})
entries = asyncio.run(_collect(walk, _root("/m/data", "data")))
assert [(e.virtual, e.is_dir) for e in entries] == [("/m/data/sub", True)]
def test_readdir_walk_skips_an_entry_that_vanished_mid_walk():
walk = _backend({
"/m/data": {
"children": ["/m/data/gone.txt", "/m/data/here.txt"],
"stat": FileStat(name="data", type=FileType.DIRECTORY),
},
"/m/data/here.txt": {
"stat":
FileStat(name="here.txt", size=1, modified="T", fingerprint="fp")
},
})
entries = asyncio.run(_collect(walk, _root("/m/data", "data")))
assert [e.virtual for e in entries] == ["/m/data/here.txt"]
def test_readdir_walk_treats_a_missing_root_as_empty():
entries = asyncio.run(_collect(_backend({}), _root("/m/gone", "gone")))
assert entries == []
def test_readdir_walk_propagates_a_non_absence_error():
async def readdir(spec: PathSpec, index: IndexCacheStore) -> list[str]:
raise PermissionError("rate limited")
async def stat(spec: PathSpec, index: IndexCacheStore) -> FileStat:
raise AssertionError("unreachable")
with pytest.raises(PermissionError):
asyncio.run(
_collect(ReaddirWalk(readdir, stat), _root("/m/data", "data")))
def test_readdir_walk_starts_from_an_empty_index_on_every_call():
seen: list[IndexCacheStore] = []
async def readdir(spec: PathSpec, index: IndexCacheStore) -> list[str]:
seen.append(index)
return ["/m/data/a.txt"] if spec.virtual == "/m/data" else []
async def stat(spec: PathSpec, index: IndexCacheStore) -> FileStat:
return FileStat(name="a.txt", size=1, modified="T", fingerprint="fp")
walk = ReaddirWalk(readdir, stat)
root = _root("/m/data", "data")
asyncio.run(_collect(walk, root))
asyncio.run(_collect(walk, root))
# Two pulls, two distinct index instances: nothing a pull learned
# can leak into the next one's snapshot.
assert seen[0] is not seen[-1]
@@ -26,10 +26,7 @@ import {
import { read as githubRead } from '@struktoai/mirage-core/core/github/read'
import { readdir as githubReaddir } from '@struktoai/mirage-core/core/github/readdir'
import { stat as githubStat } from '@struktoai/mirage-core/core/github/stat'
import {
buildTreeMap as githubBuildTreeMap,
populateIndex as githubPopulateIndex,
} from '@struktoai/mirage-core/core/github/tree'
import { buildTreeMap as githubBuildTreeMap } from '@struktoai/mirage-core/core/github/tree'
import { GITHUB_OPS } from '@struktoai/mirage-core/ops/github/index'
import type { RegisteredOp } from '@struktoai/mirage-core/ops/registry'
import type { Resource } from '@struktoai/mirage-core/resource/base'
@@ -87,8 +84,9 @@ export class GitHubResource implements Resource {
truncated,
tree: treeMap,
})
// Not seeded here: the index is keyed by mount prefix, which only a
// PathSpec knows, so the first read seeds it from the accessor's tree.
const index = new RAMIndexCacheStore({ ttl: 86_400 })
await githubPopulateIndex(index, tree)
return new GitHubResource(config, accessor, index)
}
+16 -10
View File
@@ -46,6 +46,19 @@ export class CacheManager {
this.cachesReads = cachesReads
}
/**
* Drop one directory's cached listing.
*
* Both spellings of the directory go, because a backend may have keyed
* it with or without its trailing slash and an eviction that hits no
* key is silent.
*/
private async evictDir(virtual: string): Promise<void> {
if (this.index === null) return
await this.index.invalidateDir(virtual)
await this.index.invalidateDir(virtual + '/')
}
private virtual(path: string | PathSpec): string {
let p = path instanceof PathSpec ? path.mountPath : path
if (!p.startsWith('/')) p = '/' + p
@@ -87,10 +100,7 @@ export class CacheManager {
if (this.cachesReads && this.fileCache !== null) {
await this.fileCache.remove(virtual)
}
if (this.index !== null) {
await this.index.invalidateDir(virtual)
await this.index.invalidateDir(virtual + '/')
}
await this.evictDir(virtual)
await this.invalidateParent(virtual)
}
@@ -109,8 +119,7 @@ export class CacheManager {
let parent = virtual.slice(0, Math.max(virtual.lastIndexOf('/'), 0))
while (parent !== '' && parent !== this.prefix) {
parent = parent.slice(0, Math.max(parent.lastIndexOf('/'), 0))
await this.index.invalidateDir(parent === '' ? '/' : parent)
await this.index.invalidateDir(parent + '/')
await this.evictDir(parent === '' ? '/' : parent)
}
}
@@ -133,10 +142,7 @@ export class CacheManager {
}
private async invalidateParent(virtual: string): Promise<void> {
if (this.index === null) return
const lastSlash = virtual.lastIndexOf('/')
const parent = lastSlash > 0 ? virtual.slice(0, lastSlash) : '/'
await this.index.invalidateDir(parent)
await this.index.invalidateDir(parent + '/')
await this.evictDir(lastSlash > 0 ? virtual.slice(0, lastSlash) : '/')
}
}
@@ -82,6 +82,7 @@ export async function readdir(
}
const isDir = it.type === 'folder'
const filename = it.name
const sha1 = typeof it.sha1 === 'string' && it.sha1 !== '' ? it.sha1 : null
const entry = new IndexEntry({
id: it.id,
name: filename,
@@ -89,6 +90,7 @@ export async function readdir(
remoteTime: it.modified_at ?? '',
vfsName: filename,
size: isDir ? null : typeof it.size === 'number' ? it.size : null,
extra: sha1 === null ? {} : { sha1 },
})
entries.push({ name: filename, entry, isDir })
}
+13 -5
View File
@@ -34,14 +34,19 @@ function statFromItem(item: BoxItem): FileStat {
})
}
const size = typeof item.size === 'number' ? item.size : null
// Box returns the content sha1 in the same listing, so prefer it: it
// is content-addressed, where modified_at cannot tell two writes in
// the same second apart and does not move at all on a re-upload of
// identical bytes.
const sha1 = typeof item.sha1 === 'string' && item.sha1 !== '' ? item.sha1 : null
const modified = item.modified_at ?? ''
return new FileStat({
name: vfsName,
size,
type: guessType(vfsName),
modified: item.modified_at ?? '',
fingerprint:
item.modified_at !== undefined && item.modified_at !== '' ? item.modified_at : null,
extra: { box_id: item.id, resource_type: rt },
modified,
fingerprint: sha1 ?? (modified !== '' ? modified : null),
extra: { box_id: item.id, resource_type: rt, ...(sha1 === null ? {} : { sha1 }) },
})
}
@@ -109,15 +114,18 @@ export async function stat(
extra: { box_id: result.entry.id },
})
}
const cachedSha1 = result.entry.extra.sha1
const sha1 = typeof cachedSha1 === 'string' && cachedSha1 !== '' ? cachedSha1 : null
return new FileStat({
name: result.entry.vfsName !== '' ? result.entry.vfsName : result.entry.name,
size: result.entry.size,
type: guessType(result.entry.vfsName),
modified: result.entry.remoteTime,
fingerprint: result.entry.remoteTime !== '' ? result.entry.remoteTime : null,
fingerprint: sha1 ?? (result.entry.remoteTime !== '' ? result.entry.remoteTime : null),
extra: {
box_id: result.entry.id,
resource_type: result.entry.resourceType,
...(sha1 === null ? {} : { sha1 }),
},
})
}
@@ -0,0 +1,40 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { BoxAccessor } from '../../accessor/box.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { FileStat, PathSpec } from '../../types.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { ReaddirWalk } from '../../watch/walk.ts'
import { readdir } from './readdir.ts'
import { stat } from './stat.ts'
/**
* Build the Box delta hook.
*
* Box keys its tree by folder id and has no recursive listing, so the pull is
* one `/folders/{id}/items` request per directory. Box does offer an
* account-wide `/events` feed, which is the cheaper signal and belongs in a
* push receiver, not here.
*
* Fingerprints on `modified_at`, which is what Box stat reports.
*/
export function buildDeltaHook(accessor: BoxAccessor): DeltaHook {
const walk = new ReaddirWalk(
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -0,0 +1,81 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it, vi } from 'vitest'
import type * as ApiModule from './api.ts'
vi.mock('./api.ts', async () => {
const actual = await vi.importActual<typeof ApiModule>('./api.ts')
return { ...actual, listFolder: vi.fn() }
})
import { DropboxAccessor } from '../../accessor/dropbox.ts'
import { PathSpec, type WalkEntry } from '../../types.ts'
import type { DropboxTokenManager } from './_client.ts'
import * as api from './api.ts'
import { DropboxWalk } from './watch.ts'
const STUB_TM = {} as DropboxTokenManager
function accessor(rootPath: string): DropboxAccessor {
return new DropboxAccessor({ tokenManager: STUB_TM, rootPath })
}
function root(): PathSpec {
return new PathSpec({ virtual: '/m', directory: '/m', resourcePath: '' })
}
async function collect(walk: DropboxWalk, at: PathSpec): Promise<WalkEntry[]> {
const out: WalkEntry[] = []
for await (const entry of walk.walk(at)) out.push(entry)
return out
}
describe('DropboxWalk root stripping', () => {
it('strips the root when the server casing differs', async () => {
// Dropbox paths are case-insensitive: path_display carries the
// server's casing and rootPath the user's. Comparing them exactly
// left the root on the front of every virtual path, which put every
// event outside the watch scope and silently disabled delivery.
vi.mocked(api.listFolder).mockResolvedValue([
{
'.tag': 'file',
id: 'id:1',
name: 'notes.txt',
path_display: '/Team/notes.txt',
path_lower: '/team/notes.txt',
size: 4,
rev: 'r1',
},
])
const entries = await collect(new DropboxWalk(accessor('/team')), root())
expect(entries.map((e) => e.virtual)).toEqual(['/m/notes.txt'])
})
it('preserves the casing below the root', async () => {
vi.mocked(api.listFolder).mockResolvedValue([
{
'.tag': 'file',
id: 'id:1',
name: 'Report.TXT',
path_display: '/Team/Notes/Report.TXT',
path_lower: '/team/notes/report.txt',
size: 4,
rev: 'r1',
},
])
const entries = await collect(new DropboxWalk(accessor('/team')), root())
expect(entries.map((e) => e.virtual)).toEqual(['/m/Notes/Report.TXT'])
})
})
@@ -0,0 +1,102 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { DropboxAccessor } from '../../accessor/dropbox.ts'
import type { PathSpec, WalkEntry } from '../../types.ts'
import { mountPrefixOf } from '../../utils/key_prefix.ts'
import { stripSlash } from '../../utils/slash.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { statFingerprint } from '../../watch/fingerprint.ts'
import { DropboxApiError } from './_client.ts'
import { listFolder } from './api.ts'
import { dropboxPathOf } from './paths.ts'
/**
* One recursive `list_folder` feeding the generic listing differ.
*
* Reads the account directly, never through mirage's caches, as the DeltaHook
* contract requires.
*
* Fingerprints on `content_hash`, Dropbox's own content digest, so an upload of
* identical bytes is correctly reported as no change; `rev` is the fallback,
* and it moves on any write.
*
* Dropbox also offers a cursor: the same endpoint returns one, and
* `list_folder/continue` replays only what changed since. That is a faster
* pull, not a more correct one, and it cannot replace this walk, because the
* server may invalidate a cursor at any time and the only answer to that is a
* full listing. When the fast path is added it belongs behind `pull`, with this
* walk as its reset path.
*/
export class DropboxWalk {
private readonly accessor: DropboxAccessor
constructor(accessor: DropboxAccessor) {
this.accessor = accessor
}
async *walk(root: PathSpec): AsyncGenerator<WalkEntry> {
const accessor = this.accessor
const prefix = mountPrefixOf(root.virtual, root.resourcePath)
const apiRoot = dropboxPathOf(accessor, root)
let found
try {
found = await listFolder(accessor.tokenManager, apiRoot, { recursive: true })
} catch (error) {
// list_folder 409s on a missing path and on a file operand;
// either way there is nothing under this root to report.
if (error instanceof DropboxApiError && error.status === 409) return
throw error
}
// Dropbox paths are case-insensitive: `path_display` carries the
// server's casing while `rootPath` carries the user's, so a configured
// `/team` whose displayed path is `/Team` matched nothing and every
// event landed outside the watch scope. The comparison folds case; the
// slice keeps the server's casing for everything below the root, and is
// safe because `path_lower` is `path_display` lowercased, same length.
const base = accessor.rootPath
const folded = base.toLowerCase()
for (const entry of found) {
const display = entry.path_display ?? entry.path_lower
if (display === undefined || display === '') continue
const trimmed =
base !== '' && display.toLowerCase().startsWith(folded)
? display.slice(base.length)
: display
const relative = stripSlash(trimmed)
if (relative === '') continue
const virtual = prefix !== '' ? `${prefix}/${relative}` : `/${relative}`
if (entry['.tag'] === 'folder') {
yield { virtual, isDir: true, fingerprint: null }
continue
}
const modified = entry.server_modified ?? entry.client_modified ?? null
const size = typeof entry.size === 'number' ? entry.size : null
const version = entry.content_hash ?? entry.rev ?? null
yield {
virtual,
isDir: false,
fingerprint: statFingerprint(version, modified, size),
size,
modified,
}
}
}
}
export function buildDeltaHook(accessor: DropboxAccessor): DeltaHook {
const walk = new DropboxWalk(accessor)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -0,0 +1,43 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { GDriveAccessor } from '../../accessor/gdrive.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { FileStat, PathSpec } from '../../types.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { ReaddirWalk } from '../../watch/walk.ts'
import { readdir } from './readdir.ts'
import { stat } from './stat.ts'
/**
* Build the Google Drive delta hook.
*
* Drive addresses files by id and returns `parents` rather than paths, so a
* whole-corpus `files.list` would still have to rebuild the tree before it
* could name anything; the walk descends per folder instead, which is the same
* shape `find` already uses here.
*
* Fingerprints on `modifiedTime`, which is what Drive stat reports. Drive also
* has `changes.list` with a page token, an account-wide feed that is cheaper
* than any walk and would have to be filtered back down to the watch root; that
* belongs behind `pull` as a fast path, with this walk as its reset.
*/
export function buildDeltaHook(accessor: GDriveAccessor): DeltaHook {
const walk = new ReaddirWalk(
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -63,10 +63,14 @@ function accessorFor(sha: string, probe: Probe): GitHubAccessor {
async function seeded(sha: string): Promise<RAMIndexCacheStore> {
const index = new RAMIndexCacheStore()
await populateIndex(index, [
{ path: 'src', type: 'tree' as const, sha: 'aaa' },
{ path: 'src/main.py', type: 'blob' as const, sha, size: 4 },
])
await populateIndex(
index,
{
src: { path: 'src', type: 'tree', sha: 'aaa', size: null },
'src/main.py': { path: 'src/main.py', type: 'blob', sha, size: 4 },
},
'',
)
return index
}
@@ -17,9 +17,9 @@ import { mountPrefixOf } from '../../utils/key_prefix.ts'
import type { GitHubAccessor } from '../../accessor/github.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { PathSpec } from '../../types.ts'
import { refillIndex } from './tree.ts'
import { ensureLiveIndex, refillIndex } from './tree.ts'
import { fetchBlob } from './_client.ts'
import { stripSlash } from '../../utils/slash.ts'
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
import { eisdir, enoent } from '../../utils/errors.ts'
function stripPrefix(path: PathSpec): string {
@@ -31,11 +31,6 @@ function stripPrefix(path: PathSpec): string {
return p
}
function indexKey(p: string): string {
const trimmed = stripSlash(p)
return trimmed === '' ? '/' : `/${trimmed}`
}
function parentKey(key: string): string {
const cut = key.lastIndexOf('/')
return cut <= 0 ? '/' : key.slice(0, cut)
@@ -46,9 +41,12 @@ export async function read(
path: PathSpec,
index?: IndexCacheStore,
): Promise<Uint8Array> {
const prefix = mountPrefixOf(path.virtual, path.resourcePath)
const p = stripPrefix(path)
if (index === undefined) throw enoent(path)
const key = indexKey(p)
const rel = stripSlash(p)
const key =
rel === '' ? (prefix === '' ? '/' : rstripSlash(prefix)) : `${rstripSlash(prefix)}/${rel}`
// Freshness is tracked per directory, never per entry, so a blob's row
// is exactly as fresh as its parent's listing and `get` can never report
// staleness of its own. The parent is therefore the probe: after a write
@@ -56,9 +54,10 @@ export async function read(
// sha, and reading it back served the old bytes. A miss is not a probe
// either -- against a live index it is a real absence, and refetching the
// whole tree on every ENOENT costs a recursive-tree call per miss.
await ensureLiveIndex(accessor, index, prefix)
if (!accessor.truncated) {
const parent = await index.listDir(parentKey(key))
if (parent.status === LookupStatus.EXPIRED) await refillIndex(accessor, index)
if (parent.status === LookupStatus.EXPIRED) await refillIndex(accessor, index, prefix)
}
const result = await index.get(key)
if (result.entry === undefined || result.entry === null) throw enoent(path)
@@ -42,9 +42,14 @@ function accessorFor(probe: { trees: number }): GitHubAccessor {
})
}
const TREE_MAP = {
src: { path: 'src', type: 'tree', sha: 'aaa', size: null },
'src/main.py': { path: 'src/main.py', type: 'blob', sha: 'bbb', size: 120 },
}
async function seeded(): Promise<RAMIndexCacheStore> {
const index = new RAMIndexCacheStore()
await populateIndex(index, TREE)
await populateIndex(index, TREE_MAP, '')
return index
}
@@ -18,9 +18,9 @@ import { LookupStatus } from '../../cache/index/config.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { PathSpec } from '../../types.ts'
import { fetchDirTree } from './_client.ts'
import { refillIndex } from './tree.ts'
import { ensureLiveIndex, refillIndex } from './tree.ts'
import { IndexEntry } from '../../cache/index/config.ts'
import { stripSlash } from '../../utils/slash.ts'
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
import { enoent } from '../../utils/errors.ts'
import { compareCodePoints } from '../../utils/sort.ts'
@@ -33,11 +33,6 @@ function stripPrefix(path: PathSpec): string {
return p
}
function normalizeKey(p: string): string {
const trimmed = stripSlash(p)
return trimmed === '' ? '/' : `/${trimmed}`
}
export async function readdir(
accessor: GitHubAccessor,
path: PathSpec,
@@ -47,9 +42,11 @@ export async function readdir(
throw enoent(path.virtual)
}
const prefix = mountPrefixOf(path.virtual, path.resourcePath)
const stripped = stripPrefix(path)
const key = normalizeKey(stripped)
const rel = stripSlash(stripPrefix(path))
const key =
rel === '' ? (prefix === '' ? '/' : rstripSlash(prefix)) : `${rstripSlash(prefix)}/${rel}`
await ensureLiveIndex(accessor, index, prefix)
let listing = await index.listDir(key)
// The index is the whole listing here, not a cache in front of one, so an
// *expired* answer means the tree aged out, not that the path is gone.
@@ -57,12 +54,10 @@ export async function readdir(
// absence and must not cost a tree fetch: refilling on any miss spends a
// recursive-tree call on every ENOENT.
if (listing.status === LookupStatus.EXPIRED && !accessor.truncated) {
if (await refillIndex(accessor, index)) listing = await index.listDir(key)
if (await refillIndex(accessor, index, prefix)) listing = await index.listDir(key)
}
if (listing.entries !== undefined && listing.entries !== null) {
return prefix !== '' && listing.entries.length > 0 && !listing.entries[0]?.startsWith(prefix)
? listing.entries.map((e) => prefix + e)
: listing.entries
return listing.entries
}
if (listing.status === LookupStatus.NOT_FOUND) {
if (accessor.truncated) {
@@ -79,7 +74,7 @@ async function fallbackReaddir(
index: IndexCacheStore,
prefix: string,
): Promise<string[]> {
const parentSha = await resolveDirSha(accessor, key, index)
const parentSha = await resolveDirSha(accessor, key, index, prefix)
if (parentSha === null) throw enoent(`${prefix}/${key}`)
const entries = await fetchDirTree(accessor.transport, accessor.owner, accessor.repo, parentSha)
const childKeys: string[] = []
@@ -102,23 +97,26 @@ async function fallbackReaddir(
}
childKeys.sort(compareCodePoints)
await index.setDir(key, childEntries)
return childKeys.map((k) => (prefix !== '' ? prefix + k : k))
return childKeys
}
async function resolveDirSha(
accessor: GitHubAccessor,
key: string,
index: IndexCacheStore,
prefix: string,
): Promise<string | null> {
const result = await index.get(key)
if (result.entry !== undefined && result.entry !== null) {
return result.entry.id
}
const parts = stripSlash(key)
const stem = rstripSlash(prefix)
const rest = stem !== '' && key.startsWith(stem) ? key.slice(stem.length) : key
const parts = stripSlash(rest)
.split('/')
.filter((p) => p !== '')
let currentSha = accessor.ref
let currentPath = ''
let currentPath = stem
for (const part of parts) {
const entries = await fetchDirTree(
accessor.transport,
@@ -18,7 +18,7 @@ import type { IndexCacheStore } from '../../cache/index/store.ts'
import { FileStat, FileType, PathSpec } from '../../types.ts'
import { getExtension } from '../../commands/resolve.ts'
import { readdir as coreReaddir } from './readdir.ts'
import { stripSlash } from '../../utils/slash.ts'
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
import { enoent } from '../../utils/errors.ts'
function stripPrefix(path: PathSpec): string {
@@ -30,11 +30,6 @@ function stripPrefix(path: PathSpec): string {
return p
}
function indexKey(p: string): string {
const trimmed = stripSlash(p)
return trimmed === '' ? '/' : `/${trimmed}`
}
function guessFileType(name: string): FileType {
const ext = getExtension(name)
if (ext === 'json') return FileType.JSON
@@ -54,11 +49,15 @@ export async function stat(
return new FileStat({ name: '/', type: FileType.DIRECTORY })
}
if (index === undefined) throw enoent(path)
const ikey = indexKey(p)
const ikey = `${rstripSlash(prefix)}/${trimmed}`
let result = await index.get(ikey)
if (result.entry === undefined || result.entry === null) {
const parentIdx = ikey.includes('/') ? ikey.slice(0, ikey.lastIndexOf('/')) || '/' : '/'
const parentPath = prefix !== '' ? prefix + parentIdx : parentIdx
// `ikey` is already mount-absolute, so its parent is too: prepending
// the prefix again asks for `/repo/repo`, whose listing never populates
// the entry this is here to find. stat then reports ENOENT for a file
// that exists, and the read family's implicit-directory probe finds it
// in the parent listing and answers EISDIR instead.
const parentPath = ikey.includes('/') ? ikey.slice(0, ikey.lastIndexOf('/')) || '/' : '/'
try {
await coreReaddir(
accessor,
@@ -14,6 +14,9 @@
import { describe, expect, it } from 'vitest'
import { fetchDirTree, fetchTree, type GitHubTransport } from './_client.ts'
import { GitHubAccessor } from '../../accessor/github.ts'
import { RAMIndexCacheStore } from '../../cache/index/ram.ts'
import { ensureLiveIndex, populateIndex } from './tree.ts'
const ITEMS = [
{ path: 'extern', mode: '160000', type: 'commit', sha: 'ccc' },
@@ -39,3 +42,107 @@ describe('github tree fetch', () => {
expect(entries.map((e) => e.path)).toEqual(['main.py', 'src'])
})
})
describe('ensureLiveIndex', () => {
const TREE = {
data: { path: 'data', type: 'tree', sha: 't1', size: null },
'data/keep.txt': { path: 'data/keep.txt', type: 'blob', sha: 'b1', size: 4 },
}
function accessor(calls: { n: number }): GitHubAccessor {
return new GitHubAccessor({
transport: {
get: () => {
calls.n += 1
return Promise.resolve({
tree: [
{ path: 'data', type: 'tree' as const, sha: 't1' },
{ path: 'data/keep.txt', type: 'blob' as const, sha: 'b1', size: 4 },
],
truncated: false,
})
},
} as unknown as GitHubTransport,
owner: 'acme',
repo: 'proj',
ref: 'main',
defaultBranch: 'main',
tree: TREE,
})
}
it('refetches rather than reuse the build-time tree', async () => {
// The build tree is only true at build time: a mount's first read can
// come long after it, so reusing it would key an index built from a
// repository several external writes ago.
const calls = { n: 0 }
const index = new RAMIndexCacheStore({ ttl: 600 })
expect(await ensureLiveIndex(accessor(calls), index, '/gh')).toBe(true)
expect(calls.n).toBe(1)
expect((await index.listDir('/gh/data')).entries).toEqual(['/gh/data/keep.txt'])
})
it('refetches a dropped listing', async () => {
const calls = { n: 0 }
const acc = accessor(calls)
const index = new RAMIndexCacheStore({ ttl: 600 })
await ensureLiveIndex(acc, index, '/gh')
// What invalidation does: drop the row rather than expire it, which is
// why the readers' EXPIRED probe never fires.
await index.invalidateDir('/gh')
await index.invalidateDir('/gh/data')
expect(await ensureLiveIndex(acc, index, '/gh')).toBe(true)
expect(calls.n).toBe(2)
expect((await index.listDir('/gh/data')).entries).toEqual(['/gh/data/keep.txt'])
})
it('leaves a live index alone and sends no request', async () => {
const calls = { n: 0 }
const acc = accessor(calls)
const index = new RAMIndexCacheStore({ ttl: 600 })
await ensureLiveIndex(acc, index, '/gh')
const before = calls.n
expect(await ensureLiveIndex(acc, index, '/gh')).toBe(false)
expect(calls.n).toBe(before)
})
it('skips a truncated tree', async () => {
const calls = { n: 0 }
const acc = accessor(calls)
acc.truncated = true
expect(await ensureLiveIndex(acc, new RAMIndexCacheStore({ ttl: 600 }), '/gh')).toBe(false)
expect(calls.n).toBe(0)
})
it('skips a missing index', async () => {
expect(await ensureLiveIndex(accessor({ n: 0 }), undefined, '')).toBe(false)
})
})
describe('populateIndex', () => {
const TREE = {
data: { path: 'data', type: 'tree', sha: 't1', size: null },
'data/keep.txt': { path: 'data/keep.txt', type: 'blob', sha: 'b1', size: 4 },
}
it('keys by mount-absolute path', async () => {
// Every other backend keys its index this way, which is what lets the
// shared CacheManager spell an eviction without knowing the backend.
const index = new RAMIndexCacheStore({ ttl: 600 })
await populateIndex(index, TREE, '/gh')
expect((await index.listDir('/gh')).entries).toEqual(['/gh/data'])
expect((await index.listDir('/gh/data')).entries).toEqual(['/gh/data/keep.txt'])
})
it('keeps bare paths on a root mount', async () => {
const index = new RAMIndexCacheStore({ ttl: 600 })
await populateIndex(index, TREE, '')
expect((await index.listDir('/')).entries).toEqual(['/data'])
})
it('gives an empty repo a root row', async () => {
const index = new RAMIndexCacheStore({ ttl: 600 })
await populateIndex(index, {}, '/gh')
expect((await index.listDir('/gh')).entries).toEqual([])
})
})
@@ -15,9 +15,11 @@
import type { GitHubAccessor } from '../../accessor/github.ts'
import { fetchTree } from './_client.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import { LookupStatus } from '../../cache/index/config.ts'
import type { IndexEntry } from '../../cache/index/config.ts'
import type { GitHubTreeItem } from './_client.ts'
import { indexEntryFromTree, makeTreeEntry, type TreeEntry } from './tree_entry.ts'
import { rstripSlash } from '../../utils/slash.ts'
export function buildTreeMap(tree: GitHubTreeItem[]): Record<string, TreeEntry> {
const map: Record<string, TreeEntry> = {}
@@ -25,12 +27,26 @@ export function buildTreeMap(tree: GitHubTreeItem[]): Record<string, TreeEntry>
return map
}
export async function populateIndex(index: IndexCacheStore, tree: GitHubTreeItem[]): Promise<void> {
export async function populateIndex(
index: IndexCacheStore,
tree: Record<string, TreeEntry>,
prefix: string,
): Promise<void> {
// Keyed by mount-absolute path, the way every other backend keys its
// index, so the shared cache machinery can spell an eviction without
// knowing which backend it is talking to. The tree itself stays
// repo-relative; `prefix` is what lifts it.
const stem = rstripSlash(prefix)
const dirs = new Map<string, [string, IndexEntry][]>()
for (const item of tree) {
// The repository root always exists, so it gets a row even when the tree
// is empty. Without it an empty repository is byte for byte a dropped
// index, and `ensureLiveIndex` would refetch on every read of one.
dirs.set(stem === '' ? '/' : stem, [])
for (const item of Object.values(tree)) {
const parts = item.path.split('/')
const name = parts[parts.length - 1] ?? item.path
const parent = parts.length > 1 ? `/${parts.slice(0, -1).join('/')}` : '/'
const parent =
parts.length > 1 ? `${stem}/${parts.slice(0, -1).join('/')}` : stem === '' ? '/' : stem
const arr = dirs.get(parent) ?? []
arr.push([name, indexEntryFromTree(item)])
dirs.set(parent, arr)
@@ -38,6 +54,19 @@ export async function populateIndex(index: IndexCacheStore, tree: GitHubTreeItem
await Promise.all([...dirs].map(([parent, entries]) => index.setDir(parent, entries)))
}
/**
* Write the accessor's tree into `index` under `prefix`.
*
* Mirrors Python's `seed_index`.
*/
async function seedIndex(
accessor: GitHubAccessor,
index: IndexCacheStore,
prefix: string,
): Promise<void> {
await populateIndex(index, accessor.tree, prefix)
}
/**
* Refetch the recursive tree and re-seed the index from it.
*
@@ -61,6 +90,7 @@ export async function populateIndex(index: IndexCacheStore, tree: GitHubTreeItem
export async function refillIndex(
accessor: GitHubAccessor,
index: IndexCacheStore | undefined,
prefix: string,
): Promise<boolean> {
if (index === undefined) return false
const { tree, truncated } = await fetchTree(
@@ -71,6 +101,57 @@ export async function refillIndex(
)
accessor.truncated = truncated
accessor.tree = buildTreeMap(tree)
await populateIndex(index, tree)
await seedIndex(accessor, index, prefix)
return true
}
/**
* Refetch when the index holds no listing at all.
*
* Every reader here treats a missing listing as a real absence, which is
* right against a *live* index and wrong against one that was never filled
* or has been dropped, and invalidation drops rather than expires:
* `invalidateDir` removes the directory's row outright, so the EXPIRED
* probe each reader already runs never fires. An external change (a watch
* event is the only thing that invalidates a mount with no write ops)
* therefore left the whole mount answering ENOENT permanently, since the
* seeded expiry is a year out.
*
* The root listing is what tells live from not, in one lookup and no
* request: the tree is written whole, so while the index is live every
* directory has a row and the mount root always does. One refill makes it
* live again, so this cannot cost a fetch per miss, which is what kept the
* readers from probing on absence in the first place.
*
* Not live always **refetches**, and never re-seeds the tree the mount was
* built with. That tree is only true at build time: the first read of a
* mount can come long after it, and reusing it then served an index built
* from a repository five external writes ago. It is still what
* `accessor.tree` starts as, so find and du have something to read before
* any listing happens, and every refill reseats it.
*
* Mirrors Python's `ensure_live_index`.
*
* Args:
* accessor (GitHubAccessor): the mount's accessor.
* index (IndexCacheStore | undefined): the index to check and fill.
* prefix (string): the mount prefix the index keys are built against.
*
* Returns:
* boolean: whether the index was filled.
*/
export async function ensureLiveIndex(
accessor: GitHubAccessor,
index: IndexCacheStore | undefined,
prefix: string,
): Promise<boolean> {
if (index === undefined) return false
// The liveness probe comes before anything on the accessor, so a live
// index still answers every read without one.
const root = rstripSlash(prefix) === '' ? '/' : rstripSlash(prefix)
if ((await index.listDir(root)).status !== LookupStatus.NOT_FOUND) return false
// A truncated tree is not the whole listing, so the invariant this rests
// on does not hold and readdir's per-directory fallback owns the miss.
if (accessor.truncated) return false
return refillIndex(accessor, index, prefix)
}
@@ -31,7 +31,12 @@ export function makeTreeEntry(item: GitHubTreeItem): TreeEntry {
}
}
export function indexEntryFromTree(item: GitHubTreeItem): IndexEntry {
export function indexEntryFromTree(item: {
path: string
type: string
sha: string
size?: number | null
}): IndexEntry {
const parts = item.path.split('/')
const name = parts[parts.length - 1] ?? item.path
return new IndexEntry({
@@ -0,0 +1,91 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { GitHubAccessor } from '../../accessor/github.ts'
import { PathSpec, type WalkEntry } from '../../types.ts'
import { IncompleteWalkError } from '../../watch/errors.ts'
import type { GitHubTransport } from './_client.ts'
import type { TreeEntry } from './tree_entry.ts'
import { GitHubWalk } from './watch.ts'
interface TreeItem {
path: string
type: string
sha: string
size?: number
}
function transport(tree: TreeItem[], truncated = false): GitHubTransport {
return {
get: () => Promise.resolve({ tree, truncated }),
request: () => Promise.resolve({}),
}
}
function accessor(tree: TreeItem[], stale: Record<string, TreeEntry>, truncated = false) {
return new GitHubAccessor({
transport: transport(tree, truncated),
owner: 'acme',
repo: 'proj',
ref: 'main',
defaultBranch: 'main',
tree: stale,
})
}
function root(): PathSpec {
return new PathSpec({ virtual: '/gh', directory: '/gh', resourcePath: '' })
}
async function collect(walk: GitHubWalk, at: PathSpec): Promise<WalkEntry[]> {
const out: WalkEntry[] = []
for await (const entry of walk.walk(at)) out.push(entry)
return out
}
const STALE: Record<string, TreeEntry> = {
'a.txt': { path: 'a.txt', type: 'blob', sha: 'sha-a', size: 3 },
}
describe('GitHubWalk', () => {
it('refreshes the accessor tree from the pull', async () => {
// find, du and grep's scope counter read accessor.tree directly, so a
// pull that fetched a newer tree and dropped it left them reporting
// the repository as it stood when the mount was built.
const acc = accessor(
[
{ path: 'a.txt', type: 'blob', sha: 'sha-a', size: 3 },
{ path: 'b.txt', type: 'blob', sha: 'sha-b', size: 3 },
],
STALE,
)
await collect(new GitHubWalk(acc), root())
expect(Object.keys(acc.tree).sort()).toEqual(['a.txt', 'b.txt'])
expect(acc.tree['b.txt']?.sha).toBe('sha-b')
})
it('does not adopt a truncated tree', async () => {
// A partial tree would make find report the missing half as deleted.
const acc = accessor([], STALE, true)
await expect(collect(new GitHubWalk(acc), root())).rejects.toThrow(IncompleteWalkError)
expect(Object.keys(acc.tree)).toEqual(['a.txt'])
})
it('reports blobs with their sha as the fingerprint', async () => {
const acc = accessor([{ path: 'a.txt', type: 'blob', sha: 'sha-a', size: 3 }], STALE)
const entries = await collect(new GitHubWalk(acc), root())
expect(entries.map((e) => [e.virtual, e.fingerprint])).toEqual([['/gh/a.txt', 'sha-a']])
})
})
@@ -0,0 +1,88 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { GitHubAccessor } from '../../accessor/github.ts'
import type { PathSpec, WalkEntry } from '../../types.ts'
import { mountPrefixOf } from '../../utils/key_prefix.ts'
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { IncompleteWalkError } from '../../watch/errors.ts'
import { fetchTree } from './_client.ts'
import { buildTreeMap } from './tree.ts'
/**
* One recursive git tree fetch feeding the generic listing differ.
*
* `GET /git/trees/{ref}?recursive=1` returns every path in the repository with
* its object sha, so a pull is one request whatever the repository's shape, and
* the fingerprint is the sha itself. That is the strongest fingerprint any
* mirage backend has: git is content-addressed, so identical bytes have an
* identical sha and a rewrite that changes nothing correctly reports nothing.
*
* A mount is pinned to one ref, so what this detects is that ref moving.
* Nothing is reported while the branch sits still, however much is pushed
* elsewhere in the repository.
*/
export class GitHubWalk {
private readonly accessor: GitHubAccessor
constructor(accessor: GitHubAccessor) {
this.accessor = accessor
}
async *walk(root: PathSpec): AsyncGenerator<WalkEntry> {
const accessor = this.accessor
const prefix = mountPrefixOf(root.virtual, root.resourcePath)
const { tree, truncated } = await fetchTree(
accessor.transport,
accessor.owner,
accessor.repo,
accessor.ref,
)
if (truncated) {
throw new IncompleteWalkError(
`github tree for ${accessor.owner}/${accessor.repo}@${accessor.ref} was truncated; ` +
'cannot diff a partial tree',
)
}
// A complete tree for the ref is exactly what the accessor holds, and
// find/du/grep's scope counter read it directly. Discarding it here
// left them answering from the tree the mount was built with until an
// unrelated read happened to refill the index, so a pull that reported
// a CREATE was followed by a find that could not see the file.
accessor.tree = buildTreeMap(tree)
const stem = stripSlash(rstripSlash(root.resourcePath))
const base = stem !== '' ? `${stem}/` : ''
for (const item of tree) {
if (base !== '' && !item.path.startsWith(base)) continue
const virtual = prefix !== '' ? `${prefix}/${item.path}` : `/${item.path}`
if (item.type === 'tree') {
yield { virtual, isDir: true, fingerprint: null }
continue
}
yield {
virtual,
isDir: false,
fingerprint: item.sha,
size: item.size ?? null,
}
}
}
}
export function buildDeltaHook(accessor: GitHubAccessor): DeltaHook {
const walk = new GitHubWalk(accessor)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -0,0 +1,42 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { OneDriveAccessor } from '../../accessor/onedrive.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { FileStat, PathSpec } from '../../types.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { ReaddirWalk } from '../../watch/walk.ts'
import { readdir, stat } from './index.ts'
/**
* Build the OneDrive delta hook.
*
* Fingerprints on the item's `cTag`, which Graph moves only when the content
* changes (`eTag` also moves on a metadata edit), so a rename does not read as
* a content change.
*
* Graph has a native `/delta` feed with a resumable token, which is cheaper
* than this walk and reports deletes directly. It is a fast path rather than a
* replacement: Graph can answer `resyncRequired` at any time, and the only
* response to that is a full listing. When it is added it belongs behind
* `pull`, with this walk as its reset.
*/
export function buildDeltaHook(accessor: OneDriveAccessor): DeltaHook {
const walk = new ReaddirWalk(
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -0,0 +1,178 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as ClientModule from './_client.ts'
vi.mock('./_client.ts', async () => {
const actual = await vi.importActual<typeof ClientModule>('./_client.ts')
return { ...actual, loadS3Module: vi.fn(), withClient: vi.fn() }
})
import { S3Accessor } from '../../accessor/s3.ts'
import type { S3Config } from '../../resource/s3/config.ts'
import { FileChangeKind, PathSpec, type WalkEntry } from '../../types.ts'
import * as clientMod from './_client.ts'
import { buildDeltaHook, S3Walk } from './watch.ts'
class FakeCommand {
constructor(readonly input: unknown) {}
}
interface StoredObject {
key: string
size: number
etag: string
}
function mockListing(objects: StoredObject[]): void {
vi.mocked(clientMod.loadS3Module).mockResolvedValue({
ListObjectsV2Command: FakeCommand,
} as never)
vi.mocked(clientMod.withClient).mockImplementation(async (_config, fn) => {
const client = {
send: () =>
Promise.resolve({
Contents: objects.map((obj) => ({
Key: obj.key,
Size: obj.size,
LastModified: new Date('2026-03-31T00:00:00.000Z'),
ETag: `"${obj.etag}"`,
})),
IsTruncated: false,
}),
}
return (await fn(client as never)) as never
})
}
function accessor(keyPrefix?: string): S3Accessor {
return new S3Accessor({
bucket: 'watch-bucket',
region: 'us-east-1',
keyPrefix,
} as S3Config)
}
function root(virtual: string, resourcePath: string): PathSpec {
return new PathSpec({ virtual, directory: virtual, resourcePath })
}
async function collect(walk: S3Walk, spec: PathSpec): Promise<WalkEntry[]> {
const out: WalkEntry[] = []
for await (const entry of walk.walk(spec)) out.push(entry)
return out
}
describe('S3Walk', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('yields files fingerprinted on the ETag', async () => {
mockListing([
{ key: 'data/a.txt', size: 5, etag: 'etag-a' },
{ key: 'data/b.txt', size: 4, etag: 'etag-b' },
])
const entries = await collect(new S3Walk(accessor()), root('/s3/data', 'data'))
const files = entries.filter((e) => !e.isDir)
expect(files.map((e) => e.virtual).sort()).toEqual(['/s3/data/a.txt', '/s3/data/b.txt'])
// The ETag, not the mtime|size composite: LastModified is constant
// here, so a composite would collide across files of equal size.
expect(files[0]?.fingerprint).toBe('etag-a')
expect(files[0]?.size).toBe(5)
})
it('synthesizes intermediate directories', async () => {
mockListing([{ key: 'data/sub/deep/x.txt', size: 1, etag: 'etag-x' }])
const entries = await collect(new S3Walk(accessor()), root('/s3/data', 'data'))
expect(entries.filter((e) => e.isDir).map((e) => e.virtual)).toEqual([
'/s3/data/sub/deep',
'/s3/data/sub',
])
})
it('reports an explicit marker as its own directory', async () => {
mockListing([{ key: 'data/empty/', size: 0, etag: 'etag-marker' }])
const entries = await collect(new S3Walk(accessor()), root('/s3/data', 'data'))
expect(entries.filter((e) => e.isDir).map((e) => e.virtual)).toEqual(['/s3/data/empty'])
expect(entries.filter((e) => !e.isDir)).toEqual([])
})
it('strips the key prefix', async () => {
mockListing([{ key: 'team/x/data/a.txt', size: 5, etag: 'etag-a' }])
const entries = await collect(new S3Walk(accessor('team/x/')), root('/s3/data', 'data'))
expect(entries.filter((e) => !e.isDir).map((e) => e.virtual)).toEqual(['/s3/data/a.txt'])
})
})
describe('s3 delta hook', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('reports nothing on the baseline pull, then create and update', async () => {
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
const hook = buildDeltaHook(accessor())
const spec = root('/s3/data', 'data')
const first = await hook.pull(spec, null)
expect(first.changes).toEqual([])
mockListing([
{ key: 'data/a.txt', size: 5, etag: 'etag-a2' },
{ key: 'data/new.txt', size: 3, etag: 'etag-new' },
])
const second = await hook.pull(spec, first.checkpoint)
expect(Object.fromEntries(second.changes.map((c) => [c.path.virtual, c.kind]))).toEqual({
'/s3/data/a.txt': FileChangeKind.UPDATE,
'/s3/data/new.txt': FileChangeKind.CREATE,
})
})
it('detects a delete', async () => {
mockListing([
{ key: 'data/a.txt', size: 5, etag: 'etag-a' },
{ key: 'data/b.txt', size: 4, etag: 'etag-b' },
])
const hook = buildDeltaHook(accessor())
const spec = root('/s3/data', 'data')
const first = await hook.pull(spec, null)
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
const second = await hook.pull(spec, first.checkpoint)
expect(second.changes.map((c) => [c.kind, c.path.virtual])).toEqual([
[FileChangeKind.DELETE, '/s3/data/b.txt'],
])
})
it('treats a rewrite of identical bytes as no change', async () => {
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
const hook = buildDeltaHook(accessor())
const spec = root('/s3/data', 'data')
const first = await hook.pull(spec, null)
const second = await hook.pull(spec, first.checkpoint)
expect(second.changes).toEqual([])
})
it('frames a changed path against the mount', async () => {
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a' }])
const hook = buildDeltaHook(accessor())
const spec = root('/s3/data', 'data')
const first = await hook.pull(spec, null)
mockListing([{ key: 'data/a.txt', size: 5, etag: 'etag-a2' }])
const second = await hook.pull(spec, first.checkpoint)
expect(second.changes[0]?.path.virtual).toBe('/s3/data/a.txt')
expect(second.changes[0]?.path.resourcePath).toBe('data/a.txt')
})
})
@@ -0,0 +1,113 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { S3Accessor } from '../../accessor/s3.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { statFingerprint } from '../../watch/fingerprint.ts'
import { synthDirs } from '../../watch/walk.ts'
import type { PathSpec, WalkEntry } from '../../types.ts'
import { mountPrefixOf } from '../../utils/key_prefix.ts'
import { rstripSlash, stripSlash } from '../../utils/slash.ts'
import { loadS3Module, rawPathOf, s3Key, stripKeyPrefix, withClient } from './_client.ts'
interface ListedObject {
Key?: string
Size?: number
LastModified?: Date | string
ETag?: string
}
function isoOf(value: Date | string | undefined): string | null {
if (value === undefined) return null
return value instanceof Date ? value.toISOString() : value
}
/**
* Recursive `ListObjectsV2` feeding the generic listing differ.
*
* One paginated LIST with no Delimiter covers the whole subtree, so a pull
* costs one request per 1000 keys rather than one per directory. Reads the
* bucket directly, never through mirage's caches, as the DeltaHook contract
* requires.
*
* Fingerprints on the object's ETag, which for a single-part upload is the MD5
* of the content, so an overwrite with identical bytes is correctly reported
* as no change. Multipart ETags are a digest of the part digests, which still
* changes with the content.
*/
export class S3Walk {
private readonly accessor: S3Accessor
constructor(accessor: S3Accessor) {
this.accessor = accessor
}
async *walk(root: PathSpec): AsyncGenerator<WalkEntry> {
const config = this.accessor.config
const prefix = mountPrefixOf(root.virtual, root.resourcePath)
const stem = rstripSlash(s3Key(rawPathOf(root), config))
const base = stem !== '' ? `${stem}/` : ''
const files: string[] = []
const markers: string[] = []
const rows: WalkEntry[] = []
const { ListObjectsV2Command } = await loadS3Module(config)
await withClient(config, async (client) => {
let continuationToken: string | undefined
do {
const input: Record<string, unknown> = { Bucket: config.bucket, Prefix: stem }
if (continuationToken !== undefined) input.ContinuationToken = continuationToken
const resp = (await client.send(new ListObjectsV2Command(input))) as {
Contents?: ListedObject[]
IsTruncated?: boolean
NextContinuationToken?: string
}
for (const obj of resp.Contents ?? []) {
const key = obj.Key
if (key === undefined) continue
if (!(key === stem || key.startsWith(base))) continue
const relative = stripSlash(stripKeyPrefix(key, config))
const virtual = prefix !== '' ? `${prefix}/${relative}` : `/${relative}`
if (key.endsWith('/')) {
// A directory marker: mirage's own mkdir writes one. It
// carries an ETag and a size, but it is not a file, so
// synthDirs reports it instead.
markers.push(rstripSlash(virtual))
continue
}
files.push(virtual)
const modified = isoOf(obj.LastModified)
const size = obj.Size ?? null
const etag = (obj.ETag ?? '').replace(/^"|"$/g, '') || null
rows.push({
virtual,
isDir: false,
fingerprint: statFingerprint(etag, modified, size),
size,
modified,
})
}
continuationToken = resp.IsTruncated === true ? resp.NextContinuationToken : undefined
} while (continuationToken !== undefined)
})
yield* rows
yield* synthDirs(root.virtual, files, markers)
}
}
// Build the delta hook shared by S3 and every S3-compatible alias.
export function buildDeltaHook(accessor: S3Accessor): DeltaHook {
const walk = new S3Walk(accessor)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -0,0 +1,36 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { SharePointAccessor } from '../../accessor/sharepoint.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { FileStat, PathSpec } from '../../types.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { ListingDeltaHook } from '../../watch/delta.ts'
import { ReaddirWalk } from '../../watch/walk.ts'
import { readdir, stat } from './index.ts'
/**
* Build the SharePoint delta hook.
*
* Same Graph drive surface as OneDrive, so the same `cTag` fingerprint and the
* same standing offer of a native `/delta` fast path; see
* `core/onedrive/watch.ts`.
*/
export function buildDeltaHook(accessor: SharePointAccessor): DeltaHook {
const walk = new ReaddirWalk(
(path: PathSpec, index: IndexCacheStore): Promise<string[]> => readdir(accessor, path, index),
(path: PathSpec, index: IndexCacheStore): Promise<FileStat> => stat(accessor, path, index),
)
return new ListingDeltaHook(walk.walk.bind(walk))
}
@@ -13,6 +13,8 @@ import { ResourceName, type FileStat, type PathSpec } from '../../types.ts'
import type { RegisteredCommand } from '../../commands/config.ts'
import { BaseResource, type Resource } from '../base.ts'
import { ONEDRIVE_PROMPT } from './prompt.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { buildDeltaHook } from '../../core/onedrive/watch.ts'
const resolveGlob = makeResolveGlob(readdir)
@@ -68,6 +70,10 @@ export class OneDriveResource extends BaseResource implements Resource {
return stat(this.accessor, path, this.index)
}
deltaHook(): DeltaHook {
return buildDeltaHook(this.accessor)
}
override getState(): OneDriveResourceState {
const config: OneDriveConfigRedacted = redactOneDriveConfig(this.config)
return { type: this.kind, config }
@@ -13,6 +13,8 @@ import { ResourceName, type FileStat, type PathSpec } from '../../types.ts'
import type { RegisteredCommand } from '../../commands/config.ts'
import { BaseResource, type Resource } from '../base.ts'
import { SHAREPOINT_PROMPT } from './prompt.ts'
import type { DeltaHook } from '../../watch/base.ts'
import { buildDeltaHook } from '../../core/sharepoint/watch.ts'
const resolveGlob = makeResolveGlob(readdir)
@@ -68,6 +70,10 @@ export class SharePointResource extends BaseResource implements Resource {
return stat(this.accessor, path, this.index)
}
deltaHook(): DeltaHook {
return buildDeltaHook(this.accessor)
}
override getState(): SharePointResourceState {
const config: SharePointConfigRedacted = redactSharePointConfig(this.config)
return { type: this.kind, config }
@@ -19,6 +19,18 @@ export class QueueOverflowError extends Error {
}
}
// A snapshot diff reads every path the walk did not report as a
// DELETE, so a partial listing does not degrade into fewer events, it
// invents wrong ones. A hook that knows its listing was truncated
// raises this instead, leaving the caller's checkpoint untouched so the
// next pull can still succeed.
export class IncompleteWalkError extends Error {
constructor(message: string) {
super(message)
this.name = 'IncompleteWalkError'
}
}
export class QueueClosed extends Error {
constructor(label: string) {
super(label)
+2 -1
View File
@@ -21,7 +21,7 @@ export type {
WatchRuntime,
} from './base.ts'
export { ListingDeltaHook, specFor } from './delta.ts'
export { QueueClosed, QueueOverflowError } from './errors.ts'
export { IncompleteWalkError, QueueClosed, QueueOverflowError } from './errors.ts'
export { statFingerprint } from './fingerprint.ts'
export {
type QueueFactory,
@@ -30,3 +30,4 @@ export {
type WatchQueue,
} from './queue/index.ts'
export { Watcher } from './watcher.ts'
export { entryOf, ReaddirWalk, type WalkReaddirFn, synthDirs, type WalkStatFn } from './walk.ts'
@@ -0,0 +1,176 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import type { IndexCacheStore } from '../cache/index/store.ts'
import { FileStat, FileType, PathSpec, type WalkEntry } from '../types.ts'
import { enoent } from '../utils/errors.ts'
import { entryOf, ReaddirWalk, synthDirs } from './walk.ts'
function root(virtual: string, resourcePath: string): PathSpec {
return new PathSpec({ virtual, directory: virtual, resourcePath })
}
describe('synthDirs', () => {
it('emits every ancestor of a file, excluding the root', () => {
const dirs = [...synthDirs('/m/data', ['/m/data/a/b/c.txt'], [])].map((e) => e.virtual)
expect(dirs).toEqual(['/m/data/a/b', '/m/data/a'])
})
it('reports a shared prefix once', () => {
const dirs = [...synthDirs('/m/data', ['/m/data/a/x.txt', '/m/data/a/y.txt'], [])]
expect(dirs.map((e) => e.virtual)).toEqual(['/m/data/a'])
})
it('emits an explicitly stored directory even with no children', () => {
const dirs = [...synthDirs('/m/data', [], ['/m/data/empty'])]
expect(dirs.map((e) => e.virtual)).toEqual(['/m/data/empty'])
})
it('does not double-report a prefix backed by both a marker and children', () => {
const dirs = [...synthDirs('/m/data', ['/m/data/a/x.txt'], ['/m/data/a'])]
expect(dirs.map((e) => e.virtual)).toEqual(['/m/data/a'])
})
it('marks every row as a directory with no fingerprint', () => {
const dirs = [...synthDirs('/m/data', ['/m/data/a/x.txt'], [])]
expect(dirs.every((e) => e.isDir && e.fingerprint === null)).toBe(true)
})
it('emits nothing for a file directly under the root', () => {
expect([...synthDirs('/m/data', ['/m/data/x.txt'], [])]).toEqual([])
})
})
describe('entryOf', () => {
it('reports a directory with no fingerprint', () => {
const entry = entryOf('/m/d', new FileStat({ name: 'd', type: FileType.DIRECTORY }))
expect(entry).toEqual({ virtual: '/m/d', isDir: true, fingerprint: null })
})
it('prefers the backend fingerprint over the composite', () => {
const entry = entryOf(
'/m/f.txt',
new FileStat({ name: 'f.txt', size: 3, modified: 'T', fingerprint: 'etag-1' }),
)
expect(entry.fingerprint).toBe('etag-1')
})
it('falls back to mtime|size when the backend has no version', () => {
const entry = entryOf('/m/f.txt', new FileStat({ name: 'f.txt', size: 3, modified: 'T' }))
expect(entry.fingerprint).toBe('T|3')
})
})
type FakeTree = Record<string, { children?: string[]; stat: FileStat }>
function fakeBackend(tree: FakeTree): ReaddirWalk {
const readdir = (path: PathSpec): Promise<string[]> => {
const node = tree[path.virtual]
if (node?.children === undefined) return Promise.reject(enoent(path.virtual))
return Promise.resolve(node.children)
}
const stat = (path: PathSpec): Promise<FileStat> => {
const node = tree[path.virtual]
if (node === undefined) return Promise.reject(enoent(path.virtual))
return Promise.resolve(node.stat)
}
return new ReaddirWalk(readdir as never, stat as never)
}
async function collect(walk: ReaddirWalk, spec: PathSpec): Promise<WalkEntry[]> {
const out: WalkEntry[] = []
for await (const entry of walk.walk(spec)) out.push(entry)
return out
}
describe('ReaddirWalk', () => {
it('descends into every directory and reports leaves', async () => {
const walk = fakeBackend({
'/m/data': {
children: ['/m/data/a.txt', '/m/data/sub'],
stat: new FileStat({ name: 'data', type: FileType.DIRECTORY }),
},
'/m/data/a.txt': {
stat: new FileStat({ name: 'a.txt', size: 5, modified: 'T1', fingerprint: 'fp-a' }),
},
'/m/data/sub': {
children: ['/m/data/sub/deep.txt'],
stat: new FileStat({ name: 'sub', type: FileType.DIRECTORY }),
},
'/m/data/sub/deep.txt': {
stat: new FileStat({ name: 'deep.txt', size: 4, modified: 'T2', fingerprint: 'fp-d' }),
},
})
const entries = await collect(walk, root('/m/data', 'data'))
expect(entries.map((e) => e.virtual)).toEqual([
'/m/data/a.txt',
'/m/data/sub',
'/m/data/sub/deep.txt',
])
expect(entries.filter((e) => e.isDir).map((e) => e.virtual)).toEqual(['/m/data/sub'])
})
it('trusts a trailing slash without a stat round trip', async () => {
const walk = fakeBackend({
'/m/data': {
children: ['/m/data/sub/'],
stat: new FileStat({ name: 'data', type: FileType.DIRECTORY }),
},
// No stat entry for /m/data/sub at all: the slash is the proof,
// so a stat would reject and the walk would lose the subtree.
'/m/data/sub': {
children: [],
stat: new FileStat({ name: 'sub', type: FileType.DIRECTORY }),
},
})
const entries = await collect(walk, root('/m/data', 'data'))
expect(entries).toEqual([{ virtual: '/m/data/sub', isDir: true, fingerprint: null }])
})
it('skips an entry that vanished between the readdir and the stat', async () => {
const walk = fakeBackend({
'/m/data': {
children: ['/m/data/gone.txt', '/m/data/here.txt'],
stat: new FileStat({ name: 'data', type: FileType.DIRECTORY }),
},
'/m/data/here.txt': {
stat: new FileStat({ name: 'here.txt', size: 1, modified: 'T', fingerprint: 'fp' }),
},
})
const entries = await collect(walk, root('/m/data', 'data'))
expect(entries.map((e) => e.virtual)).toEqual(['/m/data/here.txt'])
})
it('walks a missing root as empty rather than throwing', async () => {
const walk = fakeBackend({})
expect(await collect(walk, root('/m/gone', 'gone'))).toEqual([])
})
it('starts from an empty index on every call', async () => {
const seen: (IndexCacheStore | undefined)[] = []
const readdir = (path: PathSpec, index: IndexCacheStore): Promise<string[]> => {
seen.push(index)
return Promise.resolve(path.virtual === '/m/data' ? ['/m/data/a.txt'] : [])
}
const stat = (): Promise<FileStat> =>
Promise.resolve(new FileStat({ name: 'a.txt', size: 1, modified: 'T', fingerprint: 'fp' }))
const walk = new ReaddirWalk(readdir as never, stat as never)
await collect(walk, root('/m/data', 'data'))
await collect(walk, root('/m/data', 'data'))
// Two pulls, two distinct index instances: nothing a pull learned
// can leak into the next one's snapshot.
expect(seen[0]).not.toBe(seen[seen.length - 1])
})
})

Some files were not shown because too many files have changed in this diff Show More