github-featured/strukto-ai--mirage
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Merge pull request #746 from bytecii/feat/box-dropbox-io-slots

Browse Source 
fix(box,dropbox): wire the du slot in python, drop the fake find slot in ts (#609 T1-I)
This commit is contained in:
bytecii
2026-08-10 00:39:31 -07:00
committed by GitHub
parent 460ae8ac37 1d8fb1cb53
commit 9a86bf72ff
19 changed files with 390 additions and 670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
integ/unix/find/mtime.json
+12 -4
View File
@@ -45,7 +45,9 @@
"disk",
"redis",
"opfs",
"ssh"
"ssh",
"box",
"dropbox"
],
"command": "printf hi > /data/mtprobe.txt; find /data -name mtprobe.txt -mtime -1",
"expect": {
@@ -62,7 +64,9 @@
"disk",
"redis",
"opfs",
"ssh"
"ssh",
"box",
"dropbox"
],
"command": "find /data -name mtprobe.txt -mtime +5; echo code=$?",
"expect": {
@@ -79,7 +83,9 @@
"disk",
"redis",
"opfs",
"ssh"
"ssh",
"box",
"dropbox"
],
"command": "touch -d 2020-01-01 /data/mtprobe.txt; find /data -name mtprobe.txt -mtime +5",
"expect": {
@@ -96,7 +102,9 @@
"disk",
"redis",
"opfs",
"ssh"
"ssh",
"box",
"dropbox"
],
"command": "rm /data/mtprobe.txt; echo done",
"expect": {
python/mirage/commands/builtin/box/io.py
+8 -1
View File
@@ -12,9 +12,11 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic_bind import CommandIO
from mirage.commands.builtin.generic_bind import CommandIO, DuOps
from mirage.core.box.copy import copy as _copy
from mirage.core.box.create import create as _create
from mirage.core.box.du import entries as _du_entries
from mirage.core.box.du import size as _du_size
from mirage.core.box.exists import exists as _exists
from mirage.core.box.mkdir import mkdir as _mkdir
from mirage.core.box.read import read as _read
@@ -38,6 +40,11 @@ IO = CommandIO(
stat=_stat,
is_mounted=lambda a: True,
local=False,
# Own the du walk instead of taking the builder's, which is capped at
# max_du_entries and reports a partial total past it. A Box tree over
# that cap is ordinary, and a silently wrong total is worse than a
# slow one; this matches the typescript table.
du=DuOps(size=_du_size, entries=_du_entries),
write=_write,
exists=_exists,
mkdir=_mkdir,
python/mirage/commands/builtin/dropbox/io.py
+8 -2
View File
@@ -12,9 +12,11 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic_bind import CommandIO
from mirage.commands.builtin.generic_bind import CommandIO, DuOps
from mirage.core.dropbox.copy import copy as _copy
from mirage.core.dropbox.create import create as _create
from mirage.core.dropbox.du import entries as _du_entries
from mirage.core.dropbox.du import size as _du_size
from mirage.core.dropbox.exists import exists as _exists
from mirage.core.dropbox.mkdir import mkdir as _mkdir
from mirage.core.dropbox.read import read as _read
@@ -28,7 +30,7 @@ from mirage.core.dropbox.unlink import unlink as _unlink
from mirage.core.dropbox.write import write_bytes as _write
# copy_v2 copies folder subtrees server-side, so dir_copy is the same
# call as copy. du falls back to the generic readdir+stat walk.
# call as copy.
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
@@ -37,6 +39,10 @@ IO = CommandIO(
stat=_stat,
is_mounted=lambda a: True,
local=False,
# Own the du walk instead of taking the builder's, which is capped at
# max_du_entries and reports a partial total past it. See the Box
# table; list_folder's recursive mode would make this a real pushdown.
du=DuOps(size=_du_size, entries=_du_entries),
write=_write,
exists=_exists,
mkdir=_mkdir,
python/mirage/core/dropbox/du/__init__.py
+18
View File
@@ -0,0 +1,18 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.core.dropbox.du.entries import entries
from mirage.core.dropbox.du.size import size
__all__ = ["entries", "size"]
python/mirage/core/dropbox/du/entries.py
+46
View File
@@ -0,0 +1,46 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.dropbox import DropboxAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.core.dropbox.du.walk import walk
from mirage.core.dropbox.stat import stat
from mirage.types import FileType, PathSpec
async def entries(
accessor: DropboxAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
) -> tuple[list[tuple[str, int]], int]:
"""Per-file sizes under a path plus their total.
A file has no tree to walk, so it reports no entries and the caller
falls back to its own size.
Args:
accessor (DropboxAccessor): Dropbox accessor.
path (PathSpec): target path.
index (IndexCacheStore): path->metadata index cache.
"""
try:
info = await stat(accessor, path, index)
except FileNotFoundError:
info = None
if info is not None and info.type != FileType.DIRECTORY:
return [], info.size or 0
found: list[tuple[str, int]] = []
total = await walk(accessor, path, index, found)
found.sort()
return found, total
python/mirage/core/dropbox/du/size.py
+33
View File
@@ -0,0 +1,33 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.dropbox import DropboxAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.core.dropbox.du.walk import walk
from mirage.types import PathSpec
async def size(
accessor: DropboxAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
) -> int:
"""Recursive byte size of everything under a path.
Args:
accessor (DropboxAccessor): Dropbox accessor.
path (PathSpec): target path.
index (IndexCacheStore): path->metadata index cache.
"""
return await walk(accessor, path, index, None)
python/mirage/core/dropbox/du/walk.py
+60
View File
@@ -0,0 +1,60 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.dropbox import DropboxAccessor
from mirage.cache.index import IndexCacheStore
from mirage.core.dropbox.readdir import readdir
from mirage.core.dropbox.stat import stat
from mirage.types import FileType, PathSpec
from mirage.utils.key_prefix import mount_key, mount_prefix_of
async def walk(
accessor: DropboxAccessor,
path: PathSpec,
index: IndexCacheStore,
results: list[tuple[str, int]] | None,
) -> int:
"""Sum file sizes under a path, optionally collecting each one.
Args:
accessor (DropboxAccessor): Dropbox accessor.
path (PathSpec): directory or file to walk.
index (IndexCacheStore): path->metadata index cache.
results (list[tuple[str, int]] | None): when given, collects
mount-relative (path, size) pairs for each file found.
"""
try:
info = await stat(accessor, path, index)
except FileNotFoundError:
return 0
prefix = mount_prefix_of(path.virtual, path.resource_path)
if info.type != FileType.DIRECTORY:
size = info.size or 0
if results is not None:
results.append(("/" + mount_key(path.virtual, prefix), size))
return size
try:
children = await readdir(accessor, path, index)
except FileNotFoundError:
return 0
total = 0
for child in children:
trimmed = child.rstrip("/")
child_spec = PathSpec(virtual=trimmed,
directory=trimmed,
resolved=False,
resource_path=mount_key(trimmed, prefix))
total += await walk(accessor, child_spec, index, results)
return total
python/tests/core/dropbox/du/test_entries.py
+98
View File
@@ -0,0 +1,98 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import patch
import pytest
from mirage.accessor.dropbox import DropboxAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.core.dropbox._client import DropboxTokenManager
from mirage.core.dropbox.du import entries
from mirage.resource.dropbox.config import DropboxConfig
from mirage.types import PathSpec
_TREE = {
"": [{
".tag": "folder",
"id": "id:data",
"name": "data",
"path_display": "/data",
}],
"/data": [
{
".tag": "folder",
"id": "id:sub",
"name": "sub",
"path_display": "/data/sub",
},
{
".tag": "file",
"id": "id:a",
"name": "a.txt",
"path_display": "/data/a.txt",
"size": 27,
"server_modified": "2026-04-01T00:00:00Z",
},
],
"/data/sub": [{
".tag": "file",
"id": "id:b",
"name": "b.txt",
"path_display": "/data/sub/b.txt",
"size": 12,
"server_modified": "2026-04-01T00:00:00Z",
}],
}
async def _fake_list(_tm, path):
return _TREE[path]
@pytest.fixture
def accessor():
config = DropboxConfig(client_id="c", client_secret="s", refresh_token="r")
return DropboxAccessor(config, DropboxTokenManager(config))
@pytest.fixture
def index():
return RAMIndexCacheStore()
@pytest.mark.asyncio
async def test_entries_lists_files_with_total(accessor, index):
with patch("mirage.core.dropbox.readdir.list_folder", new=_fake_list):
found, total = await entries(
accessor,
PathSpec(resource_path="data", virtual="/data", directory="/"),
index)
assert found == [
("/data/a.txt", 27),
("/data/sub/b.txt", 12),
]
assert total == 39
@pytest.mark.asyncio
async def test_entries_on_file_returns_empty(accessor, index):
with patch("mirage.core.dropbox.readdir.list_folder", new=_fake_list):
found, total = await entries(
accessor,
PathSpec(resource_path="data/a.txt",
virtual="/data/a.txt",
directory="/data"), index)
assert found == []
assert total == 27
python/tests/core/dropbox/du/test_size.py
+92
View File
@@ -0,0 +1,92 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import patch
import pytest
from mirage.accessor.dropbox import DropboxAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.core.dropbox._client import DropboxTokenManager
from mirage.core.dropbox.du import size
from mirage.resource.dropbox.config import DropboxConfig
from mirage.types import PathSpec
_TREE = {
"": [{
".tag": "folder",
"id": "id:data",
"name": "data",
"path_display": "/data",
}],
"/data": [
{
".tag": "folder",
"id": "id:sub",
"name": "sub",
"path_display": "/data/sub",
},
{
".tag": "file",
"id": "id:a",
"name": "a.txt",
"path_display": "/data/a.txt",
"size": 27,
"server_modified": "2026-04-01T00:00:00Z",
},
],
"/data/sub": [{
".tag": "file",
"id": "id:b",
"name": "b.txt",
"path_display": "/data/sub/b.txt",
"size": 12,
"server_modified": "2026-04-01T00:00:00Z",
}],
}
async def _fake_list(_tm, path):
return _TREE[path]
@pytest.fixture
def accessor():
config = DropboxConfig(client_id="c", client_secret="s", refresh_token="r")
return DropboxAccessor(config, DropboxTokenManager(config))
@pytest.fixture
def index():
return RAMIndexCacheStore()
@pytest.mark.asyncio
async def test_size_walks_directory_tree(accessor, index):
with patch("mirage.core.dropbox.readdir.list_folder", new=_fake_list):
total = await size(
accessor,
PathSpec(resource_path="data", virtual="/data", directory="/"),
index)
assert total == 39
@pytest.mark.asyncio
async def test_size_missing_path_is_zero(accessor, index):
with patch("mirage.core.dropbox.readdir.list_folder", new=_fake_list):
total = await size(
accessor,
PathSpec(resource_path="ghost", virtual="/ghost", directory="/"),
index)
assert total == 0
spec/parity_exceptions.json
+2 -2
View File
@@ -1,10 +1,10 @@
{
"command_io": {
"box": {
"slots": "typescript pushes du and find down to the Box API while python takes the capped readdir walk (core/box/du/ is written but unwired); python range-reads while typescript's core read has no window. Both halves of T1-I."
"slots": "python range-reads; the typescript core read has no window argument."
},
"dropbox": {
"slots": "same as box: du/find are pushed down in typescript only, and the range read is python only."
"slots": "python range-reads; the typescript core read has no window argument."
},
"gdrive": {
"slots": "python range-reads; the typescript core read has no window argument."
spec/python/resources.json
+2
View File
@@ -426,6 +426,7 @@
"copy",
"create",
"dir_copy",
"du",
"exists",
"is_mounted",
"mkdir",
@@ -534,6 +535,7 @@
"slots": [
"copy",
"create",
"du",
"exists",
"is_mounted",
"mkdir",
spec/typescript/browser/resources.json
-2
View File
@@ -334,7 +334,6 @@
"dir_copy",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
@@ -416,7 +415,6 @@
"create",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
spec/typescript/node/resources.json
-2
View File
@@ -428,7 +428,6 @@
"dir_copy",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
@@ -537,7 +536,6 @@
"create",
"du",
"exists",
"find",
"is_mounted",
"mkdir",
"read_bytes",
typescript/packages/core/src/commands/builtin/box/io.ts
+7 -2
View File
@@ -14,7 +14,6 @@
import type { BoxAccessor } from '../../../accessor/box.ts'
import { size as boxDu, entries as boxDuAll } from '../../../core/box/du/index.ts'
import { find as boxFind } from '../../../core/box/find.ts'
import { read as boxRead, stream as boxStream } from '../../../core/box/read.ts'
import { readdir as boxReaddir } from '../../../core/box/readdir.ts'
import { stat as boxStat } from '../../../core/box/stat.ts'
@@ -51,5 +50,11 @@ export const BOX_IO: CommandIO<BoxAccessor> = {
dirCopy: boxCopy,
create: boxCreate,
truncate: boxTruncate,
find: boxFind,
// No `find` slot on purpose, matching python. A native op is worth
// wiring only when it pushes the search down to the API; Box has no
// such call, so the op could only re-walk readdir/stat — which is what
// the find and cp builders already do, except they walk with the
// mount's own index, the namespace stat overlay (`find -mtime` after a
// `touch -d`) and the symlink table (`-empty`). Wiring the walk as an
// op silently dropped all three.
}
typescript/packages/core/src/commands/builtin/dropbox/io.ts
+4 -2
View File
@@ -17,7 +17,6 @@ import { copy as dropboxCopy } from '../../../core/dropbox/copy.ts'
import { create as dropboxCreate } from '../../../core/dropbox/create.ts'
import { size as dropboxDu, entries as dropboxDuAll } from '../../../core/dropbox/du/index.ts'
import { exists as dropboxExists } from '../../../core/dropbox/exists.ts'
import { find as dropboxFind } from '../../../core/dropbox/find.ts'
import { mkdir as dropboxMkdir } from '../../../core/dropbox/mkdir.ts'
import { read as dropboxRead, stream as dropboxStream } from '../../../core/dropbox/read.ts'
import { readdir as dropboxReaddir } from '../../../core/dropbox/readdir.ts'
@@ -49,5 +48,8 @@ export const DROPBOX_IO: CommandIO<DropboxAccessor> = {
// rejects an existing destination).
copy: dropboxCopy,
create: dropboxCreate,
find: dropboxFind,
// No `find` slot on purpose, matching python — see the note in the Box
// table. Dropbox does have a recursive list_folder, so a real pushdown
// is possible here later; the op this replaced was not one, it was the
// builders' own walk minus the index, the stat overlay and the links.
}
typescript/packages/core/src/core/box/find.test.ts
-284
View File
@@ -1,284 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { mountKey } from '../../utils/key_prefix.ts'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as ReaddirModule from './readdir.ts'
import type * as StatModule from './stat.ts'
vi.mock('./readdir.ts', async () => {
const actual = await vi.importActual<typeof ReaddirModule>('./readdir.ts')
return { ...actual, readdir: vi.fn() }
})
vi.mock('./stat.ts', async () => {
const actual = await vi.importActual<typeof StatModule>('./stat.ts')
return { ...actual, stat: vi.fn() }
})
import { BoxAccessor } from '../../accessor/box.ts'
import { FileStat, FileType, PathSpec } from '../../types.ts'
import type { BoxTokenManager } from './_client.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { FindOptions } from '../../resource/base.ts'
import { walkFind } from '../generic/find.ts'
import * as readdirMod from './readdir.ts'
import * as statMod from './stat.ts'
async function find(
accessor: BoxAccessor,
path: PathSpec,
options: FindOptions = {},
index?: IndexCacheStore,
): Promise<string[]> {
return walkFind(
path,
{
readdir: (spec, idx) => readdirMod.readdir(accessor, spec, idx),
stat: (spec, idx) => statMod.stat(accessor, spec, idx),
},
options,
index,
)
}
const STUB_TM = {} as BoxTokenManager
function makeAccessor(): BoxAccessor {
return new BoxAccessor({ tokenManager: STUB_TM })
}
function enoent(p: string): Error {
const e = new Error(`ENOENT: ${p}`) as Error & { code: string }
e.code = 'ENOENT'
return e
}
function mockTree(tree: Record<string, string[]>): void {
vi.mocked(readdirMod.readdir).mockImplementation((_accessor, spec) => {
const children = tree[spec.virtual]
if (children === undefined) return Promise.reject(enoent(spec.virtual))
return Promise.resolve(children)
})
}
function mockStats(stats: Record<string, { size?: number; modified?: string }>): void {
vi.mocked(statMod.stat).mockImplementation((_accessor, spec) => {
const entry = stats[spec.virtual]
if (entry === undefined) return Promise.reject(enoent(spec.virtual))
const name = spec.virtual.split('/').pop() ?? ''
return Promise.resolve(
new FileStat({
name,
size: entry.size ?? null,
modified: entry.modified ?? null,
type: entry.size === undefined ? FileType.DIRECTORY : FileType.TEXT,
}),
)
})
}
const TREE: Record<string, string[]> = {
'/': ['/docs/', '/notes.txt'],
'/docs': ['/docs/readme.md', '/docs/inner/'],
'/docs/inner': ['/docs/inner/deep.md'],
}
const ROOT = new PathSpec({ resourcePath: '', virtual: '/', directory: '/' })
const SIZES: Record<string, { size?: number; modified?: string }> = {
'/docs': { modified: '2026-01-05T00:00:00Z' },
'/docs/inner': { modified: '2026-01-01T00:00:00Z' },
'/docs/inner/deep.md': { size: 500_000, modified: '2026-01-01T00:00:00Z' },
'/docs/readme.md': { size: 2048, modified: '2026-01-05T00:00:00Z' },
'/notes.txt': { size: 10, modified: '2026-01-10T00:00:00Z' },
}
describe('box core find', () => {
beforeEach(() => {
vi.mocked(readdirMod.readdir).mockReset()
vi.mocked(statMod.stat).mockReset()
mockStats(SIZES)
})
it('walks recursively returning files and dirs sorted without trailing slashes', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT)
expect(out).toEqual([
'/docs',
'/docs/inner',
'/docs/inner/deep.md',
'/docs/readme.md',
'/notes.txt',
])
})
it('filters by name glob', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { name: '*.md' })
expect(out).toEqual(['/docs/inner/deep.md', '/docs/readme.md'])
})
it('filters by type f and type d', async () => {
mockTree(TREE)
const files = await find(makeAccessor(), ROOT, { type: 'f' })
expect(files).toEqual(['/docs/inner/deep.md', '/docs/readme.md', '/notes.txt'])
const dirs = await find(makeAccessor(), ROOT, { type: 'd' })
expect(dirs).toEqual(['/docs', '/docs/inner'])
})
it('honors maxDepth and minDepth', async () => {
mockTree(TREE)
const shallow = await find(makeAccessor(), ROOT, { maxDepth: 1 })
expect(shallow).toEqual(['/docs', '/notes.txt'])
const deep = await find(makeAccessor(), ROOT, { minDepth: 2 })
expect(deep).toEqual(['/docs/inner', '/docs/inner/deep.md', '/docs/readme.md'])
})
it('strips the mount prefix from returned keys', async () => {
mockTree({
'/mnt/box': ['/mnt/box/docs/', '/mnt/box/notes.txt'],
'/mnt/box/docs': ['/mnt/box/docs/readme.md'],
})
const root = new PathSpec({
virtual: '/mnt/box',
directory: '/mnt/box',
resourcePath: mountKey('/mnt/box', '/mnt/box'),
})
const out = await find(makeAccessor(), root)
expect(out).toEqual(['/docs', '/docs/readme.md', '/notes.txt'])
})
it('does not stat slash-marked directory entries', async () => {
mockTree(TREE)
await find(makeAccessor(), ROOT, { name: '*.md' })
const statted = vi.mocked(statMod.stat).mock.calls.map((c) => c[1].virtual)
expect(statted).not.toContain('/docs')
expect(statted).not.toContain('/docs/inner')
})
it('filters by minSize with directories contributing size 0', async () => {
mockTree(TREE)
mockStats(SIZES)
const out = await find(makeAccessor(), ROOT, { minSize: 1024 })
expect(out).toEqual(['/docs/inner/deep.md', '/docs/readme.md'])
})
it('filters files by maxSize', async () => {
mockTree(TREE)
mockStats(SIZES)
const out = await find(makeAccessor(), ROOT, { maxSize: 100, type: 'f' })
expect(out).toEqual(['/notes.txt'])
})
it('stats the start path plus files for type detection and size filtering only', async () => {
mockTree(TREE)
await find(makeAccessor(), ROOT, { name: '*.md', minSize: 1024 })
const statted = [...new Set(vi.mocked(statMod.stat).mock.calls.map((c) => c[1].virtual))]
// '/' is the start-point stat that emits the search root itself.
expect(statted.sort()).toEqual(['/', '/docs/inner/deep.md', '/docs/readme.md', '/notes.txt'])
})
it('filters by mtimeMin and mtimeMax on files and dirs', async () => {
mockTree(TREE)
mockStats(SIZES)
const cutoff = Date.parse('2026-01-03T00:00:00Z') / 1000
const recent = await find(makeAccessor(), ROOT, { mtimeMin: cutoff })
expect(recent).toEqual(['/docs', '/docs/readme.md', '/notes.txt'])
const old = await find(makeAccessor(), ROOT, { mtimeMax: cutoff })
expect(old).toEqual(['/docs/inner', '/docs/inner/deep.md'])
})
it('excludes entries without a modified time when mtime filter is set', async () => {
mockTree(TREE)
mockStats({ ...SIZES, '/notes.txt': { size: 10 } })
const out = await find(makeAccessor(), ROOT, { mtimeMin: 0 })
expect(out).toEqual(['/docs', '/docs/inner', '/docs/inner/deep.md', '/docs/readme.md'])
})
it('filters by pathPattern against the full path', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { pathPattern: '*/inner/*' })
expect(out).toEqual(['/docs/inner/deep.md'])
})
it('matches pathPattern against the display path', async () => {
mockTree({
'/mnt/box': ['/mnt/box/docs/', '/mnt/box/notes.txt'],
'/mnt/box/docs': ['/mnt/box/docs/readme.md'],
})
const root = new PathSpec({
virtual: '/mnt/box',
directory: '/mnt/box',
resourcePath: mountKey('/mnt/box', '/mnt/box'),
})
const out = await find(makeAccessor(), root, { pathPattern: '/mnt/box/docs/*' })
expect(out).toEqual(['/docs/readme.md'])
})
it('matches any of orNames patterns', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { orNames: ['*.txt', 'deep.*'] })
expect(out).toEqual(['/docs/inner/deep.md', '/notes.txt'])
})
it('excludes names matching nameExclude', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { nameExclude: '*.md' })
expect(out).toEqual(['/docs', '/docs/inner', '/notes.txt'])
})
it('detects directories via stat when cached readdir entries lack trailing slashes', async () => {
mockTree({
'/': ['/docs', '/notes.txt'],
'/docs': ['/docs/readme.md'],
})
mockStats(SIZES)
const files = await find(makeAccessor(), ROOT, { type: 'f' })
expect(files).toEqual(['/docs/readme.md', '/notes.txt'])
const dirs = await find(makeAccessor(), ROOT, { type: 'd' })
expect(dirs).toEqual(['/docs'])
})
it('sorts by codepoint, not locale', async () => {
mockTree({ '/': ['/Zeta.txt', '/alpha.txt'] })
mockStats({ '/Zeta.txt': { size: 1 }, '/alpha.txt': { size: 1 } })
const out = await find(makeAccessor(), ROOT)
expect(out).toEqual(['/Zeta.txt', '/alpha.txt'])
})
it('keeps a child whose readdir raises ENOENT but stops descending', async () => {
mockTree({ '/': ['/ghost/'] })
const out = await find(makeAccessor(), ROOT)
expect(out).toEqual(['/ghost'])
})
it('propagates non-ENOENT readdir errors', async () => {
vi.mocked(readdirMod.readdir).mockImplementation((_accessor, spec) => {
if (spec.virtual === '/') return Promise.resolve(['/bad/'])
return Promise.reject(new Error('rate limited'))
})
await expect(find(makeAccessor(), ROOT)).rejects.toThrow('rate limited')
})
it('parses naive modified timestamps as UTC', async () => {
mockTree({ '/': ['/naive.txt'] })
mockStats({ '/naive.txt': { size: 1, modified: '2026-01-05T00:00:00' } })
const out = await find(makeAccessor(), ROOT, {
mtimeMin: Date.parse('2026-01-04T23:30:00Z') / 1000,
mtimeMax: Date.parse('2026-01-05T00:30:00Z') / 1000,
})
expect(out).toEqual(['/naive.txt'])
})
})
typescript/packages/core/src/core/box/find.ts
-42
View File
@@ -1,42 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { BoxAccessor } from '../../accessor/box.ts'
import { RAMIndexCacheStore } from '../../cache/index/ram.ts'
import type { FindOptions } from '../../resource/base.ts'
import type { PathSpec } from '../../types.ts'
import { walkFind } from '../generic/find.ts'
import { readdir } from './readdir.ts'
import { stat } from './stat.ts'
export function find(
accessor: BoxAccessor,
path: PathSpec,
options: FindOptions,
): Promise<string[]> {
// Box readdir/stat resolve folder ids through an index cache. The generic
// cp/find builders may call find without threading one (unlike Python,
// whose cp threads the resource index), so walk with a scratch index that
// this call populates as it descends.
const idx = new RAMIndexCacheStore({ ttl: 86_400 })
return walkFind(
path,
{
readdir: (spec, i) => readdir(accessor, spec, i),
stat: (spec, i) => stat(accessor, spec, i),
},
options,
idx,
)
}
typescript/packages/core/src/core/dropbox/find.test.ts
-284
View File
@@ -1,284 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { mountKey } from '../../utils/key_prefix.ts'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as ReaddirModule from './readdir.ts'
import type * as StatModule from './stat.ts'
vi.mock('./readdir.ts', async () => {
const actual = await vi.importActual<typeof ReaddirModule>('./readdir.ts')
return { ...actual, readdir: vi.fn() }
})
vi.mock('./stat.ts', async () => {
const actual = await vi.importActual<typeof StatModule>('./stat.ts')
return { ...actual, stat: vi.fn() }
})
import { DropboxAccessor } from '../../accessor/dropbox.ts'
import { FileStat, FileType, PathSpec } from '../../types.ts'
import type { DropboxTokenManager } from './_client.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { FindOptions } from '../../resource/base.ts'
import { walkFind } from '../generic/find.ts'
import * as readdirMod from './readdir.ts'
import * as statMod from './stat.ts'
async function find(
accessor: DropboxAccessor,
path: PathSpec,
options: FindOptions = {},
index?: IndexCacheStore,
): Promise<string[]> {
return walkFind(
path,
{
readdir: (spec, idx) => readdirMod.readdir(accessor, spec, idx),
stat: (spec, idx) => statMod.stat(accessor, spec, idx),
},
options,
index,
)
}
const STUB_TM = {} as DropboxTokenManager
function makeAccessor(): DropboxAccessor {
return new DropboxAccessor({ tokenManager: STUB_TM })
}
function enoent(p: string): Error {
const e = new Error(`ENOENT: ${p}`) as Error & { code: string }
e.code = 'ENOENT'
return e
}
function mockTree(tree: Record<string, string[]>): void {
vi.mocked(readdirMod.readdir).mockImplementation((_accessor, spec) => {
const children = tree[spec.virtual]
if (children === undefined) return Promise.reject(enoent(spec.virtual))
return Promise.resolve(children)
})
}
function mockStats(stats: Record<string, { size?: number; modified?: string }>): void {
vi.mocked(statMod.stat).mockImplementation((_accessor, spec) => {
const entry = stats[spec.virtual]
if (entry === undefined) return Promise.reject(enoent(spec.virtual))
const name = spec.virtual.split('/').pop() ?? ''
return Promise.resolve(
new FileStat({
name,
size: entry.size ?? null,
modified: entry.modified ?? null,
type: entry.size === undefined ? FileType.DIRECTORY : FileType.TEXT,
}),
)
})
}
const TREE: Record<string, string[]> = {
'/': ['/docs/', '/notes.txt'],
'/docs': ['/docs/readme.md', '/docs/inner/'],
'/docs/inner': ['/docs/inner/deep.md'],
}
const ROOT = new PathSpec({ resourcePath: '', virtual: '/', directory: '/' })
const SIZES: Record<string, { size?: number; modified?: string }> = {
'/docs': { modified: '2026-01-05T00:00:00Z' },
'/docs/inner': { modified: '2026-01-01T00:00:00Z' },
'/docs/inner/deep.md': { size: 500_000, modified: '2026-01-01T00:00:00Z' },
'/docs/readme.md': { size: 2048, modified: '2026-01-05T00:00:00Z' },
'/notes.txt': { size: 10, modified: '2026-01-10T00:00:00Z' },
}
describe('dropbox core find', () => {
beforeEach(() => {
vi.mocked(readdirMod.readdir).mockReset()
vi.mocked(statMod.stat).mockReset()
mockStats(SIZES)
})
it('walks recursively returning files and dirs sorted without trailing slashes', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT)
expect(out).toEqual([
'/docs',
'/docs/inner',
'/docs/inner/deep.md',
'/docs/readme.md',
'/notes.txt',
])
})
it('filters by name glob', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { name: '*.md' })
expect(out).toEqual(['/docs/inner/deep.md', '/docs/readme.md'])
})
it('filters by type f and type d', async () => {
mockTree(TREE)
const files = await find(makeAccessor(), ROOT, { type: 'f' })
expect(files).toEqual(['/docs/inner/deep.md', '/docs/readme.md', '/notes.txt'])
const dirs = await find(makeAccessor(), ROOT, { type: 'd' })
expect(dirs).toEqual(['/docs', '/docs/inner'])
})
it('honors maxDepth and minDepth', async () => {
mockTree(TREE)
const shallow = await find(makeAccessor(), ROOT, { maxDepth: 1 })
expect(shallow).toEqual(['/docs', '/notes.txt'])
const deep = await find(makeAccessor(), ROOT, { minDepth: 2 })
expect(deep).toEqual(['/docs/inner', '/docs/inner/deep.md', '/docs/readme.md'])
})
it('strips the mount prefix from returned keys', async () => {
mockTree({
'/mnt/dbx': ['/mnt/dbx/docs/', '/mnt/dbx/notes.txt'],
'/mnt/dbx/docs': ['/mnt/dbx/docs/readme.md'],
})
const root = new PathSpec({
virtual: '/mnt/dbx',
directory: '/mnt/dbx',
resourcePath: mountKey('/mnt/dbx', '/mnt/dbx'),
})
const out = await find(makeAccessor(), root)
expect(out).toEqual(['/docs', '/docs/readme.md', '/notes.txt'])
})
it('does not stat slash-marked directory entries', async () => {
mockTree(TREE)
await find(makeAccessor(), ROOT, { name: '*.md' })
const statted = vi.mocked(statMod.stat).mock.calls.map((c) => c[1].virtual)
expect(statted).not.toContain('/docs')
expect(statted).not.toContain('/docs/inner')
})
it('filters by minSize with directories contributing size 0', async () => {
mockTree(TREE)
mockStats(SIZES)
const out = await find(makeAccessor(), ROOT, { minSize: 1024 })
expect(out).toEqual(['/docs/inner/deep.md', '/docs/readme.md'])
})
it('filters files by maxSize', async () => {
mockTree(TREE)
mockStats(SIZES)
const out = await find(makeAccessor(), ROOT, { maxSize: 100, type: 'f' })
expect(out).toEqual(['/notes.txt'])
})
it('stats the start path plus files for type detection and size filtering only', async () => {
mockTree(TREE)
await find(makeAccessor(), ROOT, { name: '*.md', minSize: 1024 })
const statted = [...new Set(vi.mocked(statMod.stat).mock.calls.map((c) => c[1].virtual))]
// '/' is the start-point stat that emits the search root itself.
expect(statted.sort()).toEqual(['/', '/docs/inner/deep.md', '/docs/readme.md', '/notes.txt'])
})
it('filters by mtimeMin and mtimeMax on files and dirs', async () => {
mockTree(TREE)
mockStats(SIZES)
const cutoff = Date.parse('2026-01-03T00:00:00Z') / 1000
const recent = await find(makeAccessor(), ROOT, { mtimeMin: cutoff })
expect(recent).toEqual(['/docs', '/docs/readme.md', '/notes.txt'])
const old = await find(makeAccessor(), ROOT, { mtimeMax: cutoff })
expect(old).toEqual(['/docs/inner', '/docs/inner/deep.md'])
})
it('excludes entries without a modified time when mtime filter is set', async () => {
mockTree(TREE)
mockStats({ ...SIZES, '/notes.txt': { size: 10 } })
const out = await find(makeAccessor(), ROOT, { mtimeMin: 0 })
expect(out).toEqual(['/docs', '/docs/inner', '/docs/inner/deep.md', '/docs/readme.md'])
})
it('filters by pathPattern against the full path', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { pathPattern: '*/inner/*' })
expect(out).toEqual(['/docs/inner/deep.md'])
})
it('matches pathPattern against the display path', async () => {
mockTree({
'/mnt/dbx': ['/mnt/dbx/docs/', '/mnt/dbx/notes.txt'],
'/mnt/dbx/docs': ['/mnt/dbx/docs/readme.md'],
})
const root = new PathSpec({
virtual: '/mnt/dbx',
directory: '/mnt/dbx',
resourcePath: mountKey('/mnt/dbx', '/mnt/dbx'),
})
const out = await find(makeAccessor(), root, { pathPattern: '/mnt/dbx/docs/*' })
expect(out).toEqual(['/docs/readme.md'])
})
it('matches any of orNames patterns', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { orNames: ['*.txt', 'deep.*'] })
expect(out).toEqual(['/docs/inner/deep.md', '/notes.txt'])
})
it('excludes names matching nameExclude', async () => {
mockTree(TREE)
const out = await find(makeAccessor(), ROOT, { nameExclude: '*.md' })
expect(out).toEqual(['/docs', '/docs/inner', '/notes.txt'])
})
it('detects directories via stat when cached readdir entries lack trailing slashes', async () => {
mockTree({
'/': ['/docs', '/notes.txt'],
'/docs': ['/docs/readme.md'],
})
mockStats(SIZES)
const files = await find(makeAccessor(), ROOT, { type: 'f' })
expect(files).toEqual(['/docs/readme.md', '/notes.txt'])
const dirs = await find(makeAccessor(), ROOT, { type: 'd' })
expect(dirs).toEqual(['/docs'])
})
it('sorts by codepoint, not locale', async () => {
mockTree({ '/': ['/Zeta.txt', '/alpha.txt'] })
mockStats({ '/Zeta.txt': { size: 1 }, '/alpha.txt': { size: 1 } })
const out = await find(makeAccessor(), ROOT)
expect(out).toEqual(['/Zeta.txt', '/alpha.txt'])
})
it('keeps a child whose readdir raises ENOENT but stops descending', async () => {
mockTree({ '/': ['/ghost/'] })
const out = await find(makeAccessor(), ROOT)
expect(out).toEqual(['/ghost'])
})
it('propagates non-ENOENT readdir errors', async () => {
vi.mocked(readdirMod.readdir).mockImplementation((_accessor, spec) => {
if (spec.virtual === '/') return Promise.resolve(['/bad/'])
return Promise.reject(new Error('rate limited'))
})
await expect(find(makeAccessor(), ROOT)).rejects.toThrow('rate limited')
})
it('parses naive modified timestamps as UTC', async () => {
mockTree({ '/': ['/naive.txt'] })
mockStats({ '/naive.txt': { size: 1, modified: '2026-01-05T00:00:00' } })
const out = await find(makeAccessor(), ROOT, {
mtimeMin: Date.parse('2026-01-04T23:30:00Z') / 1000,
mtimeMax: Date.parse('2026-01-05T00:30:00Z') / 1000,
})
expect(out).toEqual(['/naive.txt'])
})
})
typescript/packages/core/src/core/dropbox/find.ts
-43
View File
@@ -1,43 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { DropboxAccessor } from '../../accessor/dropbox.ts'
import { RAMIndexCacheStore } from '../../cache/index/ram.ts'
import type { FindOptions } from '../../resource/base.ts'
import type { PathSpec } from '../../types.ts'
import { walkFind } from '../generic/find.ts'
import { readdir } from './readdir.ts'
import { stat } from './stat.ts'
// Same readdir/stat walk the generic fallback uses, wired as a find op
// so the cp builder (which plans recursive copies through find) works.
export function find(
accessor: DropboxAccessor,
path: PathSpec,
options: FindOptions,
): Promise<string[]> {
// An index-less dropbox stat hits files/get_metadata per child, so a
// bare walk is an N+1 API sweep. Walk with a scratch index that the
// readdirs populate as the walk descends, like the Box find does.
const idx = new RAMIndexCacheStore({ ttl: 86_400 })
return walkFind(
path,
{
readdir: (spec, index) => readdir(accessor, spec, index),
stat: (spec, index) => stat(accessor, spec, index),
},
options,
idx,
)
}