github-featured/strukto-ai--mirage
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix(commands): tar/unzip member selectors and relay extraction, grep file filters, direct path execution

Browse Source
This commit is contained in:
Zecheng Zhang
2026-08-16 00:52:36 -07:00
parent 14f83208ab
commit 3e614781a6
63 changed files with 3085 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
integ/bash/exec/path.json
+190
View File
@@ -0,0 +1,190 @@
{
"cases": [
{
"id": "exec_path_runs_a_script_on_a_mount",
"seq": 962330,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"gridfs",
"s3-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "printf 'echo ran from script\\n' > /data/run.sh && /data/run.sh",
"expect": {
"exit": 0,
"stdout": "ran from script\n",
"stderr": ""
}
},
{
"id": "exec_path_relative_resolves_against_cwd",
"seq": 962331,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"gridfs",
"s3-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "bash -c 'cd /data && ./run.sh'",
"expect": {
"exit": 0,
"stdout": "ran from script\n",
"stderr": ""
}
},
{
"id": "exec_path_shebang_options_apply",
"seq": 962332,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"gridfs",
"s3-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "printf '#!/bin/sh -x\\necho traced\\n' > /data/t.sh && /data/t.sh",
"expect": {
"exit": 0,
"stdout": "traced\n",
"stderr": "+ echo traced\n"
}
},
{
"id": "exec_path_missing_file_is_127",
"seq": 962333,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"gridfs",
"s3-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "/data/nope.sh; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=127\n",
"stderr": "/data/nope.sh: No such file or directory\n"
}
},
{
"id": "exec_path_directory_is_126",
"seq": 962334,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"gridfs",
"s3-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data/adir && /data/adir; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=126\n",
"stderr": "/data/adir: Is a directory\n"
}
},
{
"id": "exec_path_unknown_interpreter_is_127",
"seq": 962335,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"gridfs",
"s3-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "printf '#!/usr/bin/env ruby\\nputs 1\\n' > /data/r.rb && /data/r.rb; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=127\n",
"stderr": "ruby: command not found\n"
}
}
]
}
integ/crossmount/tar/extract.json
+106
View File
@@ -0,0 +1,106 @@
{
"cases": [
{
"id": "xm_tar_extract_dash_C_into_the_other_mount",
"seq": 962310,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data/xtar && echo xm | tee /data/xtar/f.txt > /dev/null && tar -czf /data/xtar.tgz -C /data xtar && tar -xzf /data/xtar.tgz -C /data2/xout && cat /data2/xout/xtar/f.txt",
"expect": {
"exit": 0,
"stdout": "xm\n",
"stderr": ""
}
},
{
"id": "xm_tar_selector_does_not_join_routing",
"seq": 962311,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "tar -xOzf /data/xtar.tgz xtar/f.txt",
"expect": {
"exit": 0,
"stdout": "xm\n",
"stderr": ""
}
},
{
"id": "xm_tar_create_span_keeps_the_refusal",
"seq": 962312,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data2/xsrc && echo s | tee /data2/xsrc/s.txt > /dev/null && tar -czf /data/xspan.tgz /data2/xsrc; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=1\n",
"stderr": "tar: paths span multiple mounts (/data/, /data2/), cross-mount not supported\n"
}
}
]
}
integ/crossmount/unzip/extract.json
+38
View File
@@ -0,0 +1,38 @@
{
"cases": [
{
"id": "xm_unzip_dash_d_into_the_other_mount",
"seq": 962315,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data/xzip && echo zx | tee /data/xzip/z.txt > /dev/null && zip -q -r /data/xz.zip /data/xzip && unzip -q -d /data2/xzout /data/xz.zip && cat /data2/xzout/data/xzip/z.txt",
"expect": {
"exit": 0,
"stdout": "zx\n",
"stderr": ""
}
}
]
}
integ/unix/arch/tar.json
+206 -2
View File
@@ -529,13 +529,217 @@
"command": "mkdir -p /data/ml && echo m > /data/ml/m.txt && tar -cf /data/ml.tar -C /data ml && tar -tf /data/ml.tar data; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "ml/\nml/m.txt\nexit=0\n",
"stderr": ""
"stdout": "exit=2\n",
"stderr": "tar: data: Not found in archive\ntar: Exiting with failure status due to previous errors\n"
},
"flags": [
"c",
"C"
]
},
{
"id": "arch_tar_member_selector_lists_the_subtree",
"seq": 962301,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data/msel/sub && echo one | tee /data/msel/a.txt > /dev/null && echo two | tee /data/msel/sub/b.txt > /dev/null && tar -czf /data/msel.tgz -C /data msel && tar -tzf /data/msel.tgz msel/sub",
"expect": {
"exit": 0,
"stdout": "msel/sub/\nmsel/sub/b.txt\n",
"stderr": ""
}
},
{
"id": "arch_tar_member_selector_miss_exits_2",
"seq": 962302,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "tar -tzf /data/msel.tgz nope; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=2\n",
"stderr": "tar: nope: Not found in archive\ntar: Exiting with failure status due to previous errors\n"
}
},
{
"id": "arch_tar_extract_to_stdout",
"seq": 962303,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "tar -xOzf /data/msel.tgz msel/a.txt",
"expect": {
"exit": 0,
"stdout": "one\n",
"stderr": ""
}
},
{
"id": "arch_tar_xvO_lists_names_on_stderr",
"seq": 962304,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "tar -xvOzf /data/msel.tgz msel/a.txt",
"expect": {
"exit": 0,
"stdout": "one\n",
"stderr": "msel/a.txt\n"
}
},
{
"id": "arch_tar_extract_selector_takes_only_its_subtree",
"seq": 962305,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "tar -xzf /data/msel.tgz -C /data/mselout msel/sub && ls /data/mselout/msel && cat /data/mselout/msel/sub/b.txt",
"expect": {
"exit": 0,
"stdout": "sub\ntwo\n",
"stderr": ""
}
},
{
"id": "arch_tar_extract_defaults_to_the_cwd",
"seq": 962306,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data/cwdx && bash -c 'cd /data/cwdx && tar -xzf /data/msel.tgz && ls msel'",
"expect": {
"exit": 0,
"stdout": "a.txt\nsub\n",
"stderr": ""
}
}
]
}
integ/unix/grep/filters.json
+242
View File
@@ -0,0 +1,242 @@
{
"cases": [
{
"id": "grep_include_filters_the_walk",
"seq": 962320,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "mkdir -p /data/gflt/sub && echo 'score 9' | tee /data/gflt/notes.tex > /dev/null && echo 'score 8' | tee /data/gflt/notes.txt > /dev/null && echo 'score 7' | tee /data/gflt/sub/inner.tex > /dev/null && grep -RInE --include='*.tex' score /data/gflt",
"expect": {
"exit": 0,
"stdout": "/data/gflt/notes.tex:1:score 9\n/data/gflt/sub/inner.tex:1:score 7\n",
"stderr": ""
}
},
{
"id": "grep_exclude_wins_over_include",
"seq": 962321,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "grep -r --include='*.tex' --exclude='notes.*' score /data/gflt",
"expect": {
"exit": 0,
"stdout": "/data/gflt/sub/inner.tex:score 7\n",
"stderr": ""
}
},
{
"id": "grep_exclude_dir_prunes_the_walk",
"seq": 962322,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "grep -r --include='*.tex' --exclude-dir=sub score /data/gflt",
"expect": {
"exit": 0,
"stdout": "/data/gflt/notes.tex:score 9\n",
"stderr": ""
}
},
{
"id": "grep_include_with_a_slash_matches_nothing",
"seq": 962323,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "grep -r --include='sub/*.tex' score /data/gflt; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=1\n",
"stderr": ""
}
},
{
"id": "grep_include_filters_an_explicit_operand_in_silence",
"seq": 962324,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "grep --include='*.tex' -n score /data/gflt/notes.txt; echo exit=$?",
"expect": {
"exit": 0,
"stdout": "exit=1\n",
"stderr": ""
}
},
{
"id": "grep_dash_a_reads_binary_extensions_in_the_walk",
"seq": 962325,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "echo 'score binary' | tee /data/gflt/d.parquet > /dev/null && grep -r score /data/gflt | grep -c parquet; grep -ra score /data/gflt | grep -c parquet",
"expect": {
"exit": 0,
"stdout": "0\n1\n",
"stderr": ""
}
},
{
"id": "grep_dash_a_is_accepted_on_an_explicit_operand",
"seq": 962326,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "grep -aoiE 'SCORE' /data/gflt/notes.tex",
"expect": {
"exit": 0,
"stdout": "score\n",
"stderr": ""
}
}
]
}
python/mirage/commands/builtin/generic/archive/extract.py
+80
View File
@@ -0,0 +1,80 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import Awaitable, Callable
from mirage.commands.builtin.generic.archive.walk import StatFn
from mirage.types import FileType, PathSpec
from mirage.utils.errors import FS_ERRORS
def extract_dest(explicit: PathSpec | str | None, cwd: PathSpec | str,
relay: bool) -> str:
"""Where extraction lands: the explicit operand, else the cwd.
Relay doors route by full virtual path, accessor doors by
mount-relative path, so the same operand renders differently per
door space. Outside a workspace the cwd arrives as a plain string
and the two spaces coincide.
Args:
explicit (PathSpec | str | None): tar's last -C or unzip's -d,
when the line named one.
cwd (PathSpec | str): the session working directory.
relay (bool): True when the doors are dispatch-relayed.
"""
target = explicit if explicit is not None else cwd
if isinstance(target, PathSpec):
return target.virtual if relay else target.mount_path
return target or "/"
async def dir_exists(stat: StatFn, level: str) -> bool:
"""Whether a directory already stands at this path.
Args:
stat (StatFn): stat door in the same path space as ``level``.
level (str): the directory path to probe.
"""
try:
found = await stat(PathSpec.from_str_path(level))
except FS_ERRORS:
return False
return found is not None and found.type == FileType.DIRECTORY
async def ensure_dir(dir_path: str, mkdir_fn: Callable[..., Awaitable[None]],
stat: StatFn, made: set[str]) -> None:
"""Create one directory chain top-down, skipping what exists.
The dispatch mkdir op is single-level on most backends (the
``mkdir_parents`` knob is a per-backend exception), so extraction
walks the chain itself the way relay cp does: probe, then create
only what is missing, memoized per run so an archive of many files
stats each ancestor once.
Args:
dir_path (str): the directory whose chain must exist.
mkdir_fn (Callable): mkdir door, single level.
stat (StatFn): stat door in the same path space.
made (set[str]): levels already ensured this run, updated here.
"""
parts = [p for p in dir_path.strip("/").split("/") if p]
for i in range(1, len(parts) + 1):
level = "/" + "/".join(parts[:i])
if level in made:
continue
if not await dir_exists(stat, level):
await mkdir_fn(PathSpec.from_str_path(level))
made.add(level)
python/mirage/commands/builtin/generic/crossmount/constants.py
+4 -2
View File
@@ -22,6 +22,8 @@ FANOUT_COMMANDS = frozenset({
Cmd.STAT, Cmd.STRINGS, Cmd.TAC, Cmd.LS, Cmd.FIND, Cmd.RM, Cmd.RMDIR,
Cmd.UNLINK, Cmd.TOUCH, Cmd.MKDIR, Cmd.TEE
})
RELAY_COMMANDS = frozenset(
{Cmd.CP, Cmd.MV, Cmd.DIFF, Cmd.CMP, Cmd.PASTE, Cmd.COMM, Cmd.JOIN})
RELAY_COMMANDS = frozenset({
Cmd.CP, Cmd.MV, Cmd.DIFF, Cmd.CMP, Cmd.PASTE, Cmd.COMM, Cmd.JOIN, Cmd.TAR,
Cmd.UNZIP
})
CROSS_MOUNT_COMMANDS = STREAM_COMMANDS | FANOUT_COMMANDS | RELAY_COMMANDS
python/mirage/commands/builtin/generic/crossmount/relay/relay.py
+10 -1
View File
@@ -21,6 +21,8 @@ from mirage.commands.builtin.generic.crossmount.relay.diff import run_diff
from mirage.commands.builtin.generic.crossmount.relay.join import run_join
from mirage.commands.builtin.generic.crossmount.relay.mv import run_mv
from mirage.commands.builtin.generic.crossmount.relay.paste import run_paste
from mirage.commands.builtin.generic.crossmount.relay.tar import run_tar
from mirage.commands.builtin.generic.crossmount.relay.unzip import run_unzip
from mirage.commands.builtin.generic.crossmount.types import Cmd, CrossResult
from mirage.commands.spec.types import FlagValue
from mirage.runtime.types import DispatchFn
@@ -30,6 +32,7 @@ from mirage.types import PathSpec
async def run_relay(
cmd_name: str,
scopes: list[PathSpec],
text_args: list[str],
flag_kwargs: dict[str, FlagValue],
dispatch: DispatchFn,
storage_key: Callable[[PathSpec], str] | None = None) -> CrossResult:
@@ -40,8 +43,10 @@ async def run_relay(
its primitive mode, so output matches the single-mount commands.
Args:
cmd_name (str): One of cp, mv, diff, cmp, paste, comm, join.
cmd_name (str): One of cp, mv, diff, cmp, paste, comm, join, tar.
scopes (list[PathSpec]): Path operands in command-line order.
text_args (list[str]): Positional text operands (tar's member
selectors; empty for the transfer and merge commands).
flag_kwargs (dict): Flags parsed against the shared command spec.
dispatch (DispatchFn): Workspace operation dispatcher.
storage_key (Callable | None): Maps an operand to its storage
@@ -60,4 +65,8 @@ async def run_relay(
return await run_comm(scopes, flag_kwargs, dispatch)
if cmd_name == Cmd.JOIN:
return await run_join(scopes, flag_kwargs, dispatch)
if cmd_name == Cmd.TAR:
return await run_tar(scopes, text_args, flag_kwargs, dispatch)
if cmd_name == Cmd.UNZIP:
return await run_unzip(scopes, text_args, flag_kwargs, dispatch)
return await run_cmp(scopes, flag_kwargs, dispatch)
python/mirage/commands/builtin/generic/crossmount/relay/tar.py
+76
View File
@@ -0,0 +1,76 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic.crossmount.types import CrossResult
from mirage.commands.builtin.generic.crossmount.utils import \
transfer_primitives
from mirage.commands.builtin.generic.tar.tar import tar
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagValue, FlagView
from mirage.runtime.types import DispatchFn
from mirage.types import PathSpec
async def _no_walk(path: PathSpec, find_type: str) -> list[str]:
raise ValueError("tar: create never runs on the relay tier")
async def _no_dir_probe(path: PathSpec) -> bool:
raise ValueError("tar: create never runs on the relay tier")
async def run_tar(scopes: list[PathSpec], text_args: list[str],
flag_kwargs: dict[str, FlagValue],
dispatch: DispatchFn) -> CrossResult:
"""Run a -t/-x tar whose archive and -C destination span mounts.
Pure wiring: the shared generic runs on dispatch-relayed doors, so
the archive is read from its mount and every extracted path lands
on whichever mount owns it. Create mode never reaches here: the
executor keeps a create-mode span on the plain refusal, because its
planner walks one backend's tree and relay doors would cross nested
mount boundaries the planner is required to refuse.
Args:
scopes (list[PathSpec]): Path operands in command-line order.
text_args (list[str]): The -t/-x member selectors, as typed.
flag_kwargs (dict): Flags parsed against the shared tar spec,
with path-valued flags as resolved virtual strings.
dispatch (DispatchFn): Workspace operation dispatcher.
"""
fl = FlagView(flag_kwargs, spec=SPECS["tar"])
prim = transfer_primitives(dispatch)
archive = fl.as_str("f")
directories = [str(part) for part in fl.as_list("C")]
return await tar(
[],
read_bytes=prim["read_bytes"],
write_bytes=prim["write"],
mkdir_fn=prim["mkdir"],
stat=prim["stat"],
walk=_no_walk,
is_dir=_no_dir_probe,
selectors=list(text_args),
x=fl.as_bool("x"),
t=fl.as_bool("t"),
z=fl.as_bool("z"),
j=fl.as_bool("j"),
J=fl.as_bool("J"),
v=fl.as_bool("v"),
to_stdout=fl.as_bool("to_stdout"),
f=PathSpec.from_str_path(archive) if archive else None,
C=[PathSpec.from_str_path(d) for d in directories] or None,
strip_components=fl.as_str("strip_components"),
relay=True,
)
python/mirage/commands/builtin/generic/crossmount/relay/unzip.py
+61
View File
@@ -0,0 +1,61 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic.crossmount.types import CrossResult
from mirage.commands.builtin.generic.crossmount.utils import \
transfer_primitives
from mirage.commands.builtin.generic.unzip import unzip
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagValue, FlagView
from mirage.runtime.types import DispatchFn
from mirage.types import PathSpec
async def run_unzip(scopes: list[PathSpec], text_args: list[str],
flag_kwargs: dict[str, FlagValue],
dispatch: DispatchFn) -> CrossResult:
"""Run an unzip whose archive and -d destination span mounts.
Pure wiring: the shared generic runs on dispatch-relayed doors, so
the archive is read from its mount and every extracted path lands
on whichever mount owns it.
Args:
scopes (list[PathSpec]): Path operands; the archive is first.
text_args (list[str]): Info-ZIP member filespecs, as typed.
flag_kwargs (dict): Flags parsed against the shared unzip spec,
with path-valued flags as resolved virtual strings.
dispatch (DispatchFn): Workspace operation dispatcher.
"""
fl = FlagView(flag_kwargs, spec=SPECS["unzip"])
prim = transfer_primitives(dispatch)
# Scopes arrive in line order and include the -d flag's value, so
# the archive is the first scope that is not the destination.
dest = fl.as_str("d")
operands = [s for s in scopes if s.virtual != dest]
return await unzip(
operands or scopes,
read_bytes=prim["read_bytes"],
write_bytes=prim["write"],
mkdir_fn=prim["mkdir"],
stat=prim["stat"],
members=tuple(text_args),
o=fl.as_bool("o"),
args_l=fl.as_bool("args_l"),
d=fl.as_str("d"),
q=fl.as_bool("q"),
p=fl.as_bool("p"),
t=fl.as_bool("t"),
relay=True,
)
python/mirage/commands/builtin/generic/crossmount/route.py
+2 -2
View File
@@ -66,8 +66,8 @@ async def handle_cross_mount(
try:
strategy = strategy_for(cmd_name, flag_kwargs)
if strategy is Strategy.RELAY:
return await run_relay(cmd_name, scopes, flag_kwargs, dispatch,
storage_key)
return await run_relay(cmd_name, scopes, text_args, flag_kwargs,
dispatch, storage_key)
if strategy is Strategy.STREAM:
return await run_stream(cmd_name, scopes, text_args, flag_kwargs,
run_single)
python/mirage/commands/builtin/generic/crossmount/types.py
+2
View File
@@ -85,6 +85,8 @@ class Cmd(StrEnum):
PASTE = "paste"
COMM = "comm"
JOIN = "join"
TAR = "tar"
UNZIP = "unzip"
CrossResult = tuple[ByteSource | None, IOResult]
python/mirage/commands/builtin/generic/grep.py
+25 -4
View File
@@ -5,10 +5,15 @@ from functools import partial
from mirage.cache.read_through import (cache_aware_bound_bytes,
cache_aware_bound_stream)
from mirage.commands.builtin.grep_helper import ( # yapf: disable
compile_pattern, count_exit_stream, count_records_have_matches,
exit_code_for, grep_files_only, grep_lines, grep_recursive, grep_stream,
prefix_lines, resolve_pattern)
from mirage.commands.builtin.grep_helper import WalkFilters # yapf: disable
from mirage.commands.builtin.grep_helper import \
compile_pattern # yapf: disable
from mirage.commands.builtin.grep_helper import (count_exit_stream,
count_records_have_matches,
exit_code_for, file_admitted,
grep_files_only, grep_lines,
grep_recursive, grep_stream,
prefix_lines, resolve_pattern)
from mirage.commands.builtin.utils.lines import split_lines
from mirage.commands.builtin.utils.output import (format_optional_records,
format_records)
@@ -44,6 +49,7 @@ class GrepFlags:
max_count: int | None
after_context: int
before_context: int
filters: WalkFilters
def parse_flags(fl: FlagView, never_match: bool) -> GrepFlags:
@@ -76,6 +82,10 @@ def parse_flags(fl: FlagView, never_match: bool) -> GrepFlags:
max_count=fl.as_int("m"),
after_context=a_ctx if a_ctx is not None else (c_ctx or 0),
before_context=b_ctx if b_ctx is not None else (c_ctx or 0),
filters=WalkFilters(include=tuple(fl.as_list("include")),
exclude=tuple(fl.as_list("exclude")),
exclude_dir=tuple(fl.as_list("exclude_dir")),
text=fl.as_bool("text")),
)
@@ -149,6 +159,7 @@ async def grep(
basic=f.basic_regexp,
warnings=warnings,
read_stream_fn=None,
filters=f.filters,
)
results.extend(respell_raw(hits, p.virtual, p.raw_path))
stderr = format_optional_records(warnings)
@@ -197,8 +208,11 @@ async def grep(
max_count=f.max_count,
warnings=warnings,
read_stream_fn=None,
filters=f.filters,
)
all_results.extend(respell_raw(res, p.virtual, p.raw_path))
elif not file_admitted(p.virtual, f.filters):
continue
else:
data = split_lines(
(await rb(p.virtual)).decode(errors="replace"))
@@ -237,6 +251,8 @@ async def grep(
multi_warnings.append(
f"grep: {p.raw_path}: Is a directory")
continue
if not file_admitted(p.virtual, f.filters):
continue
data = split_lines((await
rb(p.virtual)).decode(errors="replace"))
# -l returned at the top of this function, so every path
@@ -272,6 +288,11 @@ async def grep(
stderr = f"grep: {paths[0].raw_path}: Is a directory\n".encode()
return b"", IOResult(exit_code=2, stderr=stderr)
if not file_admitted(paths[0].virtual, f.filters):
# GNU passes over a command-line file --include leaves out
# in silence: no output, no diagnostic, exit "no match".
return b"", IOResult(exit_code=1)
if read_stream is not None:
source: AsyncIterator[bytes] = read_stream(paths[0])
else:
python/mirage/commands/builtin/generic/tar/mode.py
+48
View File
@@ -0,0 +1,48 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from collections.abc import Sequence
def is_create_mode(argv: Sequence[str]) -> bool:
"""Whether a tar line reads the filesystem rather than an archive.
Only ``-c`` makes tar's operands source paths. Under ``-t`` and
``-x`` they are member selectors matched against names inside the
archive, so a selector that happens to spell a mount root is not a
mount at all and refusing it would deny an ordinary listing.
Scanned raw because both callers fire before flag parsing: the
mount-root policy at admission, and the cross-mount router deciding
whether a span is real. Only the first word may be GNU's dashless
option cluster (``tar cf a.tar d``), so a later bare word is an
operand and cannot turn the mode on.
Args:
argv (Sequence[str]): raw argv after the command name.
"""
for i, tok in enumerate(argv):
if not isinstance(tok, str):
continue
if tok == "--create":
return True
if tok.startswith("--"):
continue
if tok.startswith("-"):
if "c" in tok[1:]:
return True
continue
if i == 0 and "c" in tok:
return True
return False
python/mirage/commands/builtin/generic/tar/tar.py
+170 -24
View File
@@ -3,6 +3,8 @@ import tarfile
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from mirage.commands.builtin.generic.archive.extract import (ensure_dir,
extract_dest)
from mirage.commands.builtin.generic.archive.walk import (DirProbe, StatFn,
WalkFn)
from mirage.commands.builtin.generic.tar.constants import (READ_MODES,
@@ -41,6 +43,79 @@ def _stderr(lines: list[str]) -> bytes:
return ("\n".join(lines) + "\n").encode() if lines else b""
MISS_TRAILER = "tar: Exiting with failure status due to previous errors"
DOTDOT_NOTICE = "tar: Removing leading `../' from member names"
def _matches(name: str, selector: str) -> bool:
"""Whether one -t/-x member selector keeps an archive member.
GNU matches the stored spelling exactly (``memory/x`` does not find
``./memory/x``), and a selector naming a directory takes its whole
subtree, with or without the trailing slash.
Args:
name (str): the member name as stored in the archive.
selector (str): the operand as typed.
"""
base = selector.rstrip("/")
trimmed = name.rstrip("/")
return trimmed == base or trimmed.startswith(base + "/")
def _selected(names: list[str],
selectors: list[str]) -> tuple[set[int], list[str]]:
"""Member indices the selectors keep, and the misses they report.
No selector keeps everything. A selector that matches nothing is
GNU's per-operand diagnostic, reported in operand order; the caller
appends the one failure trailer.
Args:
names (list[str]): member names in archive order.
selectors (list[str]): the -t/-x operands as typed.
"""
if not selectors:
return set(range(len(names))), []
keep: set[int] = set()
misses: list[str] = []
for sel in selectors:
hit = False
for idx, name in enumerate(names):
if _matches(name, sel):
keep.add(idx)
hit = True
if not hit:
misses.append(f"tar: {sel}: Not found in archive")
return keep, misses
def _out_parts(name: str, strip_n: int, notices: list[str]) -> list[str]:
"""The destination-relative components one member extracts to.
GNU strips ``--strip-components`` off the stored spelling first, in
which a leading ``.`` counts as a component (``--strip-components=1``
turns ``./a/b`` into ``a/b``). Only then is the remainder cleaned
for the filesystem: ``.`` components vanish (a real OS resolves
them; a virtual path must not keep a literal ``.`` directory) and a
leading ``..`` is removed with GNU's one notice per run.
Args:
name (str): the member name as stored in the archive.
strip_n (int): components to strip off the stored name.
notices (list[str]): run-level notice sink, appended in place.
"""
parts = name.rstrip("/").split("/")
if strip_n > 0:
parts = parts[strip_n:]
parts = [p for p in parts if p not in ("", ".")]
while parts and parts[0] == "..":
if DOTDOT_NOTICE not in notices:
notices.append(DOTDOT_NOTICE)
parts.pop(0)
return parts
def _info(member: Member, size: int) -> tarfile.TarInfo:
"""The header for one member, typed the way its kind demands.
@@ -88,6 +163,7 @@ async def _create_archive(
async def _list_archive(
archive_path: PathSpec,
mode_suffix: CompressionSuffix,
selectors: list[str],
read_bytes: Callable[..., Awaitable[bytes]],
) -> tuple[ByteSource | None, IOResult]:
data = await read_bytes(archive_path)
@@ -97,7 +173,13 @@ async def _list_archive(
member.name + "/" if member.isdir() else member.name
for member in tf.getmembers()
]
return ("\n".join(names) + "\n").encode(), IOResult()
keep, misses = _selected(names, selectors)
shown = [name for idx, name in enumerate(names) if idx in keep]
stdout = ("\n".join(shown) + "\n").encode() if shown else None
if misses:
return stdout, IOResult(exit_code=2,
stderr=_stderr(misses + [MISS_TRAILER]))
return stdout, IOResult()
async def _extract_archive(
@@ -106,47 +188,86 @@ async def _extract_archive(
mode_suffix: CompressionSuffix,
strip_n: int,
verbose: bool,
to_stdout: bool,
selectors: list[str],
relay: bool,
read_bytes: Callable[..., Awaitable[bytes]],
write_bytes: Callable[..., Awaitable[None]],
mkdir_fn: Callable[..., Awaitable[None]],
stat: StatFn,
) -> tuple[ByteSource | None, IOResult]:
data = await read_bytes(archive_path)
writes: dict[str, ByteSource] = {}
names: list[str] = []
notices: list[str] = []
made: set[str] = set()
extracted_bytes: list[bytes] = []
with tarfile.open(fileobj=io.BytesIO(data),
mode=_read_mode(mode_suffix)) as tf:
for member in tf.getmembers():
members = tf.getmembers()
listed = [
member.name + "/" if member.isdir() else member.name
for member in members
]
keep, misses = _selected(listed, selectors)
for idx, member in enumerate(members):
if idx not in keep:
continue
# A symlink member has no bytes to write and no namespace to
# write into from here (links are workspace state, not the
# backend's), so extraction skips it rather than dropping an
# empty file where a link belongs.
if not member.isfile() and not member.isdir():
continue
name_parts = member.name.rstrip("/").split("/")
if strip_n > 0:
name_parts = name_parts[strip_n:]
if not name_parts or name_parts == [""]:
continue
out_path = dest_path.rstrip("/") + "/" + "/".join(name_parts)
if member.isdir():
# A directory member is the only record an empty
# directory leaves, so it has to be recreated even
# though nothing is written inside it.
await mkdir_fn(PathSpec.from_str_path(out_path), parents=True)
names.append(member.name.rstrip("/") + "/")
if not to_stdout:
# A directory member is the only record an empty
# directory leaves, so it has to be recreated even
# though nothing is written inside it. Under -O
# nothing reaches the filesystem at all.
parts = _out_parts(member.name, strip_n, notices)
if parts:
out_dir = dest_path.rstrip("/") + "/" + "/".join(parts)
await ensure_dir(out_dir, mkdir_fn, stat, made)
names.append(member.name.rstrip("/") + "/")
continue
extracted = tf.extractfile(member)
if not extracted:
continue
content = extracted.read()
if to_stdout:
extracted_bytes.append(content)
names.append(member.name)
continue
parts = _out_parts(member.name, strip_n, notices)
if not parts:
continue
out_path = dest_path.rstrip("/") + "/" + "/".join(parts)
parent = out_path.rsplit("/", 1)[0] or "/"
if parent != "/":
await mkdir_fn(PathSpec.from_str_path(parent), parents=True)
await write_bytes(PathSpec.from_str_path(out_path), content)
writes[out_path] = content
await ensure_dir(parent, mkdir_fn, stat, made)
await write_bytes(PathSpec.from_str_path(out_path), data=content)
if not relay:
# Relay writes land on whichever mount owns each path
# and invalidate through the dispatcher; keying them here
# would have the runner prefix them onto this mount.
writes[out_path] = content
names.append(member.name)
stdout = ("\n".join(names) + "\n").encode() if verbose and names else None
return stdout, IOResult(writes=writes)
if to_stdout:
# GNU moves the verbose listing to stderr when stdout carries
# the member bytes.
stdout: ByteSource | None = b"".join(extracted_bytes) or None
stderr_lines = notices + (names if verbose else [])
else:
listing = ("\n".join(names) + "\n").encode() if verbose and names \
else None
stdout = listing
stderr_lines = list(notices)
if misses:
stderr_lines = stderr_lines + misses + [MISS_TRAILER]
return stdout, IOResult(exit_code=2 if misses else 0,
stderr=_stderr(stderr_lines),
writes=writes)
async def tar(
@@ -158,6 +279,7 @@ async def tar(
stat: StatFn,
walk: WalkFn,
is_dir: DirProbe,
selectors: list[str] | None = None,
c: bool = False,
x: bool = False,
t: bool = False,
@@ -166,16 +288,25 @@ async def tar(
J: bool = False,
v: bool = False,
h: bool = False,
to_stdout: bool = False,
f: PathSpec | None = None,
C: list[PathSpec] | None = None,
strip_components: str | None = None,
exclude: str | None = None,
links: LinkView | None = None,
mounts: MountView | None = None,
cwd: PathSpec | str = "/",
relay: bool = False,
) -> tuple[ByteSource | None, IOResult]:
archive = f if f else None
if relay and archive is not None:
# Relay doors address by full virtual path (flat_scopes'
# convention), not by the mount-relative key the wrapper's
# accessor stamped.
archive = PathSpec.from_str_path(archive.virtual)
# Only the last -C is a destination; create checks every one.
dest_path = C[-1].mount_path if C else "/"
dest_path = extract_dest(C[-1] if C else None, cwd, relay)
chosen = list(selectors or [])
mode_suffix = _compression_suffix(z, j, J)
strip_n = int(strip_components) if strip_components else 0
if c:
@@ -199,12 +330,13 @@ async def tar(
if t:
if archive is None:
raise ValueError("tar: -f is required")
return await _list_archive(archive, mode_suffix, read_bytes)
return await _list_archive(archive, mode_suffix, chosen, read_bytes)
if x:
if archive is None:
raise ValueError("tar: -f is required")
return await _extract_archive(archive, dest_path, mode_suffix, strip_n,
v, read_bytes, write_bytes, mkdir_fn)
v, to_stdout, chosen, relay, read_bytes,
write_bytes, mkdir_fn, stat)
raise ValueError("tar: must specify -c, -x, or -t")
@@ -221,6 +353,7 @@ class TarFlags:
xz: bool = False
verbose: bool = False
deref: bool = False
to_stdout: bool = False
archive: PathSpec | None = None
directories: tuple[PathSpec, ...] = ()
strip_components: str | None = None
@@ -239,6 +372,7 @@ def parse_flags(flags: Mapping[str, FlagValue]) -> TarFlags:
xz=fl.as_bool("J"),
verbose=fl.as_bool("v"),
deref=fl.as_bool("h"),
to_stdout=fl.as_bool("to_stdout"),
archive=archive if isinstance(archive, PathSpec) else None,
directories=tuple(fl.as_paths("C")),
strip_components=fl.as_str("strip_components"),
@@ -246,8 +380,16 @@ def parse_flags(flags: Mapping[str, FlagValue]) -> TarFlags:
)
async def tar_generic(paths, texts, opts: CommandOpts, read_bytes, write_bytes,
mkdir_fn, stat, walk, is_dir):
async def tar_generic(paths,
texts,
opts: CommandOpts,
read_bytes,
write_bytes,
mkdir_fn,
stat,
walk,
is_dir,
relay: bool = False):
parsed = parse_flags(opts.flags)
return await tar(paths,
read_bytes=read_bytes,
@@ -256,6 +398,7 @@ async def tar_generic(paths, texts, opts: CommandOpts, read_bytes, write_bytes,
stat=stat,
walk=walk,
is_dir=is_dir,
selectors=list(texts),
c=parsed.create,
x=parsed.extract,
t=parsed.list_only,
@@ -264,9 +407,12 @@ async def tar_generic(paths, texts, opts: CommandOpts, read_bytes, write_bytes,
J=parsed.xz,
v=parsed.verbose,
h=parsed.deref,
to_stdout=parsed.to_stdout,
f=parsed.archive,
C=list(parsed.directories) or None,
strip_components=parsed.strip_components,
exclude=parsed.exclude,
links=opts.ns.links if opts.ns is not None else None,
mounts=opts.ns.mounts if opts.ns is not None else None)
mounts=opts.ns.mounts if opts.ns is not None else None,
cwd=opts.cwd,
relay=relay)
python/mirage/commands/builtin/generic/unzip.py
+57 -20
View File
@@ -4,6 +4,9 @@ import zipfile
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from mirage.commands.builtin.generic.archive.extract import (ensure_dir,
extract_dest)
from mirage.commands.builtin.generic.archive.walk import StatFn
from mirage.commands.config import CommandOpts
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagValue, FlagView
@@ -15,16 +18,6 @@ from mirage.utils.key_prefix import mount_prefix_of
CAUTION_PREFIX = "caution: filename not matched: "
def _resolve_dest(d: str | PathSpec | None, mount_prefix: str) -> str:
d_str = d.virtual if isinstance(d, PathSpec) else d
dest_raw = d_str if d_str else "/"
if mount_prefix and dest_raw.startswith(mount_prefix + "/"):
return dest_raw[len(mount_prefix):]
if dest_raw == mount_prefix:
return "/"
return dest_raw
def _spec_index(name: bytes, members: tuple[bytes, ...]) -> int | None:
for i, member in enumerate(members):
if fnmatch.fnmatchcase(name, member):
@@ -60,12 +53,34 @@ def _cautions(unmatched: list[str]) -> str:
return "".join(CAUTION_PREFIX + member + "\n" for member in unmatched)
async def _make_dirs(dir_path: str, mkdir_fn: Callable[..., Awaitable[None]],
stat: StatFn | None, made: set[str]) -> None:
"""Create the chain for one entry, per door space.
With a stat door the shared single-level walk runs (dispatch mkdir
is single-level on most backends); without one the accessor's own
mkdir handles the chain, which is the pre-workspace construction
path where no dispatcher exists.
Args:
dir_path (str): the directory whose chain must exist.
mkdir_fn (Callable): mkdir door.
stat (StatFn | None): stat door in the same path space, if any.
made (set[str]): levels already ensured this run.
"""
if stat is None:
await mkdir_fn(PathSpec.from_str_path(dir_path), parents=True)
return
await ensure_dir(dir_path, mkdir_fn, stat, made)
async def unzip(
paths: list[PathSpec],
*,
read_bytes: Callable[..., Awaitable[bytes]],
write_bytes: Callable[..., Awaitable[None]],
mkdir_fn: Callable[..., Awaitable[None]],
stat: StatFn | None = None,
members: tuple[str, ...] = (),
o: bool = False,
args_l: bool = False,
@@ -73,10 +88,17 @@ async def unzip(
q: bool = False,
p: bool = False,
t: bool = False,
cwd: PathSpec | str = "/",
relay: bool = False,
) -> tuple[ByteSource | None, IOResult]:
if not paths:
raise ValueError("unzip: missing operand")
archive_path = paths[0]
if relay:
# Relay doors address by full virtual path (flat_scopes'
# convention), not by the mount-relative key the wrapper's
# accessor stamped.
archive_path = PathSpec.from_str_path(archive_path.virtual)
data = await read_bytes(archive_path)
with zipfile.ZipFile(io.BytesIO(data), "r") as zf:
selected, unmatched = _select(zf.infolist(), members)
@@ -128,28 +150,34 @@ async def unzip(
mount_prefix = mount_prefix_of(
archive_path.virtual, archive_path.resource_path) if isinstance(
archive_path, PathSpec) else ""
dest = _resolve_dest(d, mount_prefix)
dest = extract_dest(d, cwd, relay)
writes: dict[str, ByteSource] = {}
made: set[str] = set()
output_lines: list[str] = []
for info in selected:
entry_name = info.filename.lstrip("/")
out_path = dest.rstrip("/") + "/" + entry_name.rstrip("/")
report_path = (mount_prefix +
out_path) if mount_prefix else out_path
report_path = out_path if relay else ((
mount_prefix + out_path) if mount_prefix else out_path)
if info.is_dir():
# A directory entry is the only record an empty
# directory leaves, so it has to be recreated even
# though nothing is written inside it.
await mkdir_fn(PathSpec.from_str_path(out_path), parents=True)
await _make_dirs(out_path, mkdir_fn, stat, made)
if not q:
output_lines.append(f" creating: {report_path}/")
continue
content = zf.read(info)
parent = out_path.rsplit("/", 1)[0] or "/"
if parent != "/":
await mkdir_fn(PathSpec.from_str_path(parent), parents=True)
await write_bytes(PathSpec.from_str_path(out_path), content)
writes[out_path] = content
await _make_dirs(parent, mkdir_fn, stat, made)
await write_bytes(PathSpec.from_str_path(out_path), data=content)
if not relay:
# Relay writes land on whichever mount owns each path
# and invalidate through the dispatcher; keying them
# here would have the runner prefix them onto this
# mount.
writes[out_path] = content
if not q:
output_lines.append(f" inflating: {report_path}")
output = ("\n".join(output_lines) +
@@ -187,17 +215,26 @@ def parse_flags(flags: Mapping[str, FlagValue]) -> UnzipFlags:
)
async def unzip_generic(paths, texts, opts: CommandOpts, read_bytes,
write_bytes, mkdir_fn):
async def unzip_generic(paths,
texts,
opts: CommandOpts,
read_bytes,
write_bytes,
mkdir_fn,
stat: StatFn | None = None,
relay: bool = False):
parsed = parse_flags(opts.flags)
return await unzip(paths,
read_bytes=read_bytes,
write_bytes=write_bytes,
mkdir_fn=mkdir_fn,
stat=stat,
members=tuple(texts),
o=parsed.overwrite,
args_l=parsed.list_only,
d=parsed.dest,
q=parsed.quiet,
p=parsed.to_stdout,
t=parsed.test_only)
t=parsed.test_only,
cwd=opts.cwd,
relay=relay)
python/mirage/commands/builtin/generic_bind/builders/tar.py
+21
View File
@@ -15,11 +15,15 @@
from functools import partial
from mirage.accessor.base import Accessor
from mirage.commands.builtin.generic.crossmount.utils import \
transfer_primitives
from mirage.commands.builtin.generic.tar import tar_generic
from mirage.commands.builtin.generic_bind.adapter import (Builder, CommandIO,
Operation, bound_op)
from mirage.commands.builtin.generic_bind.archive_io import is_dir_of, walk_of
from mirage.commands.config import CommandOpts
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagView
from mirage.io.types import ByteSource, IOResult
from mirage.types import PathSpec
@@ -30,6 +34,23 @@ async def tar(ops: CommandIO, accessor: Accessor, paths: list[PathSpec],
if not ops.is_mounted(accessor):
raise ValueError("tar: missing operand")
resolved = await ops.resolve_glob(accessor, paths, opts.index)
fl = FlagView(opts.flags, spec=SPECS["tar"])
if opts.dispatch is not None and not fl.as_bool("c"):
# -t reads and -x writes wherever cwd or -C says, which needs
# not be this mount, so both run on dispatch-relayed doors and
# each path routes to the mount that owns it. Only -c stays on
# the accessor: its planner walks this mount's tree.
prim = transfer_primitives(opts.dispatch)
return await tar_generic(resolved,
list(texts),
opts,
prim["read_bytes"],
prim["write"],
prim["mkdir"],
prim["stat"],
walk_of(ops, accessor, opts.index),
is_dir_of(ops, accessor, opts.index),
relay=True)
return await tar_generic(resolved, list(texts), opts,
bound_op(ops.read_bytes, accessor, opts.index),
partial(ops.require(Operation.WRITE), accessor),
python/mirage/commands/builtin/generic_bind/builders/unzip.py
+15
View File
@@ -15,6 +15,8 @@
from functools import partial
from mirage.accessor.base import Accessor
from mirage.commands.builtin.generic.crossmount.utils import \
transfer_primitives
from mirage.commands.builtin.generic.unzip import unzip_generic
from mirage.commands.builtin.generic_bind.adapter import (Builder, CommandIO,
Operation, bound_op)
@@ -29,6 +31,19 @@ async def unzip(ops: CommandIO, accessor: Accessor, paths: list[PathSpec],
if not ops.is_mounted(accessor) or not paths:
raise ValueError("unzip: missing operand")
resolved = await ops.resolve_glob(accessor, paths, opts.index)
if opts.dispatch is not None:
# Extraction writes wherever cwd or -d says, which need not be
# this mount, so the doors are dispatch-relayed and each path
# routes to the mount that owns it.
prim = transfer_primitives(opts.dispatch)
return await unzip_generic(resolved,
list(texts),
opts,
prim["read_bytes"],
prim["write"],
prim["mkdir"],
stat=prim["stat"],
relay=True)
return await unzip_generic(resolved, list(texts), opts,
bound_op(ops.read_bytes, accessor, opts.index),
partial(ops.require(Operation.WRITE), accessor),
python/mirage/commands/builtin/grep_helper.py
+71 -2
View File
@@ -12,9 +12,11 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import fnmatch
import re
from collections.abc import (AsyncIterator, Awaitable, Callable, Mapping,
Sequence)
from dataclasses import dataclass
from mirage.commands.builtin.constants import PatternType
from mirage.commands.builtin.grep_context import grep_context_lines
@@ -44,6 +46,59 @@ BINARY_EXTENSIONS = frozenset({
NEVER_MATCH = r"(?!)"
@dataclass(frozen=True, slots=True)
class WalkFilters:
"""GNU's file-selection flags, threaded as one value.
``include``/``exclude`` gate files, ``exclude_dir`` prunes
directories from the -r walk, and ``text`` (-a) lets the walk read
the extensions it would otherwise skip as binary. The empty value
admits everything, which is what every caller without the flags
passes.
"""
include: tuple[str, ...] = ()
exclude: tuple[str, ...] = ()
exclude_dir: tuple[str, ...] = ()
text: bool = False
NO_FILTERS = WalkFilters()
def file_admitted(path: str, filters: WalkFilters) -> bool:
"""GNU's --include/--exclude gate for one candidate file.
Globs match the base name with fnmatch wildcards, case sensitively
(a glob carrying a slash therefore matches nothing, which is what
GNU 3.11 answers too), and --exclude wins when both match. Applies
to command-line files exactly as to walked ones, which is pinned
GNU behavior: an explicit operand --include passes over is silently
no match, not an error.
Args:
path (str): candidate file path, any path space.
filters (WalkFilters): the parsed selection flags.
"""
base = path.rstrip("/").rsplit("/", 1)[-1]
if any(fnmatch.fnmatchcase(base, glob) for glob in filters.exclude):
return False
if filters.include:
return any(fnmatch.fnmatchcase(base, glob) for glob in filters.include)
return True
def dir_admitted(path: str, filters: WalkFilters) -> bool:
"""Whether the -r walk may descend into this directory.
Args:
path (str): candidate directory path, any path space.
filters (WalkFilters): the parsed selection flags.
"""
base = path.rstrip("/").rsplit("/", 1)[-1]
return not any(
fnmatch.fnmatchcase(base, glob) for glob in filters.exclude_dir)
def classify_pattern(
pattern: str,
fixed_string: bool,
@@ -164,9 +219,10 @@ def search_query(pattern: str, fixed_string: bool) -> str | None:
_PUSHDOWN_SHAPING_BOOL = ("v", "n", "c", "args_l", "w", "o", "q", "H", "h",
"args_I")
"args_I", "text")
_PUSHDOWN_SHAPING_INT = ("m", "A", "B", "C")
_PUSHDOWN_FILTER_STR = ("type", "glob")
_PUSHDOWN_FILTER_LIST = ("include", "exclude", "exclude_dir")
def has_search_shaping_flags(flags: Mapping[str, FlagValue] | None) -> bool:
@@ -187,6 +243,8 @@ def has_search_shaping_flags(flags: Mapping[str, FlagValue] | None) -> bool:
return True
if any(fl.as_int(k) is not None for k in _PUSHDOWN_SHAPING_INT):
return True
if any(fl.as_list(k) for k in _PUSHDOWN_FILTER_LIST):
return True
return any(fl.as_str(k) is not None for k in _PUSHDOWN_FILTER_STR)
@@ -584,6 +642,7 @@ async def grep_recursive(
max_count: int | None,
warnings: list[str] | None = None,
read_stream_fn=None,
filters: WalkFilters = NO_FILTERS,
) -> list[str]:
results: list[str] = []
try:
@@ -600,6 +659,8 @@ async def grep_recursive(
warnings.append(f"grep: {entry}: {exc}")
continue
if s.type == FileType.DIRECTORY:
if not dir_admitted(entry, filters):
continue
results.extend(await grep_recursive(
readdir_fn,
stat_fn,
@@ -614,9 +675,12 @@ async def grep_recursive(
max_count,
warnings,
read_stream_fn,
filters,
))
continue
if get_extension(entry) in BINARY_EXTENSIONS:
if not file_admitted(entry, filters):
continue
if not filters.text and get_extension(entry) in BINARY_EXTENSIONS:
continue
if read_stream_fn is not None:
try:
@@ -769,6 +833,7 @@ async def grep_files_only(
basic: bool,
warnings: list[str] | None,
read_stream_fn=None,
filters: WalkFilters = NO_FILTERS,
) -> list[str]:
compiled = compile_pattern(pattern, ignore_case, fixed_string, whole_word,
basic)
@@ -804,6 +869,7 @@ async def grep_files_only(
max_count,
warnings,
read_stream_fn,
filters,
)
# GNU names a directory operand and moves on without descending into
@@ -814,6 +880,9 @@ async def grep_files_only(
warnings.append(f"grep: {path}: Is a directory")
return []
if not file_admitted(path, filters):
return []
try:
data = await read_bytes_fn(path)
except WALK_ERRORS as exc:
python/mirage/commands/spec/builtin_specs/archive.py
+7 -1
View File
@@ -27,6 +27,7 @@ SPECS: dict[str, CommandSpec] = {
Option(short="-v"),
# -h archives what a symlink points at instead of the link.
Option(short="-h"),
Option(short="-O", long="--to-stdout"),
Option(short="-f", type="path"),
# Every occurrence is kept, in order: GNU chdirs at each
# one and fails at the first it cannot enter, so the
@@ -35,7 +36,12 @@ SPECS: dict[str, CommandSpec] = {
Option(long="--strip-components", type="str"),
Option(long="--exclude", type="str"),
),
rest=Operand(type="path"),
# Only -c reads the rest operands from the filesystem. Under -t
# and -x each one is a member selector matched against names
# inside the archive (the same mode split MountRootPolicy keys
# on), so those lines keep them textual, exactly as unzip's
# member patterns are.
rest=Operand(type="path", text_when=("-x", "-t")),
# `tar xzf a.tgz` is the spelling everyone types.
old_option_style=True,
# -C is a chdir for the operands after it, not a flag the command
python/mirage/commands/spec/builtin_specs/search.py
+7
View File
@@ -42,6 +42,13 @@ SPECS: dict[str, CommandSpec] = {
Option(short="-C", type="str"),
Option(short="-e", type="str", multiple=True),
Option(short="-f", type="path", multiple=True),
# -a searches the extensions the -r walk skips as binary;
# explicit operands are always read as text, which is the
# documented divergence (no "binary file matches" rows).
Option(short="-a", long="--text"),
Option(long="--include", type="str", multiple=True),
Option(long="--exclude", type="str", multiple=True),
Option(long="--exclude-dir", type="str", multiple=True),
# Accepted no-ops: output is never a tty, so plain output is
# exactly what GNU produces with --color=auto (#471).
Option(long="--color", type="str", value_optional=True),
python/mirage/policy/builtin/mount_root.py
+1 -32
View File
@@ -14,6 +14,7 @@
from collections.abc import Sequence
from mirage.commands.builtin.generic.tar.mode import is_create_mode
from mirage.policy.base import Policy
from mirage.policy.types import Action, CommandContext, Deny, MountRootQuery
from mirage.types import PathSpec
@@ -37,38 +38,6 @@ def has_symlink_flag(argv: tuple[str, ...]) -> bool:
return False
def is_create_mode(argv: tuple[str, ...]) -> bool:
"""Whether a tar line reads the filesystem rather than an archive.
Only ``-c`` makes tar's operands source paths. Under ``-t`` and
``-x`` they are member selectors matched against names inside the
archive, so a selector that happens to spell a mount root is not a
mount at all and refusing it would deny an ordinary listing.
Scanned raw for the same reason :func:`has_symlink_flag` is: the
policy fires before flag parsing. Only the first word may be GNU's
dashless option cluster (``tar cf a.tar d``), so a later bare word
is an operand and cannot turn the mode on.
Args:
argv (tuple[str, ...]): raw argv after the command name.
"""
for i, tok in enumerate(argv):
if not isinstance(tok, str):
continue
if tok == "--create":
return True
if tok.startswith("--"):
continue
if tok.startswith("-"):
if "c" in tok[1:]:
return True
continue
if i == 0 and "c" in tok:
return True
return False
def has_parents_flag(argv: tuple[str, ...]) -> bool:
"""Spot mkdir's -p/--parents by raw token scan.
python/mirage/workspace/executor/builtins/__init__.py
+2
View File
@@ -34,6 +34,7 @@ from mirage.workspace.executor.builtins.metadata import (handle_chgrp,
from mirage.workspace.executor.builtins.scope import _scope_path, _to_scope
from mirage.workspace.executor.builtins.script import (handle_bash,
handle_eval,
handle_exec_path,
handle_sleep,
handle_source)
from mirage.workspace.executor.builtins.text import (_interpret_escapes,
@@ -86,6 +87,7 @@ __all__ = [
'handle_set',
'handle_shift',
'handle_sleep',
'handle_exec_path',
'handle_source',
'handle_test',
'handle_timeout',
python/mirage/workspace/executor/builtins/script.py
+74
View File
@@ -15,6 +15,7 @@
import asyncio
import math
import re
import shlex
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
@@ -376,6 +377,79 @@ async def handle_bash(
return io.stdout, io, ExecutionNode(command=label, exit_code=io.exit_code)
def _shebang_words(script: str) -> list[str]:
"""The interpreter words a script's first line names, env resolved.
Args:
script (str): the script text.
"""
first = script.split("\n", 1)[0]
if not first.startswith("#!"):
return []
words = first[2:].strip().split()
if words and words[0].rsplit("/", 1)[-1] == "env":
words = words[1:]
return words
if words:
words[0] = words[0].rsplit("/", 1)[-1]
return words
async def handle_exec_path(
dispatch: DispatchFn,
execute_fn: Callable[..., Any],
path: str,
args: list[str],
session: Session,
stdin: ByteSource | None = None,
) -> tuple[ByteSource | None, IOResult, ExecutionNode]:
"""Run a slash-carrying head word as a program, bash's loader rule.
bash hands a word containing a slash straight to the loader: no
builtin, function, or install can claim it, and the file either
runs or the shell reports why not. Two deliberate divergences from
bash, both consequences of the VFS: there is no exec bit to check
(``chmod`` is stored, not enforced; mount mode does real access
control), so an existing file runs without ``+x``; and the shell
prefix bash puts on the diagnostic is dropped, matching every other
mirage diagnostic.
A shebang naming sh or bash (directly or via env) runs through the
nested-shell machinery, as does a script with none. Any other
interpreter word is re-dispatched as a command line, so
``#!/usr/bin/env python3`` reaches the python3 command wherever the
workspace routes it, and an interpreter nobody registers answers
with its own "command not found".
Args:
dispatch (DispatchFn): op dispatcher, used to read the file.
execute_fn (Callable): runs a program line in this session.
path (str): the head word, as typed.
args (list[str]): the words after it, positional for the script.
session (Session): shell session state.
stdin (ByteSource | None): input stream for the script.
"""
try:
script = await read_script_text(dispatch, path, session.cwd)
except FS_ERRORS as exc:
strerror = fs_strerror(exc)
if strerror is None:
raise
code = 127 if isinstance(exc, FileNotFoundError) else 126
return script_error(path, strerror, code)
words = _shebang_words(script)
interp = words[0] if words else "sh"
if interp in ("sh", "bash"):
return await handle_bash(dispatch, execute_fn,
[*words[1:], path, *args], session, stdin,
interp)
line = shlex.join([*words, path, *args])
io = await execute_fn(line, session_id=session.session_id, stdin=stdin)
return io.stdout, io, ExecutionNode(command=f"{path} " +
" ".join(args) if args else path,
exit_code=io.exit_code)
# Finite non-negative decimals only ("0", "0.2", ".5", "1.", "+1", "1e-3").
# GNU sleep additionally accepts "inf" and sleeps forever; an agent shell
# must never hang, so non-finite intervals are rejected (deliberate
python/mirage/workspace/executor/command/command.py
+8 -2
View File
@@ -20,6 +20,7 @@ from mirage.commands.builtin.generic.crossmount import (handle_cross_mount,
is_cross_mount)
from mirage.commands.builtin.generic.crossmount.detect import strategy_for
from mirage.commands.builtin.generic.crossmount.types import Strategy
from mirage.commands.builtin.generic.tar.mode import is_create_mode
from mirage.commands.builtin.utils.limit import maybe_with_timeout
from mirage.commands.config import version_request
from mirage.commands.spec import SPECS
@@ -190,8 +191,13 @@ async def handle_command(
stderr=msg.encode())
# Path-valued flags count: `cp -t /other/mount/dir src` spans mounts
# exactly like a positional destination would.
if is_cross_mount(cmd_name, routing_scopes, registry):
# exactly like a positional destination would. A create-mode tar is
# the one relay member kept out: its planner walks a single
# backend's tree, so a span there falls through to the refusal
# below instead of a relay run that would cross nested mounts.
if is_cross_mount(
cmd_name, routing_scopes,
registry) and not (cmd_name == "tar" and is_create_mode(raw_argv)):
# Cross-mount execution bypasses a resource command handler. Parse
# against the shared spec so flags and text operands do not depend on
# the source mount. The bound single-mount runner lets the strategy
python/mirage/workspace/node/command_dispatch.py
+17 -7
View File
@@ -47,13 +47,14 @@ from mirage.shell.helpers import ( # isort: skip
get_process_sub_direction, get_text, split_env_prefix)
from mirage.workspace.executor.builtins import ( # isort: skip
accepts_line, follow_paths, handle_bash, handle_cd, handle_chgrp,
handle_chmod, handle_chown, handle_command_builtin, handle_df, handle_echo,
handle_env, handle_eval, handle_exit, handle_export, handle_getopts,
handle_history, handle_ln, handle_local, handle_man, handle_printenv,
handle_printf, handle_read, handle_readlink, handle_return, handle_set,
handle_shift, handle_sleep, handle_source, handle_test, handle_timeout,
handle_touch, handle_trap, handle_type, handle_unset, handle_which,
handle_whoami, handle_xargs, link_flags, prepare_mv, strip_link_operands)
handle_exec_path, handle_chmod, handle_chown, handle_command_builtin,
handle_df, handle_echo, handle_env, handle_eval, handle_exit,
handle_export, handle_getopts, handle_history, handle_ln, handle_local,
handle_man, handle_printenv, handle_printf, handle_read, handle_readlink,
handle_return, handle_set, handle_shift, handle_sleep, handle_source,
handle_test, handle_timeout, handle_touch, handle_trap, handle_type,
handle_unset, handle_which, handle_whoami, handle_xargs, link_flags,
prepare_mv, strip_link_operands)
_CdArgs = list[str | PathSpec]
@@ -363,6 +364,15 @@ async def _run_argv(
exit_code=deny.exit_code,
stderr=err)
# ── path execution ─────────────────────────
# bash hands a slash-carrying head word to the loader, never to
# command lookup: no builtin, function, or CLI can claim it. After
# the admission gate so a policy sees the line like any other.
if name and "/" in name:
return await handle_exec_path(dispatch, execute_fn, name,
[word_text(a) for a in args], session,
stdin)
# ── unsupported bash builtins ──────────────
# Constructs the parser accepts but the executor cannot honor.
# Returning a clear error lets LLMs detect a capability gap instead
python/tests/commands/builtin/generic/archive/test_extract.py
+83
View File
@@ -0,0 +1,83 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
from mirage.commands.builtin.generic.archive.extract import (dir_exists,
ensure_dir,
extract_dest)
from mirage.types import FileStat, FileType, PathSpec
def test_extract_dest_prefers_the_explicit_operand():
explicit = PathSpec.from_str_path("/work/out")
cwd = PathSpec.from_str_path("/elsewhere")
assert extract_dest(explicit, cwd, True) == "/work/out"
def test_extract_dest_falls_back_to_cwd_per_space():
cwd = PathSpec(virtual="/work/sub",
directory="/work/sub",
resource_path="sub")
assert extract_dest(None, cwd, True) == "/work/sub"
assert extract_dest(None, cwd, False) == "/sub"
def test_extract_dest_string_cwd_defaults_to_root():
assert extract_dest(None, "", False) == "/"
assert extract_dest(None, "/w", True) == "/w"
def _stat_factory(dirs: set[str]):
async def stat(path: PathSpec) -> FileStat:
if path.virtual in dirs:
return FileStat(path=path.virtual,
name=path.virtual.rsplit("/", 1)[-1],
type=FileType.DIRECTORY)
raise FileNotFoundError(path.virtual)
return stat
def test_ensure_dir_creates_only_missing_levels():
created: list[str] = []
async def mkdir(path: PathSpec) -> None:
created.append(path.virtual)
made: set[str] = set()
asyncio.run(ensure_dir("/work/a/b", mkdir, _stat_factory({"/work"}), made))
assert created == ["/work/a", "/work/a/b"]
assert made == {"/work", "/work/a", "/work/a/b"}
def test_ensure_dir_memoizes_across_members():
calls: list[str] = []
async def mkdir(path: PathSpec) -> None:
calls.append(path.virtual)
made: set[str] = set()
stat = _stat_factory(set())
asyncio.run(ensure_dir("/a/b", mkdir, stat, made))
asyncio.run(ensure_dir("/a/b/c", mkdir, stat, made))
assert calls == ["/a", "/a/b", "/a/b/c"]
def test_dir_exists_answers_false_on_a_miss():
stat = _stat_factory(set())
assert asyncio.run(dir_exists(stat, "/nope")) is False
stat = _stat_factory({"/yes"})
assert asyncio.run(dir_exists(stat, "/yes")) is True
python/tests/commands/builtin/generic/tar/test_mode.py
+34
View File
@@ -0,0 +1,34 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic.tar.mode import is_create_mode
def test_dash_c_and_long_create_turn_the_mode_on():
assert is_create_mode(("-czf", "out.tgz", "dir"))
assert is_create_mode(("--create", "-f", "out.tar", "dir"))
def test_extract_and_list_stay_off():
assert not is_create_mode(("-xzf", "a.tgz", "./m/x.json"))
assert not is_create_mode(("-tzf", "a.tgz"))
def test_only_the_first_word_may_be_a_dashless_cluster():
assert is_create_mode(("cf", "a.tar", "d"))
assert not is_create_mode(("-xf", "a.tar", "crate"))
def test_long_options_never_read_as_clusters():
assert not is_create_mode(("--exclude", "c", "-xf", "a.tar"))
python/tests/commands/builtin/generic/test_unzip.py
+3 -3
View File
@@ -58,11 +58,11 @@ class _Recorder:
def __init__(self) -> None:
self.written: dict[str, bytes] = {}
async def __call__(self, p: PathSpec, content: bytes) -> None:
self.written[p.virtual] = content
async def __call__(self, p: PathSpec, data: bytes) -> None:
self.written[p.virtual] = data
async def _no_write(_p: PathSpec, _content: bytes) -> None:
async def _no_write(_p: PathSpec, data: bytes) -> None:
raise AssertionError("write_bytes must not be called")
python/tests/commands/native/test_grep_filters.py
+74
View File
@@ -0,0 +1,74 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
def _seed(env) -> None:
env.create_file("notes.tex", b"score 9\n")
env.create_file("notes.txt", b"score 8\n")
env.create_file("sub/inner.tex", b"score 7\n")
def test_include_filters_the_recursive_walk(env):
_seed(env)
out = env.mirage("grep -RInE --include='*.tex' score /data")
assert out == ("/data/notes.tex:1:score 9\n"
"/data/sub/inner.tex:1:score 7\n")
def test_exclude_wins_over_include(env):
_seed(env)
result = asyncio.run(
env.ws.execute(
"grep -r --include='*.tex' --exclude='notes.*' score /data"))
assert result.exit_code == 0
assert result.stdout == b"/data/sub/inner.tex:score 7\n"
def test_exclude_dir_prunes_the_walk(env):
_seed(env)
out = env.mirage("grep -r --include='*.tex' --exclude-dir=sub score /data")
assert out == "/data/notes.tex:score 9\n"
def test_include_with_a_slash_matches_nothing(env):
_seed(env)
result = asyncio.run(
env.ws.execute("grep -r --include='sub/*.tex' score /data"))
assert result.exit_code == 1
assert result.stdout in (None, b"", b"\n") or not result.stdout
def test_include_filters_an_explicit_operand_in_silence(env):
_seed(env)
result = asyncio.run(
env.ws.execute("grep --include='*.tex' -n score /data/notes.txt"))
assert result.exit_code == 1
assert not result.stderr
def test_dash_a_reads_binary_extensions_in_the_walk(env):
_seed(env)
env.create_file("data.parquet", b"score binary\n")
without = env.mirage("grep -r score /data")
assert "data.parquet" not in without
with_a = env.mirage("grep -ra score /data")
assert "/data/data.parquet:score binary" in with_a
def test_dash_a_is_accepted_on_an_explicit_operand(env):
_seed(env)
out = env.mirage("grep -aoiE 'SCORE' /data/notes.tex")
assert out == "score\n"
python/tests/commands/native/test_tar_members.py
+101
View File
@@ -0,0 +1,101 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import gzip
import io
import tarfile
def _tgz_bytes() -> bytes:
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w") as tf:
folder = tarfile.TarInfo(name="./memory")
folder.type = tarfile.DIRTYPE
tf.addfile(folder)
for name in ("./memory/memory.json", "./other.txt"):
info = tarfile.TarInfo(name=name)
data = f"content:{name}\n".encode()
info.size = len(data)
tf.addfile(info, io.BytesIO(data))
return gzip.compress(buf.getvalue())
def _seed_archive(env) -> None:
env.mirage("tee /data/files.tar.gz", stdin=_tgz_bytes())
def test_tar_t_selector_lists_only_the_member(env):
_seed_archive(env)
out = env.mirage("tar -tzf /data/files.tar.gz ./memory/memory.json")
assert out == "./memory/memory.json\n"
def test_tar_t_dir_selector_takes_the_subtree(env):
_seed_archive(env)
out = env.mirage("tar -tzf /data/files.tar.gz ./memory")
assert out == "./memory/\n./memory/memory.json\n"
def test_tar_t_selector_matches_stored_spelling_only(env):
_seed_archive(env)
result = asyncio.run(
env.ws.execute("tar -tzf /data/files.tar.gz memory/memory.json"))
assert result.exit_code == 2
stderr = result.stderr.decode()
assert "tar: memory/memory.json: Not found in archive" in stderr
assert "Exiting with failure status due to previous errors" in stderr
def test_tar_xO_streams_member_bytes_to_stdout(env):
_seed_archive(env)
out = env.mirage("tar -xOzf /data/files.tar.gz ./memory/memory.json")
assert out == "content:./memory/memory.json\n"
listing = env.mirage("find /data -name 'memory.json'")
assert listing.strip() == ""
def test_tar_xvO_lists_names_on_stderr(env):
_seed_archive(env)
result = asyncio.run(
env.ws.execute("tar -xvOzf /data/files.tar.gz ./memory/memory.json"))
assert result.exit_code == 0
assert result.stderr.decode() == "./memory/memory.json\n"
def test_tar_x_selector_extracts_only_the_member(env):
_seed_archive(env)
env.mirage("mkdir -p /data/out")
env.mirage("tar -xzf /data/files.tar.gz -C /data/out ./other.txt")
assert env.mirage("cat /data/out/other.txt") == "content:./other.txt\n"
assert env.mirage("find /data/out -name 'memory.json'").strip() == ""
def test_tar_x_mixed_hit_and_miss_extracts_and_exits_2(env):
_seed_archive(env)
env.mirage("mkdir -p /data/out2")
result = asyncio.run(
env.ws.execute(
"tar -xzf /data/files.tar.gz -C /data/out2 ./other.txt nope"))
assert result.exit_code == 2
assert "tar: nope: Not found in archive" in result.stderr.decode()
assert env.mirage("cat /data/out2/other.txt") == "content:./other.txt\n"
def test_tar_x_extracts_into_cwd_with_dot_components_cleaned(env):
_seed_archive(env)
env.mirage("tar -xzf /data/files.tar.gz")
assert env.mirage(
"cat /data/memory/memory.json") == "content:./memory/memory.json\n"
assert "/data/./memory" not in env.mirage("find /data")
python/tests/workspace/executor/builtins/test_exec_path.py
+98
View File
@@ -0,0 +1,98 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import pytest
from mirage.resource import RAMResource
from mirage.types import MountMode
from mirage.workspace import Workspace
from mirage.workspace.executor.builtins.script import _shebang_words
@pytest.fixture()
def ws() -> Workspace:
return Workspace(
resources={
"/": (RAMResource(), MountMode.WRITE),
"/work/": (RAMResource(), MountMode.WRITE),
})
def _run(ws: Workspace, line: str):
return asyncio.run(ws.execute(line))
def test_slash_head_word_runs_the_file(ws):
_run(ws, "printf 'echo ran\\n' > /work/run.sh")
result = _run(ws, "/work/run.sh")
assert result.exit_code == 0
assert result.stdout == b"ran\n"
def test_relative_path_resolves_against_cwd(ws):
_run(ws, "printf 'echo rel\\n' > /work/run.sh")
result = _run(ws, "cd /work && ./run.sh")
assert result.stdout == b"rel\n"
def test_script_gets_dollar_zero_and_positionals(ws):
# A line-final positional hits a pre-existing expansion bug that
# `sh FILE` shows identically, so the line ends on a literal and
# this pins only what path execution adds.
_run(ws, "printf 'echo $0:$1:end\\n' > /work/args.sh")
result = _run(ws, "/work/args.sh a b")
assert result.stdout == b"/work/args.sh:a:end\n"
def test_shebang_interpreter_options_apply(ws):
_run(ws, "printf '#!/bin/bash -x\\necho traced\\n' > /work/t.sh")
result = _run(ws, "/work/t.sh")
assert result.stdout == b"traced\n"
assert result.stderr == b"+ echo traced\n"
def test_missing_file_is_127(ws):
result = _run(ws, "/work/nope.sh")
assert result.exit_code == 127
assert result.stderr == b"/work/nope.sh: No such file or directory\n"
def test_directory_is_126(ws):
_run(ws, "mkdir -p /work/adir")
result = _run(ws, "/work/adir")
assert result.exit_code == 126
assert result.stderr == b"/work/adir: Is a directory\n"
def test_unknown_interpreter_reports_command_not_found(ws):
_run(ws, "printf '#!/usr/bin/env ruby\\nputs 1\\n' > /work/r.rb")
result = _run(ws, "/work/r.rb")
assert result.exit_code == 127
assert result.stderr == b"ruby: command not found\n"
def test_child_shell_state_does_not_leak(ws):
_run(ws, "printf 'cd /work\\nexit 3\\n' > /child.sh")
result = _run(ws, "/child.sh; echo \"$?:$(pwd)\"")
assert result.stdout == b"3:/\n"
def test_shebang_words_resolves_env_and_basenames():
assert _shebang_words("#!/bin/sh\necho hi\n") == ["sh"]
assert _shebang_words("#!/usr/bin/env bash\n") == ["bash"]
assert _shebang_words("#!/bin/bash -x\n") == ["bash", "-x"]
assert _shebang_words("#!/usr/bin/env python3\n") == ["python3"]
assert _shebang_words("echo no shebang\n") == []
python/tests/workspace/executor/test_archive_relay.py
+102
View File
@@ -0,0 +1,102 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import gzip
import io
import tarfile
import zipfile
import pytest
from mirage.resource import RAMResource
from mirage.types import MountMode
from mirage.workspace import Workspace
def _tgz_bytes() -> bytes:
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w") as tf:
for name in ("./memory/memory.json", "./other.txt"):
info = tarfile.TarInfo(name=name)
data = f"content:{name}\n".encode()
info.size = len(data)
tf.addfile(info, io.BytesIO(data))
return gzip.compress(buf.getvalue())
def _zip_bytes() -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("memory/memory.json", "zipped\n")
return buf.getvalue()
@pytest.fixture()
def ws() -> Workspace:
work = RAMResource()
work._store.files["/files.tar.gz"] = _tgz_bytes()
work._store.files["/files.zip"] = _zip_bytes()
return Workspace(resources={
"/": (RAMResource(), MountMode.WRITE),
"/work/": (work, MountMode.WRITE),
})
def _run(ws: Workspace, line: str):
return asyncio.run(ws.execute(line))
def test_tar_selector_does_not_join_routing(ws):
# cwd is /, the archive is on /work: the selector must not count as
# a path operand or the line refuses as a cross-mount span.
result = _run(ws, "tar -xOzf /work/files.tar.gz ./memory/memory.json")
assert result.exit_code == 0
assert result.stdout == b"content:./memory/memory.json\n"
def test_tar_extract_lands_in_cwd_across_mounts(ws):
result = _run(ws, "tar -xzf /work/files.tar.gz")
assert result.exit_code == 0
out = _run(ws, "cat /memory/memory.json")
assert out.stdout == b"content:./memory/memory.json\n"
def test_tar_extract_dash_C_into_another_mount(ws):
result = _run(ws, "tar -xzf /work/files.tar.gz -C /dest")
assert result.exit_code == 0
out = _run(ws, "cat /dest/memory/memory.json")
assert out.stdout == b"content:./memory/memory.json\n"
def test_tar_create_span_keeps_the_refusal(ws):
_run(ws, "mkdir -p /src && echo hi > /src/f.txt")
result = _run(ws, "tar -czf /work/backup.tgz /src")
assert result.exit_code == 1
assert b"paths span multiple mounts" in result.stderr
def test_unzip_extracts_into_cwd(ws):
_run(ws, "cd /")
result = _run(ws, "unzip -q /work/files.zip")
assert result.exit_code == 0
out = _run(ws, "cat /memory/memory.json")
assert out.stdout == b"zipped\n"
def test_unzip_dash_d_into_another_mount(ws):
result = _run(ws, "unzip -q -d /dest /work/files.zip")
assert result.exit_code == 0
out = _run(ws, "cat /dest/memory/memory.json")
assert out.stdout == b"zipped\n"
spec/python/general/grep.json
+20
View File
@@ -352,6 +352,26 @@
"short": "-f",
"type": "path"
},
{
"long": "--text",
"short": "-a",
"type": "bool"
},
{
"long": "--include",
"multiple": true,
"type": "str"
},
{
"long": "--exclude",
"multiple": true,
"type": "str"
},
{
"long": "--exclude-dir",
"multiple": true,
"type": "str"
},
{
"long": "--color",
"type": "str",
spec/python/general/tar.json
+9
View File
@@ -142,6 +142,11 @@
"short": "-h",
"type": "bool"
},
{
"long": "--to-stdout",
"short": "-O",
"type": "bool"
},
{
"short": "-f",
"type": "path"
@@ -161,6 +166,10 @@
}
],
"rest": {
"text_when": [
"-x",
"-t"
],
"type": "path"
}
}
spec/typescript/browser/general/grep.json
+20
View File
@@ -310,6 +310,26 @@
"short": "-f",
"type": "path"
},
{
"long": "--text",
"short": "-a",
"type": "bool"
},
{
"long": "--include",
"multiple": true,
"type": "str"
},
{
"long": "--exclude",
"multiple": true,
"type": "str"
},
{
"long": "--exclude-dir",
"multiple": true,
"type": "str"
},
{
"long": "--color",
"type": "str",
spec/typescript/browser/general/tar.json
+9
View File
@@ -107,6 +107,11 @@
"short": "-h",
"type": "bool"
},
{
"long": "--to-stdout",
"short": "-O",
"type": "bool"
},
{
"short": "-f",
"type": "path"
@@ -126,6 +131,10 @@
}
],
"rest": {
"text_when": [
"-x",
"-t"
],
"type": "path"
}
}
spec/typescript/node/general/grep.json
+20
View File
@@ -373,6 +373,26 @@
"short": "-f",
"type": "path"
},
{
"long": "--text",
"short": "-a",
"type": "bool"
},
{
"long": "--include",
"multiple": true,
"type": "str"
},
{
"long": "--exclude",
"multiple": true,
"type": "str"
},
{
"long": "--exclude-dir",
"multiple": true,
"type": "str"
},
{
"long": "--color",
"type": "str",
spec/typescript/node/general/tar.json
+9
View File
@@ -163,6 +163,11 @@
"short": "-h",
"type": "bool"
},
{
"long": "--to-stdout",
"short": "-O",
"type": "bool"
},
{
"short": "-f",
"type": "path"
@@ -182,6 +187,10 @@
}
],
"rest": {
"text_when": [
"-x",
"-t"
],
"type": "path"
}
}
typescript/packages/core/src/commands/builtin/generic/archive/extract.ts
+77
View File
@@ -0,0 +1,77 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { type FileStat, FileType, PathSpec } from '../../../../types.ts'
import { mountKey } from '../../../../utils/key_prefix.ts'
export type StatDoor = (p: PathSpec) => Promise<FileStat>
export type MkdirDoor = (p: PathSpec, parents?: boolean) => Promise<void>
/**
* Where extraction lands: the explicit operand, else the cwd.
*
* The explicit operand is tar's last -C or unzip's -d; both arrive as
* resolved virtual-path strings in the TypeScript flag bag, and the cwd
* is virtual too, so no per-door-space split exists here the way it
* does in Python (whose accessor doors speak mount-relative paths).
*/
export function extractDest(explicit: string | null, cwd: string): string {
const target = explicit ?? cwd
return target !== '' ? target : '/'
}
export function flatPathSpec(virtual: string): PathSpec {
return new PathSpec({
virtual,
directory: virtual,
resourcePath: mountKey(virtual, ''),
resolved: true,
})
}
async function dirExists(stat: StatDoor, level: PathSpec): Promise<boolean> {
try {
return (await stat(level)).type === FileType.DIRECTORY
} catch {
// A stat miss here only means "not standing yet"; mkdir answers.
return false
}
}
/**
* Create one directory chain top-down, skipping what exists.
*
* The dispatch mkdir op is single-level on most backends (the
* `mkdirParents` knob is a per-backend exception), so extraction walks
* the chain itself the way relay cp does: probe, then create only what
* is missing, memoized per run so an archive of many files stats each
* ancestor once.
*/
export async function ensureDir(
dirPath: string,
toSpec: (virtual: string) => PathSpec,
mkdir: MkdirDoor,
stat: StatDoor,
made: Set<string>,
): Promise<void> {
const parts = dirPath.split('/').filter((part) => part !== '')
for (let i = 1; i <= parts.length; i += 1) {
const level = `/${parts.slice(0, i).join('/')}`
if (made.has(level)) continue
if (!(await dirExists(stat, toSpec(level)))) {
await mkdir(toSpec(level))
}
made.add(level)
}
}
typescript/packages/core/src/commands/builtin/generic/crossmount/constants.ts
+2
View File
@@ -57,6 +57,8 @@ export const RELAY_COMMANDS: ReadonlySet<string> = new Set([
Cmd.PASTE,
Cmd.COMM,
Cmd.JOIN,
Cmd.TAR,
Cmd.UNZIP,
])
export const CROSS_MOUNT_COMMANDS: ReadonlySet<string> = new Set([
...STREAM_COMMANDS,
typescript/packages/core/src/commands/builtin/generic/crossmount/relay/relay.ts
+5
View File
@@ -20,6 +20,8 @@ import { runDiff } from './diff.ts'
import { runJoin } from './join.ts'
import { runMv } from './mv.ts'
import { runPaste } from './paste.ts'
import { runTar } from './tar.ts'
import { runUnzip } from './unzip.ts'
import { Cmd, type CrossResult, type DispatchFn } from '../types.ts'
import type { FlagValue } from '../../../../spec/types.ts'
@@ -30,6 +32,7 @@ import type { FlagValue } from '../../../../spec/types.ts'
export async function runRelay(
cmdName: Cmd,
scopes: PathSpec[],
textArgs: string[],
flagKwargs: Record<string, FlagValue>,
dispatch: DispatchFn,
// Maps an operand to its storage identity, for the transfer commands
@@ -43,5 +46,7 @@ export async function runRelay(
if (cmdName === Cmd.PASTE) return runPaste(scopes, flagKwargs, dispatch)
if (cmdName === Cmd.COMM) return runComm(scopes, flagKwargs, dispatch)
if (cmdName === Cmd.JOIN) return runJoin(scopes, flagKwargs, dispatch)
if (cmdName === Cmd.TAR) return runTar(textArgs, flagKwargs, dispatch)
if (cmdName === Cmd.UNZIP) return runUnzip(scopes, textArgs, flagKwargs, dispatch)
return runCmp(scopes, flagKwargs, dispatch)
}
typescript/packages/core/src/commands/builtin/generic/crossmount/relay/tar.ts
+55
View File
@@ -0,0 +1,55 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { IOResult } from '../../../../../io/types.ts'
import { tarGeneric } from '../../tar.ts'
import { crossOpts, statOp, streamOp } from '../utils.ts'
import type { CrossResult, DispatchFn } from '../types.ts'
import type { FlagValue } from '../../../../spec/types.ts'
/**
* Run a -t/-x tar whose archive and -C destination span mounts.
*
* Pure wiring: the shared generic runs on dispatch-relayed doors, so
* the archive is read from its mount and every extracted path lands on
* whichever mount owns it. Create mode never reaches here: the executor
* keeps a create-mode span on the plain refusal, because its planner
* walks one backend's tree and relay doors would cross nested mount
* boundaries the planner is required to refuse.
*/
export async function runTar(
textArgs: string[],
flagKwargs: Record<string, FlagValue>,
dispatch: DispatchFn,
): Promise<CrossResult> {
const result = await tarGeneric(
[],
textArgs,
crossOpts(flagKwargs),
{
stream: streamOp(dispatch),
write: async (p, data) => {
await dispatch('write', p, [data])
},
mkdir: async (p) => {
await dispatch('mkdir', p)
},
stat: statOp(dispatch),
walk: () => Promise.resolve([]),
isDir: () => Promise.resolve(false),
},
true,
)
return result ?? [null, new IOResult()]
}
typescript/packages/core/src/commands/builtin/generic/crossmount/relay/unzip.ts
+57
View File
@@ -0,0 +1,57 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { specOf } from '../../../../spec/builtins.ts'
import { FlagView } from '../../../../spec/types.ts'
import type { PathSpec } from '../../../../../types.ts'
import { IOResult } from '../../../../../io/types.ts'
import { unzipGeneric } from '../../unzip.ts'
import { crossOpts, statOp, streamOp } from '../utils.ts'
import type { CrossResult, DispatchFn } from '../types.ts'
import type { FlagValue } from '../../../../spec/types.ts'
/**
* Run an unzip whose archive and -d destination span mounts.
*
* Pure wiring: the shared generic runs on dispatch-relayed doors, so
* the archive is read from its mount and every extracted path lands on
* whichever mount owns it.
*/
export async function runUnzip(
scopes: PathSpec[],
textArgs: string[],
flagKwargs: Record<string, FlagValue>,
dispatch: DispatchFn,
): Promise<CrossResult> {
const fl = new FlagView(flagKwargs, specOf('unzip'))
// Scopes arrive in line order and include the -d flag's value, so the
// archive is the first scope that is not the destination.
const dest = fl.asStr('d')
const operands = scopes.filter((s) => s.virtual !== dest)
const result = await unzipGeneric(
operands.length > 0 ? operands : scopes,
textArgs,
crossOpts(flagKwargs),
streamOp(dispatch),
async (p, data) => {
await dispatch('write', p, [data])
},
async (p) => {
await dispatch('mkdir', p)
},
statOp(dispatch),
true,
)
return result ?? [null, new IOResult()]
}
typescript/packages/core/src/commands/builtin/generic/crossmount/route.ts
+1 -1
View File
@@ -47,7 +47,7 @@ export async function handleCrossMount(
const cmd = cmdName as Cmd
const strategy = strategyFor(cmd, flagKwargs)
if (strategy === Strategy.RELAY) {
return await runRelay(cmd, scopes, flagKwargs, dispatch, storageKey)
return await runRelay(cmd, scopes, textArgs, flagKwargs, dispatch, storageKey)
}
if (strategy === Strategy.STREAM) {
return await runStream(cmd, scopes, textArgs, flagKwargs, runSingle)
typescript/packages/core/src/commands/builtin/generic/crossmount/types.ts
+2
View File
@@ -74,6 +74,8 @@ export enum Cmd {
PASTE = 'paste',
COMM = 'comm',
JOIN = 'join',
TAR = 'tar',
UNZIP = 'unzip',
}
export type { DispatchFn } from '../../../../runtime/types.ts'
typescript/packages/core/src/commands/builtin/generic/grep.ts
+18
View File
@@ -27,10 +27,12 @@ import {
countExitStream,
countRecordsHaveMatches,
exitCodeFor,
fileAdmitted,
grepFilesOnly,
type GrepFilesOnlyOptions,
grepLines,
grepRecursive,
type WalkFilters,
grepStream,
prefixLines,
resolvePatternFromFlags,
@@ -45,6 +47,7 @@ type Readdir = (p: PathSpec) => Promise<string[]>
type Stream = (p: PathSpec) => AsyncIterable<Uint8Array>
interface FlagSet {
filters: WalkFilters
ignoreCase: boolean
invert: boolean
lineNumbers: boolean
@@ -67,6 +70,12 @@ function parseFlags(fl: FlagView): FlagSet {
const bCtx = fl.asInt('B')
const cCtx = fl.asInt('C')
return {
filters: {
include: fl.asList('include'),
exclude: fl.asList('exclude'),
excludeDir: fl.asList('exclude_dir'),
text: fl.asBool('text'),
},
ignoreCase: fl.asBool('i'),
invert: fl.asBool('v'),
lineNumbers: fl.asBool('n'),
@@ -104,6 +113,7 @@ function makeSpec(path: string, template: PathSpec): PathSpec {
function filesOnlyOpts(f: FlagSet, recursive: boolean): GrepFilesOnlyOptions {
return {
recursive,
filters: f.filters,
ignoreCase: f.ignoreCase,
invert: f.invert,
lineNumbers: f.lineNumbers,
@@ -222,6 +232,8 @@ export async function grepGeneric(
false,
)
for (const r of respellRaw(res, p.virtual, p.rawPath)) allResults.push(r)
} else if (!fileAdmitted(p.virtual, f.filters)) {
continue
} else {
const data = splitLinesNoTrailing(DEC.decode(await readBytesFn(p.virtual)))
const hits = grepLines(p.rawPath, data, pat, f)
@@ -263,6 +275,7 @@ export async function grepGeneric(
multiWarnings.push(`${name}: ${p.rawPath}: Is a directory`)
continue
}
if (!fileAdmitted(p.virtual, f.filters)) continue
const data = splitLinesNoTrailing(DEC.decode(await materialize(stream(p))))
const hits = grepLines(p.rawPath, data, pat, f)
const label = f.noFilename ? '' : `${p.rawPath}:`
@@ -320,6 +333,11 @@ export async function grepGeneric(
}),
]
}
if (!fileAdmitted(first.virtual, f.filters)) {
// GNU passes over a command-line file --include leaves out in
// silence: no output, no diagnostic, exit "no match".
return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
}
const source = stream(first)
const matched = grepStream(source, pat, f)
if (f.quiet) {
typescript/packages/core/src/commands/builtin/generic/tar.ts
+157 -19
View File
@@ -23,10 +23,77 @@ import { readTar, writeTar, type TarEntry } from '../tar_helper.ts'
import { rstripSlash } from '../../../utils/slash.ts'
import { COMPRESSION_SIGNATURES } from './tar/constants.ts'
import { planCreate, type DirProbe, type StatFn, type WalkFn } from './tar/create.ts'
import { ensureDir, extractDest } from './archive/extract.ts'
import type { Compression, CompressionKind, CreateResult } from './tar/types.ts'
const ENC = new TextEncoder()
export const MISS_TRAILER = 'tar: Exiting with failure status due to previous errors'
const DOTDOT_NOTICE = "tar: Removing leading `../' from member names"
/**
* Whether one -t/-x member selector keeps an archive member.
*
* GNU matches the stored spelling exactly (`memory/x` does not find
* `./memory/x`), and a selector naming a directory takes its whole
* subtree, with or without the trailing slash.
*/
function matchesSelector(name: string, selector: string): boolean {
const base = rstripSlash(selector)
const trimmed = rstripSlash(name)
return trimmed === base || trimmed.startsWith(`${base}/`)
}
/**
* Member indices the selectors keep, and the misses they report.
*
* No selector keeps everything. A selector that matches nothing is
* GNU's per-operand diagnostic, reported in operand order; the caller
* appends the one failure trailer.
*/
function selectedMembers(
names: readonly string[],
selectors: readonly string[],
): { keep: Set<number>; misses: string[] } {
if (selectors.length === 0) {
return { keep: new Set(names.map((_, index) => index)), misses: [] }
}
const keep = new Set<number>()
const misses: string[] = []
for (const selector of selectors) {
let hit = false
for (const [index, name] of names.entries()) {
if (matchesSelector(name, selector)) {
keep.add(index)
hit = true
}
}
if (!hit) misses.push(`tar: ${selector}: Not found in archive`)
}
return { keep, misses }
}
/**
* The destination-relative components one member extracts to.
*
* GNU strips --strip-components off the stored spelling first, in which
* a leading `.` counts as a component (--strip-components=1 turns
* `./a/b` into `a/b`). Only then is the remainder cleaned for the
* filesystem: `.` components vanish (a real OS resolves them; a virtual
* path must not keep a literal `.` directory) and a leading `..` is
* removed with GNU's one notice per run.
*/
function outParts(name: string, stripN: number, notices: string[]): string[] {
let parts = rstripSlash(name).split('/')
if (stripN > 0) parts = parts.slice(stripN)
parts = parts.filter((part) => part !== '' && part !== '.')
while (parts.length > 0 && parts[0] === '..') {
if (!notices.includes(DOTDOT_NOTICE)) notices.push(DOTDOT_NOTICE)
parts.shift()
}
return parts
}
// What tar needs from the mount it runs on. `stat` and `walk` are what
// make a directory operand archivable at all; `isDir` answers on two
// channels so a prefix-store directory (no object of its own) is not
@@ -141,8 +208,10 @@ async function writeArchive(
export async function tarGeneric(
paths: PathSpec[],
texts: readonly string[],
opts: CommandOpts,
deps: TarDeps,
relay = false,
): Promise<CommandFnResult> {
const fl = new FlagView(opts.flags, specOf('tar'))
const create = fl.asBool('c')
@@ -163,9 +232,11 @@ export async function tarGeneric(
const CFlag = CFlags.length > 0 ? (CFlags[CFlags.length - 1] ?? null) : null
const stripN = fl.asInt('strip_components') ?? 0
const exclude = fl.asStr('exclude') ?? null
const mountPrefix = opts.mountPrefix ?? ''
const toStdout = fl.asBool('to_stdout')
const mountPrefix = relay ? '' : (opts.mountPrefix ?? '')
const archivePath = fFlag
const destPath = CFlag ?? '/'
const destPath = extractDest(CFlag, opts.cwd)
const selectors = [...texts]
const verboseLines: string[] = []
if (create) {
@@ -203,9 +274,20 @@ export async function tarGeneric(
const raw = await materialize(deps.stream(makePathSpec(archivePath, mountPrefix)))
const data = await decompress(raw, compression)
const entries = await readTar(data)
const out: ByteSource = ENC.encode(
entries.map((e) => (e.isDir === true ? `${rstripSlash(e.name)}/` : e.name)).join('\n') + '\n',
)
const names = entries.map((e) => (e.isDir === true ? `${rstripSlash(e.name)}/` : e.name))
const { keep, misses } = selectedMembers(names, selectors)
const shown = names.filter((_, index) => keep.has(index))
const out: ByteSource | null = shown.length > 0 ? ENC.encode(shown.join('\n') + '\n') : null
if (misses.length > 0) {
const missStderr = stderrOf([...misses, MISS_TRAILER])
return [
out,
new IOResult({
exitCode: 2,
...(missStderr !== null ? { stderr: missStderr } : {}),
}),
]
}
return [out, new IOResult()]
}
@@ -216,33 +298,89 @@ export async function tarGeneric(
const raw = await materialize(deps.stream(makePathSpec(archivePath, mountPrefix)))
const data = await decompress(raw, compression)
const writes: Record<string, Uint8Array> = {}
for (const entry of await readTar(data)) {
const entries = await readTar(data)
const listed = entries.map((e) => (e.isDir === true ? `${rstripSlash(e.name)}/` : e.name))
const { keep, misses } = selectedMembers(listed, selectors)
const notices: string[] = []
const made = new Set<string>()
const chunks: Uint8Array[] = []
const toSpec = (virtual: string): PathSpec => makePathSpec(virtual, mountPrefix)
for (const [index, entry] of entries.entries()) {
if (!keep.has(index)) continue
// A symlink member has no bytes to write and no namespace to write
// into from here (links are workspace state, not the backend's),
// so extraction skips it rather than dropping an empty file where
// a link belongs.
const isDir = entry.isDir === true
if (!entry.isFile && !isDir) continue
const nameParts = rstripSlash(entry.name).split('/')
const stripped = stripN > 0 ? nameParts.slice(stripN) : nameParts
if (stripped.length === 0 || (stripped.length === 1 && stripped[0] === '')) continue
const outPath = `${rstripSlash(destPath)}/${stripped.join('/')}`
if (isDir) {
// A directory member is the only record an empty directory
// leaves, so it has to be recreated even though nothing is
// written inside it.
await deps.mkdir(makePathSpec(outPath, mountPrefix), true)
if (verbose) verboseLines.push(`${rstripSlash(entry.name)}/`)
if (!toStdout) {
// A directory member is the only record an empty directory
// leaves, so it has to be recreated even though nothing is
// written inside it. Under -O nothing reaches the
// filesystem at all.
const parts = outParts(entry.name, stripN, notices)
if (parts.length > 0) {
const outDir = `${rstripSlash(destPath)}/${parts.join('/')}`
await ensureDir(outDir, toSpec, deps.mkdir, deps.stat, made)
if (verbose) verboseLines.push(`${rstripSlash(entry.name)}/`)
}
}
continue
}
if (toStdout) {
chunks.push(entry.data)
if (verbose) verboseLines.push(entry.name)
continue
}
const parts = outParts(entry.name, stripN, notices)
if (parts.length === 0) continue
const outPath = `${rstripSlash(destPath)}/${parts.join('/')}`
const parent = outPath.slice(0, outPath.lastIndexOf('/')) || '/'
if (parent !== '/') await deps.mkdir(makePathSpec(parent, mountPrefix), true)
if (parent !== '/') await ensureDir(parent, toSpec, deps.mkdir, deps.stat, made)
await deps.write(makePathSpec(outPath, mountPrefix), entry.data)
writes[outPath] = entry.data
// Relay writes land on whichever mount owns each path and
// invalidate through the dispatcher; keying them here would have
// the runner prefix them onto this mount.
if (!relay) writes[outPath] = entry.data
if (verbose) verboseLines.push(entry.name)
}
const stdout = verbose ? ENC.encode(verboseLines.join('\n') + '\n') : null
return [stdout, new IOResult({ writes })]
if (toStdout) {
// GNU moves the verbose listing to stderr when stdout carries the
// member bytes.
const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0)
const merged = new Uint8Array(total)
let offset = 0
for (const chunk of chunks) {
merged.set(chunk, offset)
offset += chunk.byteLength
}
const errLines = [
...notices,
...(verbose ? verboseLines : []),
...(misses.length > 0 ? [...misses, MISS_TRAILER] : []),
]
const stderr = stderrOf(errLines)
return [
merged.byteLength > 0 ? merged : null,
new IOResult({
exitCode: misses.length > 0 ? 2 : 0,
...(stderr !== null ? { stderr } : {}),
}),
]
}
const stdout =
verbose && verboseLines.length > 0 ? ENC.encode(verboseLines.join('\n') + '\n') : null
const errLines = [...notices, ...(misses.length > 0 ? [...misses, MISS_TRAILER] : [])]
const stderr = stderrOf(errLines)
return [
stdout,
new IOResult({
writes,
exitCode: misses.length > 0 ? 2 : 0,
...(stderr !== null ? { stderr } : {}),
}),
]
}
return [
typescript/packages/core/src/commands/builtin/generic/tar/mode.ts
+40
View File
@@ -0,0 +1,40 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
/**
* Whether a tar line reads the filesystem rather than an archive.
*
* Only -c makes tar's operands source paths. Under -t and -x they are
* member selectors matched against names inside the archive, so a
* selector that happens to spell a mount root is not a mount at all and
* refusing it would deny an ordinary listing.
*
* Scanned raw because both callers fire before flag parsing: the
* mount-root policy at admission, and the cross-mount router deciding
* whether a span is real. Only the first word may be GNU's dashless
* option cluster (`tar cf a.tar d`), so a later bare word is an operand
* and cannot turn the mode on.
*/
export function isCreateMode(argv: readonly string[]): boolean {
for (const [i, tok] of argv.entries()) {
if (tok === '--create') return true
if (tok.startsWith('--')) continue
if (tok.startsWith('-')) {
if (tok.slice(1).includes('c')) return true
continue
}
if (i === 0 && tok.includes('c')) return true
}
return false
}
typescript/packages/core/src/commands/builtin/generic/unzip.ts
+28 -6
View File
@@ -15,6 +15,7 @@
import { specOf } from '../../spec/builtins.ts'
import { FlagView } from '../../spec/types.ts'
import { mountPrefixOf } from '../../../utils/key_prefix.ts'
import { ensureDir, extractDest, type StatDoor } from './archive/extract.ts'
import { IOResult, materialize, type ByteSource } from '../../../io/types.ts'
import { PathSpec } from '../../../types.ts'
import { inflateRaw } from '../../../utils/compress.ts'
@@ -179,13 +180,18 @@ export async function unzipGeneric(
stream: (p: PathSpec) => AsyncIterable<Uint8Array>,
write: (p: PathSpec, data: Uint8Array) => Promise<void>,
mkdir: (p: PathSpec, parents?: boolean) => Promise<void>,
stat?: StatDoor,
relay = false,
): Promise<CommandFnResult> {
const fl = new FlagView(opts.flags, specOf('unzip'))
if (paths.length === 0) {
return [null, new IOResult({ exitCode: 1, stderr: ENC.encode('unzip: missing operand\n') })]
}
const archivePath = paths[0]
let archivePath = paths[0]
if (archivePath === undefined) return [null, new IOResult()]
// Relay doors address by full virtual path (flatten's convention),
// not by the mount-relative key the wrapper's accessor stamped.
if (relay) archivePath = makePathSpec(archivePath.virtual)
const data = await materialize(stream(archivePath))
const entries = await readZipEntries(data)
const { selected, unmatched } = selectEntries(entries, members)
@@ -194,8 +200,8 @@ export async function unzipGeneric(
const testMode = fl.asBool('t')
const pipeMode = fl.asBool('p')
const quiet = fl.asBool('q')
const mountPrefix = mountPrefixOf(archivePath.virtual, archivePath.resourcePath)
const destRaw = fl.asStr('d') ?? '/'
const mountPrefix = relay ? '' : mountPrefixOf(archivePath.virtual, archivePath.resourcePath)
const destRaw = extractDest(fl.asStr('d') ?? null, opts.cwd)
const dest =
mountPrefix !== '' && destRaw.startsWith(mountPrefix + '/')
? destRaw.slice(mountPrefix.length)
@@ -252,6 +258,7 @@ export async function unzipGeneric(
const writes: Record<string, Uint8Array> = {}
const outputLines: string[] = []
const made = new Set<string>()
for (const e of selected) {
const entryName = lstripSlash(e.name)
const outPath = rstripSlash(dest) + '/' + rstripSlash(entryName)
@@ -260,13 +267,28 @@ export async function unzipGeneric(
// A directory entry is the only record an empty directory leaves,
// so it has to be recreated even though nothing is written inside
// it.
await mkdir(makePathSpec(outPath), true)
if (stat !== undefined) {
await ensureDir(outPath, makePathSpec, mkdir, stat, made)
} else {
await mkdir(makePathSpec(outPath), true)
}
if (!quiet) outputLines.push(` creating: ${reportPath}/`)
continue
}
await ensureParents(mkdir, outPath)
const parentEnd = outPath.lastIndexOf('/')
const parent = parentEnd > 0 ? outPath.slice(0, parentEnd) : ''
if (parent !== '' && parent !== '/') {
if (stat !== undefined) {
await ensureDir(parent, makePathSpec, mkdir, stat, made)
} else {
await ensureParents(mkdir, outPath)
}
}
await write(makePathSpec(outPath), e.data)
writes[outPath] = e.data
// Relay writes land on whichever mount owns each path and
// invalidate through the dispatcher; keying them here would have
// the runner prefix them onto this mount.
if (!relay) writes[outPath] = e.data
if (!quiet) outputLines.push(` inflating: ${reportPath}`)
}
const stdout: ByteSource | null =
typescript/packages/core/src/commands/builtin/generic_bind/builders/tar.ts
+35 -2
View File
@@ -13,6 +13,9 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { type FileStat, FileType, type PathSpec } from '../../../../types.ts'
import { specOf } from '../../../spec/builtins.ts'
import { FlagView } from '../../../spec/types.ts'
import { readBytesOp, statOp } from '../../generic/crossmount/utils.ts'
import { tarGeneric } from '../../generic/tar.ts'
import { type Builder, resolveGlobOf } from '../adapter.ts'
import { walkOf } from '../archive_io.ts'
@@ -21,7 +24,7 @@ export const TAR_BUILDER: Builder = {
name: 'tar',
write: true,
requirements: ['write', 'mkdir'],
fn: async (ops, accessor, paths, _texts, opts) => {
fn: async (ops, accessor, paths, texts, opts) => {
const idx = opts.index ?? undefined
const { write, mkdir } = ops
if (write === undefined || mkdir === undefined) {
@@ -29,7 +32,37 @@ export const TAR_BUILDER: Builder = {
}
const resolved = paths.length > 0 ? await resolveGlobOf(ops)(accessor, paths, idx) : []
const stat = async (p: PathSpec): Promise<FileStat> => ops.stat(accessor, p, idx)
return tarGeneric(resolved, opts, {
const dispatch = opts.dispatch
const fl = new FlagView(opts.flags, specOf('tar'))
if (dispatch !== undefined && !fl.asBool('c')) {
// -t reads and -x writes wherever cwd or -C says, which need not
// be this mount, so both run on dispatch-relayed doors and each
// path routes to the mount that owns it. Only -c stays on the
// accessor: its planner walks this mount's tree.
const readBytes = readBytesOp(dispatch)
async function* streamOf(p: PathSpec): AsyncIterable<Uint8Array> {
yield await readBytes(p)
}
return tarGeneric(
resolved,
texts,
opts,
{
stream: streamOf,
write: async (p, data) => {
await dispatch('write', p, [data])
},
mkdir: async (p) => {
await dispatch('mkdir', p)
},
stat: statOp(dispatch),
walk: walkOf(ops, accessor, idx),
isDir: () => Promise.resolve(false),
},
true,
)
}
return tarGeneric(resolved, texts, opts, {
stream: (p) => ops.readStream(accessor, p, idx),
write: (p, data) => write(accessor, p, data),
mkdir: (p, parents) => mkdir(accessor, p, parents),
typescript/packages/core/src/commands/builtin/generic_bind/builders/unzip.ts
+26
View File
@@ -12,6 +12,8 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { PathSpec } from '../../../../types.ts'
import { readBytesOp, statOp } from '../../generic/crossmount/utils.ts'
import { unzipGeneric } from '../../generic/unzip.ts'
import { type Builder, resolveGlobOf } from '../adapter.ts'
@@ -26,6 +28,30 @@ export const UNZIP_BUILDER: Builder = {
throw new Error('unzip: backend provides no write op')
}
const resolved = paths.length > 0 ? await resolveGlobOf(ops)(accessor, paths, idx) : []
const dispatch = opts.dispatch
if (dispatch !== undefined) {
// Extraction writes wherever cwd or -d says, which need not be
// this mount, so the doors are dispatch-relayed and each path
// routes to the mount that owns it.
const readBytes = readBytesOp(dispatch)
async function* streamOf(p: PathSpec): AsyncIterable<Uint8Array> {
yield await readBytes(p)
}
return unzipGeneric(
resolved,
texts,
opts,
streamOf,
async (p, data) => {
await dispatch('write', p, [data])
},
async (p) => {
await dispatch('mkdir', p)
},
statOp(dispatch),
true,
)
}
return unzipGeneric(
resolved,
texts,
typescript/packages/core/src/commands/builtin/grep_helper.ts
+54 -2
View File
@@ -18,6 +18,7 @@ import { mountKey, mountPrefixOf } from '../../utils/key_prefix.ts'
import { AsyncLineIterator } from '../../io/async_line_iterator.ts'
import { materialize, type IOResult } from '../../io/types.ts'
import { type FileStat, FileType, PathSpec } from '../../types.ts'
import { fnmatch } from '../../utils/fnmatch.ts'
import { getExtension } from '../resolve.ts'
import { PatternType } from './constants.ts'
import { grepContextLines } from './grep_context.ts'
@@ -36,6 +37,48 @@ export const BINARY_EXTENSIONS: ReadonlySet<string> = new Set([
export const NEVER_MATCH = '(?!)'
/**
* GNU's file-selection flags, threaded as one value.
*
* `include`/`exclude` gate files, `excludeDir` prunes directories from
* the -r walk, and `text` (-a) lets the walk read the extensions it
* would otherwise skip as binary. The empty value admits everything,
* which is what every caller without the flags passes.
*/
export interface WalkFilters {
include: readonly string[]
exclude: readonly string[]
excludeDir: readonly string[]
text: boolean
}
export const NO_FILTERS: WalkFilters = { include: [], exclude: [], excludeDir: [], text: false }
/**
* GNU's --include/--exclude gate for one candidate file.
*
* Globs match the base name with fnmatch wildcards, case sensitively (a
* glob carrying a slash therefore matches nothing, which is what GNU
* 3.11 answers too), and --exclude wins when both match. Applies to
* command-line files exactly as to walked ones, which is pinned GNU
* behavior: an explicit operand --include passes over is silently no
* match, not an error.
*/
export function fileAdmitted(path: string, filters: WalkFilters): boolean {
const trimmed = path.endsWith('/') ? path.slice(0, -1) : path
const base = trimmed.slice(trimmed.lastIndexOf('/') + 1)
if (filters.exclude.some((glob) => fnmatch(base, glob))) return false
if (filters.include.length > 0) return filters.include.some((glob) => fnmatch(base, glob))
return true
}
/** Whether the -r walk may descend into this directory. */
export function dirAdmitted(path: string, filters: WalkFilters): boolean {
const trimmed = path.endsWith('/') ? path.slice(0, -1) : path
const base = trimmed.slice(trimmed.lastIndexOf('/') + 1)
return !filters.excludeDir.some((glob) => fnmatch(base, glob))
}
const DEC = new TextDecoder()
function escapeRegex(s: string): string {
@@ -267,7 +310,11 @@ export function hasSearchShapingFlags(flags: Record<string, FlagValue>): boolean
typeof flags.B === 'string' ||
typeof flags.C === 'string' ||
flags.type !== undefined ||
flags.glob !== undefined
flags.glob !== undefined ||
flags.text === true ||
flags.include !== undefined ||
flags.exclude !== undefined ||
flags.exclude_dir !== undefined
)
}
@@ -450,6 +497,7 @@ export interface GrepFilesOnlyOptions {
maxCount: number | null
wholeWord: boolean
basic: boolean
filters?: WalkFilters
}
export async function grepRecursive(
@@ -488,7 +536,9 @@ export async function grepRecursive(
warnings.push(`grep: ${entry}: ${err instanceof Error ? err.message : String(err)}`)
continue
}
const filters = opts.filters ?? NO_FILTERS
if (s.type === FileType.DIRECTORY) {
if (!dirAdmitted(entry, filters)) continue
const sub = await grepRecursive(
readdirFn,
statFn,
@@ -502,7 +552,8 @@ export async function grepRecursive(
for (const r of sub) results.push(r)
continue
}
if (BINARY_EXTENSIONS.has(getExtension(entry) ?? '')) continue
if (!fileAdmitted(entry, filters)) continue
if (!filters.text && BINARY_EXTENSIONS.has(getExtension(entry) ?? '')) continue
try {
const lines = new TextDecoder('utf-8', { fatal: false })
.decode(await readBytesFn(entry))
@@ -608,6 +659,7 @@ export async function grepFilesOnly(
if (warnings !== null) warnings.push(`grep: ${path}: Is a directory`)
return []
}
if (!fileAdmitted(path, opts.filters ?? NO_FILTERS)) return []
let data: Uint8Array
try {
data = await readBytesFn(path)
typescript/packages/core/src/commands/spec/builtin_specs/archive.ts
+7 -1
View File
@@ -53,6 +53,7 @@ export const SPECS: Record<string, CommandSpec> = {
new Option({ short: '-v' }),
// -h archives what a symlink points at instead of the link.
new Option({ short: '-h' }),
new Option({ short: '-O', long: '--to-stdout' }),
new Option({ short: '-f', type: 'path' }),
// Every occurrence is kept, in order: GNU chdirs at each one and
// fails at the first it cannot enter, so the planner has to see
@@ -61,7 +62,12 @@ export const SPECS: Record<string, CommandSpec> = {
new Option({ long: '--strip-components', type: 'str' }),
new Option({ long: '--exclude', type: 'str' }),
],
rest: new Operand({ type: 'path' }),
// Only -c reads the rest operands from the filesystem. Under -t
// and -x each one is a member selector matched against names
// inside the archive (the same mode split MountRootPolicy keys
// on), so those lines keep them textual, exactly as unzip's
// member patterns are.
rest: new Operand({ type: 'path', textWhen: ['-x', '-t'] }),
// `tar xzf a.tgz` is the spelling everyone types.
oldOptionStyle: true,
// -C is a chdir for the operands after it, not a flag the command
typescript/packages/core/src/commands/spec/builtin_specs/search.ts
+7
View File
@@ -50,6 +50,13 @@ export const SPECS: Record<string, CommandSpec> = {
new Option({ short: '-C', type: 'str' }),
new Option({ short: '-e', type: 'str', multiple: true }),
new Option({ short: '-f', type: 'path', multiple: true }),
// -a searches the extensions the -r walk skips as binary;
// explicit operands are always read as text, which is the
// documented divergence (no "binary file matches" rows).
new Option({ short: '-a', long: '--text' }),
new Option({ long: '--include', type: 'str', multiple: true }),
new Option({ long: '--exclude', type: 'str', multiple: true }),
new Option({ long: '--exclude-dir', type: 'str', multiple: true }),
// Accepted no-ops: output is never a tty, so plain output is
// exactly what GNU produces with --color=auto (#471).
new Option({ long: '--color', type: 'str', valueOptional: true }),
typescript/packages/core/src/policy/builtin/mount_root.ts
+1 -26
View File
@@ -14,6 +14,7 @@
import type { Policy } from '../base.ts'
import type { Action, CommandContext, Deny } from '../types.ts'
import { isCreateMode } from '../../commands/builtin/generic/tar/mode.ts'
import type { PathSpec } from '../../types.ts'
/**
@@ -45,32 +46,6 @@ export function hasParentsFlag(argv: readonly string[]): boolean {
return false
}
/**
* Whether a tar line reads the filesystem rather than an archive.
*
* Only `-c` makes tar's operands source paths. Under `-t` and `-x` they are
* member selectors matched against names inside the archive, so a selector
* that happens to spell a mount root is not a mount at all and refusing it
* would deny an ordinary listing.
*
* Scanned raw for the same reason hasSymlinkFlag is: the policy fires before
* flag parsing. Only the first word may be GNU's dashless option cluster
* (`tar cf a.tar d`), so a later bare word is an operand and cannot turn the
* mode on.
*/
function isCreateMode(argv: readonly string[]): boolean {
for (const [i, tok] of argv.entries()) {
if (tok === '--create') return true
if (tok.startsWith('--')) continue
if (tok.startsWith('-')) {
if (tok.slice(1).includes('c')) return true
continue
}
if (i === 0 && tok.includes('c')) return true
}
return false
}
function deny(message: string, exitCode = 1): Deny {
return { kind: 'deny', message, exitCode }
}
typescript/packages/core/src/workspace/executor/archive_relay.test.ts
+114
View File
@@ -0,0 +1,114 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { writeTar } from '../../commands/builtin/tar_helper.ts'
import { OpsRegistry } from '../../ops/registry.ts'
import { RAMResource } from '../../resource/ram/ram.ts'
import { MountMode } from '../../types.ts'
import { gzip } from '../../utils/compress.ts'
import { getTestParser, stderrStr, stdoutStr } from '../fixtures/workspace_fixture.ts'
import { Workspace } from '../workspace.ts'
// Direct port of tests/workspace/executor/test_archive_relay.py: member
// selectors stay off routing, extraction lands in the cwd or -C across
// mounts through relay doors, and a create-mode span keeps the refusal.
const ENC = new TextEncoder()
async function tgzBytes(): Promise<Uint8Array> {
const raw = await writeTar([
{
name: './memory/memory.json',
data: ENC.encode('content:./memory/memory.json\n'),
isFile: true,
isDir: false,
linkname: '',
},
{
name: './other.txt',
data: ENC.encode('content:./other.txt\n'),
isFile: true,
isDir: false,
linkname: '',
},
])
return gzip(raw)
}
async function makeWs(): Promise<Workspace> {
const parser = await getTestParser()
const root = new RAMResource()
const work = new RAMResource()
work.store.files.set('/files.tar.gz', await tgzBytes())
const registry = new OpsRegistry()
registry.registerResource(root)
registry.registerResource(work)
return new Workspace(
{ '/': root, '/work/': work },
{ mode: MountMode.WRITE, ops: registry, shellParser: parser },
)
}
describe('tar member selectors and relay extraction', () => {
it('a selector does not join routing', async () => {
const ws = await makeWs()
const io = await ws.execute('tar -xOzf /work/files.tar.gz ./memory/memory.json')
expect(stderrStr(io)).toBe('')
expect(io.exitCode).toBe(0)
expect(stdoutStr(io)).toBe('content:./memory/memory.json\n')
})
it('a -t selector lists only its subtree', async () => {
const ws = await makeWs()
const io = await ws.execute('tar -tzf /work/files.tar.gz ./memory')
expect(io.exitCode).toBe(0)
expect(stdoutStr(io)).toBe('./memory/memory.json\n')
})
it('a miss reports Not found in archive and exits 2', async () => {
const ws = await makeWs()
const io = await ws.execute('tar -tzf /work/files.tar.gz nope')
expect(io.exitCode).toBe(2)
expect(stderrStr(io)).toBe(
'tar: nope: Not found in archive\n' +
'tar: Exiting with failure status due to previous errors\n',
)
})
it('extraction lands in the cwd across mounts', async () => {
const ws = await makeWs()
const io = await ws.execute('tar -xzf /work/files.tar.gz')
expect(io.exitCode).toBe(0)
const cat = await ws.execute('cat /memory/memory.json')
expect(stdoutStr(cat)).toBe('content:./memory/memory.json\n')
})
it('-C extracts into another mount', async () => {
const ws = await makeWs()
const io = await ws.execute('tar -xzf /work/files.tar.gz -C /dest')
expect(stderrStr(io)).toBe('')
expect(io.exitCode).toBe(0)
const cat = await ws.execute('cat /dest/other.txt')
expect(stdoutStr(cat)).toBe('content:./other.txt\n')
})
it('a create-mode span keeps the refusal', async () => {
const ws = await makeWs()
await ws.execute('mkdir -p /src && echo hi > /src/f.txt')
const io = await ws.execute('tar -czf /work/backup.tgz /src')
expect(io.exitCode).toBe(1)
expect(stderrStr(io)).toContain('paths span multiple mounts')
})
})
typescript/packages/core/src/workspace/executor/builtins/index.ts
+1 -1
View File
@@ -43,7 +43,7 @@ export {
} from './vars.ts'
export { handleMan } from './man.ts'
export { handleHistory } from './history.ts'
export { handleBash, handleEval, handleSleep, handleSource } from './script.ts'
export { handleBash, handleEval, handleExecPath, handleSleep, handleSource } from './script.ts'
export { handleTest } from './condition/index.ts'
export { handleTimeout } from './timeout.ts'
export { handleXargs } from './xargs.ts'
typescript/packages/core/src/workspace/executor/builtins/script.ts
+79
View File
@@ -233,6 +233,85 @@ async function readScriptFile(
}
}
/** The interpreter words a script's first line names, env resolved. */
function shebangWords(script: string): string[] {
const first = script.split('\n', 1)[0] ?? ''
if (!first.startsWith('#!')) return []
let words = first
.slice(2)
.trim()
.split(/\s+/)
.filter((w) => w !== '')
const head = words[0] ?? ''
if (head.slice(head.lastIndexOf('/') + 1) === 'env') {
words = words.slice(1)
return words
}
if (words.length > 0) {
words[0] = head.slice(head.lastIndexOf('/') + 1)
}
return words
}
/** Quote one word for re-dispatch as a shell line. */
function quoteWord(word: string): string {
if (/^[A-Za-z0-9_\-./]+$/.test(word)) return word
return `'${word.replaceAll("'", `'\\''`)}'`
}
/**
* Run a slash-carrying head word as a program, bash's loader rule.
*
* bash hands a word containing a slash straight to the loader: no
* builtin, function, or install can claim it, and the file either runs
* or the shell reports why not. Two deliberate divergences from bash,
* both consequences of the VFS: there is no exec bit to check (`chmod`
* is stored, not enforced; mount mode does real access control), so an
* existing file runs without `+x`; and the shell prefix bash puts on
* the diagnostic is dropped, matching every other mirage diagnostic.
*
* A shebang naming sh or bash (directly or via env) runs through the
* nested-shell machinery, as does a script with none. Any other
* interpreter word is re-dispatched as a command line, so
* `#!/usr/bin/env python3` reaches the python3 command wherever the
* workspace routes it, and an interpreter nobody registers answers with
* its own "command not found".
*/
export async function handleExecPath(
dispatch: DispatchFn,
executeFn: ExecuteStringFn,
path: string,
args: string[],
session: Session,
stdin: ByteSource | null = null,
): Promise<Result> {
let script: string
try {
script = await readScriptText(dispatch, path, session.cwd)
} catch (exc) {
const strerror = fsStrerror(exc)
if (strerror === null) throw exc
const code = (exc as { code?: string }).code
return scriptError(path, strerror, code === 'ENOENT' ? 127 : 126)
}
const words = shebangWords(script)
const interp = words[0] ?? 'sh'
if (interp === 'sh' || interp === 'bash') {
return handleBash(
dispatch,
executeFn,
[...words.slice(1), path, ...args],
session,
stdin,
interp,
)
}
const line = [...words, path, ...args].map(quoteWord).join(' ')
const io = await executeFn(line, { sessionId: session.sessionId, stdin })
const label = args.length > 0 ? `${path} ${args.join(' ')}` : path
return [io.stdout, io, new ExecutionNode({ command: label, exitCode: io.exitCode })]
}
/**
* Run a nested shell: inline text from `-c`, or a script file.
*
typescript/packages/core/src/workspace/executor/command.ts
+9 -2
View File
@@ -32,6 +32,7 @@ import type { Session } from '../session/session.ts'
import { ExecutionNode } from '../types.ts'
import { strategyFor } from '../../commands/builtin/generic/crossmount/detect.ts'
import type { Cmd } from '../../commands/builtin/generic/crossmount/types.ts'
import { isCreateMode } from '../../commands/builtin/generic/tar/mode.ts'
import { Strategy } from '../../commands/builtin/generic/crossmount/types.ts'
import { resolveGlobs } from '../expand/globs.ts'
import type { DispatchFn } from '../../runtime/types.ts'
@@ -217,8 +218,14 @@ export async function handleCommand(
}
// Path-valued flags count: `cp -t /other/mount/dir src` spans mounts
// exactly like a positional destination would.
if (isCrossMount(cmdName, routingScopes, registry)) {
// exactly like a positional destination would. A create-mode tar is
// the one relay member kept out: its planner walks a single backend's
// tree, so a span there falls through to the refusal below instead of
// a relay run that would cross nested mounts.
if (
isCrossMount(cmdName, routingScopes, registry) &&
!(cmdName === 'tar' && isCreateMode(rawArgv))
) {
// Parse against the shared spec so flags and text operands do not
// depend on the source mount: raw argv would hand flag tokens ("-c")
// to the generic as the search pattern. The bound single-mount runner
typescript/packages/core/src/workspace/executor/exec_path.test.ts
+91
View File
@@ -0,0 +1,91 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { OpsRegistry } from '../../ops/registry.ts'
import { RAMResource } from '../../resource/ram/ram.ts'
import { MountMode } from '../../types.ts'
import { getTestParser, stderrStr, stdoutStr } from '../fixtures/workspace_fixture.ts'
import { Workspace } from '../workspace.ts'
// Direct port of tests/workspace/executor/builtins/test_exec_path.py: a
// slash-carrying head word runs the file, bash's loader rule.
async function makeWs(): Promise<Workspace> {
const parser = await getTestParser()
const root = new RAMResource()
const work = new RAMResource()
const registry = new OpsRegistry()
registry.registerResource(root)
registry.registerResource(work)
return new Workspace(
{ '/': root, '/work/': work },
{ mode: MountMode.WRITE, ops: registry, shellParser: parser },
)
}
describe('direct path execution', () => {
it('runs a script written to a mount', async () => {
const ws = await makeWs()
await ws.execute("printf 'echo ran\\n' > /work/run.sh")
const io = await ws.execute('/work/run.sh')
expect(io.exitCode).toBe(0)
expect(stdoutStr(io)).toBe('ran\n')
})
it('resolves a relative path against the cwd', async () => {
const ws = await makeWs()
await ws.execute("printf 'echo rel\\n' > /work/run.sh")
const io = await ws.execute('cd /work && ./run.sh')
expect(stdoutStr(io)).toBe('rel\n')
})
it('honors shebang interpreter options', async () => {
const ws = await makeWs()
await ws.execute("printf '#!/bin/bash -x\\necho traced\\n' > /work/t.sh")
const io = await ws.execute('/work/t.sh')
expect(stdoutStr(io)).toBe('traced\n')
expect(stderrStr(io)).toBe('+ echo traced\n')
})
it('a missing file is 127', async () => {
const ws = await makeWs()
const io = await ws.execute('/work/nope.sh')
expect(io.exitCode).toBe(127)
expect(stderrStr(io)).toBe('/work/nope.sh: No such file or directory\n')
})
it('a directory is 126', async () => {
const ws = await makeWs()
await ws.execute('mkdir -p /work/adir')
const io = await ws.execute('/work/adir')
expect(io.exitCode).toBe(126)
expect(stderrStr(io)).toBe('/work/adir: Is a directory\n')
})
it('an unknown interpreter reports command not found', async () => {
const ws = await makeWs()
await ws.execute("printf '#!/usr/bin/env ruby\\nputs 1\\n' > /work/r.rb")
const io = await ws.execute('/work/r.rb')
expect(io.exitCode).toBe(127)
expect(stderrStr(io)).toBe('ruby: command not found\n')
})
it('child shell state does not leak', async () => {
const ws = await makeWs()
await ws.execute("printf 'cd /work\\nexit 3\\n' > /child.sh")
const io = await ws.execute('/child.sh; echo "$?:$(pwd)"')
expect(stdoutStr(io)).toBe('3:/\n')
})
})
typescript/packages/core/src/workspace/executor/grep_filters.test.ts
+88
View File
@@ -0,0 +1,88 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { OpsRegistry } from '../../ops/registry.ts'
import { RAMResource } from '../../resource/ram/ram.ts'
import { MountMode } from '../../types.ts'
import { getTestParser, stderrStr, stdoutStr } from '../fixtures/workspace_fixture.ts'
import { Workspace } from '../workspace.ts'
// Direct port of tests/commands/native/test_grep_filters.py, pinned
// against GNU grep 3.11 on debian:stable-slim.
const ENC = new TextEncoder()
async function makeWs(): Promise<Workspace> {
const parser = await getTestParser()
const r = new RAMResource()
r.store.files.set('/notes.tex', ENC.encode('score 9\n'))
r.store.files.set('/notes.txt', ENC.encode('score 8\n'))
r.store.dirs.add('/sub')
r.store.files.set('/sub/inner.tex', ENC.encode('score 7\n'))
r.store.files.set('/data.parquet', ENC.encode('score binary\n'))
const registry = new OpsRegistry()
registry.registerResource(r)
return new Workspace({ '/': r }, { mode: MountMode.WRITE, ops: registry, shellParser: parser })
}
describe('grep --include/--exclude/--exclude-dir and -a', () => {
it('--include filters the recursive walk', async () => {
const ws = await makeWs()
const io = await ws.execute("grep -RInE --include='*.tex' score /")
expect(io.exitCode).toBe(0)
expect(stdoutStr(io)).toBe('/notes.tex:1:score 9\n/sub/inner.tex:1:score 7\n')
})
it('--exclude wins over --include', async () => {
const ws = await makeWs()
const io = await ws.execute("grep -r --include='*.tex' --exclude='notes.*' score /")
expect(stdoutStr(io)).toBe('/sub/inner.tex:score 7\n')
})
it('--exclude-dir prunes the walk', async () => {
const ws = await makeWs()
const io = await ws.execute("grep -r --include='*.tex' --exclude-dir=sub score /")
expect(stdoutStr(io)).toBe('/notes.tex:score 9\n')
})
it('a glob carrying a slash matches nothing', async () => {
const ws = await makeWs()
const io = await ws.execute("grep -r --include='sub/*.tex' score /")
expect(io.exitCode).toBe(1)
expect(stdoutStr(io)).toBe('')
})
it('--include filters an explicit operand in silence', async () => {
const ws = await makeWs()
const io = await ws.execute("grep --include='*.tex' -n score /notes.txt")
expect(io.exitCode).toBe(1)
expect(stdoutStr(io)).toBe('')
expect(stderrStr(io)).toBe('')
})
it('-a reads binary extensions in the walk', async () => {
const ws = await makeWs()
const without = await ws.execute('grep -r score /')
expect(stdoutStr(without)).not.toContain('data.parquet')
const withA = await ws.execute('grep -ra score /')
expect(stdoutStr(withA)).toContain('/data.parquet:score binary')
})
it('-a is accepted on an explicit operand', async () => {
const ws = await makeWs()
const io = await ws.execute("grep -aoiE 'SCORE' /notes.tex")
expect(stdoutStr(io)).toBe('score\n')
})
})
typescript/packages/core/src/workspace/node/command_dispatch.ts
+9
View File
@@ -47,6 +47,7 @@ import {
acceptsLine,
followPaths,
handleBash,
handleExecPath,
handleCd,
handleCommandBuiltin,
handleType,
@@ -510,6 +511,14 @@ async function runArgv(
}
}
// Path execution: bash hands a slash-carrying head word to the
// loader, never to command lookup: no builtin, function, or CLI can
// claim it. After the admission gate so a policy sees the line like
// any other.
if (name.includes('/')) {
return handleExecPath(dispatch, executeFn, name, args, session, stdin)
}
// Unsupported bash builtins. Constructs the parser accepts but the
// executor cannot honor. Returning a clear error lets LLMs detect a
// capability gap instead of treating it as a missing binary.