github-featured/strukto-ai--mirage
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat(du): Honor -S/--separate-dirs (py + ts) (#722)

Browse Source 
* feat(du): honor -S/--separate-dirs (py + ts)

GNU -S makes each directory total count only files that sit directly in
it, leaving subdirectory sizes out of the parent. Shared rollup and
summarize paths now match coreutils 9.7, with unit and integ coverage.

* fix(du): apply -S across nested mounts, keep the -c total recursive

The traversal fan-out re-derives du's tree centrally from the per-mount
blocks, so -S was silently dropped there: du -S over a mount printed the
recursive total. Teach the merge about it, and split the operand's own
row from what it contributes to -c, which GNU keeps recursive.

Add a guard that fails when a new du flag is neither applied centrally
nor classified as per-run, which is how -S went missing.

* chore(spec): regenerate du spec json for -S

---------

Co-authored-by: Zecheng Zhang <zechengzhang97@gmail.com>
This commit is contained in:
Thomas Hart
2026-08-13 08:45:08 -06:00
committed by GitHub
parent 0d5915457f
commit 20543fb031
16 changed files with 666 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
integ/unix/du/separate_dirs.json
+152
View File
@@ -0,0 +1,152 @@
{
"cases": [
{
"id": "du_separate_dirs",
"seq": 273,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "du -S /data/sub",
"expect": {
"exit": 0,
"stdout": "5\t/data/sub/deep\n15\t/data/sub\n",
"stderr": ""
},
"flags": [
"S"
]
},
{
"id": "du_separate_dirs_long",
"seq": 274,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "du --separate-dirs /data/sub",
"expect": {
"exit": 0,
"stdout": "5\t/data/sub/deep\n15\t/data/sub\n",
"stderr": ""
},
"flags": [
"separate-dirs"
]
},
{
"id": "du_separate_dirs_summarize",
"seq": 281,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "du -Ss /data/sub",
"expect": {
"exit": 0,
"stdout": "15\t/data/sub\n",
"stderr": ""
},
"flags": [
"S"
]
},
{
"id": "du_separate_dirs_all",
"seq": 282,
"targets": [
"ram",
"disk",
"redis",
"opfs",
"s3",
"databricks",
"gridfs",
"s3-prefix",
"databricks-prefix",
"gridfs-prefix",
"hf",
"hf-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"sharepoint",
"sharepoint-prefix",
"ssh",
"nextcloud",
"gdrive",
"gdrive-folder",
"box"
],
"command": "du -Sa /data/sub",
"expect": {
"exit": 0,
"stdout": "5\t/data/sub/deep/deeper.txt\n5\t/data/sub/deep\n15\t/data/sub/nested.txt\n15\t/data/sub\n",
"stderr": ""
},
"flags": [
"S"
]
}
]
}
python/mirage/commands/builtin/generic/crossmount/fanout/du.py
+16 -3
View File
@@ -15,7 +15,7 @@
from collections.abc import Sequence
from mirage.commands.builtin.generic.crossmount.types import OperandRun
from mirage.commands.builtin.generic.du import rollup
from mirage.commands.builtin.generic.du import rollup, separate_total
from mirage.commands.builtin.utils.formatting import _human_size
from mirage.utils.path import respell_raw
@@ -75,6 +75,7 @@ def merge_du_blocks(
c: bool,
human: bool,
max_depth: int | None,
separate_dirs: bool = False,
mount_roots: Sequence[str] = (),
) -> bytes:
"""Fold per-mount du blocks into one tree, GNU's way.
@@ -101,6 +102,9 @@ def merge_du_blocks(
c (bool): -c, append the grand total row.
human (bool): format the sizes like ``du -h`` does.
max_depth (int | None): --max-depth, prune what is printed.
separate_dirs (bool): -S, a directory counts only the files that
sit directly in it. The per-mount runs are asked without it,
because the merge needs their leaves and applies it here.
mount_roots (Sequence[str]): the descendant mount roots, which
are directories whether or not they hold anything. An empty
mount contributes only its own row, which the leaf inference
@@ -108,15 +112,24 @@ def merge_du_blocks(
"""
leaves = _leaves(_parse_rows(blocks), mount_roots)
total = sum(size for _, size in leaves)
# -S scopes to the operand's own row; GNU keeps the -c grand total
# recursive (coreutils 9.7 over a real mount: `du -bSc base` prints
# `3 base` then `10 total`).
own = separate_total(leaves, root) if separate_dirs else total
lines: list[str] = []
if not s:
rows = rollup(leaves, root, a=a, max_depth=max_depth, dirs=mount_roots)
rows = rollup(leaves,
root,
a=a,
max_depth=max_depth,
dirs=mount_roots,
separate_dirs=separate_dirs)
shown = respell_raw([node for node, _ in rows], root, label)
lines = [
_format_size(size, human) + "\t" + name
for name, (_, size) in zip(shown, rows)
]
lines.append(_format_size(total, human) + "\t" + label)
lines.append(_format_size(own, human) + "\t" + label)
if c:
lines.append(_format_size(total, human) + "\ttotal")
return ("\n".join(lines) + "\n").encode()
python/mirage/commands/builtin/generic/du.py
+80 -19
View File
@@ -38,6 +38,7 @@ class DuFlags:
a (bool): -a, list files as well as directories.
h (bool): -h, human-readable sizes.
c (bool): -c, append a grand total.
S (bool): -S/--separate-dirs, directories exclude subdirectory sizes.
max_depth (int | None): --max-depth/-d, deepest level to print.
warning (str | None): a non-fatal diagnostic GNU prints before the
output, without failing the command.
@@ -47,6 +48,7 @@ class DuFlags:
a: bool = False
h: bool = False
c: bool = False
S: bool = False
max_depth: int | None = None
warning: str | None = None
@@ -93,8 +95,13 @@ def parse_depth(text: str) -> int | None:
return None
def parse_flags(*, s: bool, a: bool, h: bool, c: bool,
max_depth: str | None) -> DuFlags:
def parse_flags(*,
s: bool,
a: bool,
h: bool,
c: bool,
max_depth: str | None,
separate_dirs: bool = False) -> DuFlags:
"""Validate a ``du`` command line the way GNU does, before any I/O.
GNU parses ``--max-depth`` as each option is read, so a bad depth is
@@ -107,6 +114,7 @@ def parse_flags(*, s: bool, a: bool, h: bool, c: bool,
h (bool): -h.
c (bool): -c.
max_depth (str | None): raw --max-depth/-d text, unparsed.
separate_dirs (bool): -S/--separate-dirs.
Raises:
UsageError: on a bad depth or a conflicting combination.
@@ -130,7 +138,13 @@ def parse_flags(*, s: bool, a: bool, h: bool, c: bool,
f"--max-depth={depth}\n{USAGE_HINT}", 1)
warning = ("du: warning: summarizing is the same as using "
"--max-depth=0")
return DuFlags(s=s, a=a, h=h, c=c, max_depth=depth, warning=warning)
return DuFlags(s=s,
a=a,
h=h,
c=c,
S=separate_dirs,
max_depth=depth,
warning=warning)
def cwd_spec(cwd: PathSpec | str) -> PathSpec:
@@ -273,13 +287,30 @@ def to_virtual(entries: Sequence[tuple[str, int]],
for entry, size in entries]
def separate_total(entries: Sequence[tuple[str, int]], root: str) -> int:
"""Bytes of the leaves sitting directly in the operand (GNU ``-S``).
This is the operand's own row under ``-S``, not what it contributes
to the ``-c`` grand total: GNU keeps that recursive (coreutils 9.7,
``du -bSc dir`` prints ``3 dir`` then ``6 total``).
Args:
entries (Sequence[tuple[str, int]]): leaf (virtual path, size).
root (str): the operand's absolute virtual path.
"""
root_key = _norm(root)
return sum(size for leaf, size in entries
if _parent(_norm(leaf)) == root_key)
def rollup(
entries: Sequence[tuple[str, int]],
root: str,
*,
a: bool,
max_depth: int | None,
dirs: Sequence[str] = (),
entries: Sequence[tuple[str, int]],
root: str,
*,
a: bool,
max_depth: int | None,
dirs: Sequence[str] = (),
separate_dirs: bool = False,
) -> list[tuple[str, int]]:
"""Derive GNU's per-directory lines from a flat list of leaf files.
@@ -291,8 +322,11 @@ def rollup(
readdir order, which is unspecified, so sorting is a deterministic
choice within the same shape.
The operand's own line is not included; the caller renders it with
the operand as typed.
With ``-S``/``--separate-dirs`` a directory only counts files that
sit directly in it: a leaf still forces every ancestor directory to
appear (possibly at size 0), but only the immediate parent gets its
bytes. The operand's own line is not included; the caller renders it
with the operand as typed.
Args:
entries (Sequence[tuple[str, int]]): leaf (virtual path, size).
@@ -303,6 +337,7 @@ def rollup(
leaf points at them. mirage cannot otherwise see an empty
directory, so this is the one case it can: an empty mount
still gets GNU's ``0`` row.
separate_dirs (bool): -S, exclude subdirectory sizes.
Returns:
list[tuple[str, int]]: (virtual path, size) in GNU's print order.
@@ -317,8 +352,16 @@ def rollup(
continue
files[node] = size
parent = _parent(node)
immediate = True
while parent != root_key and parent.startswith(prefix):
sizes[parent] = sizes.get(parent, 0) + size
if separate_dirs and not immediate:
# -S: only the directory a file sits in counts its
# bytes. The ancestors still print, at 0 when they hold
# nothing but directories.
sizes.setdefault(parent, 0)
else:
sizes[parent] = sizes.get(parent, 0) + size
immediate = False
parent = _parent(parent)
# setdefault, never assignment: a hinted directory that does hold
@@ -425,7 +468,7 @@ async def _du_one(
leaves = drop_shadowed(leaves, roots)
link_total = sum(size for _, size in leaves)
if flags.s and not roots:
if flags.s and not flags.S and not roots:
total = await compute_size(path) + link_total
return [_line(total, flags.h, label)], total
@@ -443,20 +486,30 @@ async def _du_one(
# honest number is the sum of what survived.
virtual = drop_shadowed(virtual, roots)
total = sum(size for _, size in virtual)
if flags.s:
return [_line(total, flags.h, label)], total
root_key = _norm(path.virtual)
# A file operand walks to itself. GNU prints it once, with or
# without -a, never as a leaf line plus a roll-up line.
# without -a, never as a leaf line plus a roll-up line. GNU scopes
# -S to directories, so a file operand keeps its own size in both
# its row and the grand total.
if len(virtual) == 1 and _norm(virtual[0][0]) == root_key:
return [_line(virtual[0][1], flags.h, label)], total
# -S changes what the operand's own row counts, not what the operand
# contributes to -c: GNU's grand total stays recursive (coreutils
# 9.7, `du -bSc dir` prints `3 dir` then `6 total`).
own = separate_total(virtual, path.virtual) if flags.S else total
if flags.s:
return [_line(own, flags.h, label)], total
rows = rollup(virtual, path.virtual, a=flags.a, max_depth=flags.max_depth)
rows = rollup(virtual,
path.virtual,
a=flags.a,
max_depth=flags.max_depth,
separate_dirs=flags.S)
shown = respell_raw([node for node, _ in rows], path.virtual, label)
lines = [
_line(size, flags.h, name) for name, (_, size) in zip(shown, rows)
]
lines.append(_line(total, flags.h, label))
lines.append(_line(own, flags.h, label))
return lines, total
@@ -473,6 +526,7 @@ async def run_du(
h: bool = False,
c: bool = False,
max_depth: str | None = None,
separate_dirs: bool = False,
truncated: Callable[[], bool] | None = None,
links: LinkView | None = None,
mounts: MountView | None = None,
@@ -496,6 +550,7 @@ async def run_du(
h (bool): -h.
c (bool): -c.
max_depth (str | None): raw --max-depth text.
separate_dirs (bool): -S/--separate-dirs.
truncated (Callable[[], bool] | None): whether a walk was cut off.
links (LinkView | None): the namespace's symlink facts.
mounts (MountView | None): where the mount boundaries are, so
@@ -505,7 +560,12 @@ async def run_du(
Raises:
UsageError: on a bad depth or a conflicting flag combination.
"""
flags = parse_flags(s=s, a=a, h=h, c=c, max_depth=max_depth)
flags = parse_flags(s=s,
a=a,
h=h,
c=c,
max_depth=max_depth,
separate_dirs=separate_dirs)
present, missing = await du_operands(paths,
cwd,
resolve_glob,
@@ -621,6 +681,7 @@ async def du_generic(
h=fl.as_bool("h"),
c=fl.as_bool("c"),
max_depth=fl.as_str("max_depth"),
separate_dirs=fl.as_bool("separate_dirs"),
truncated=truncated,
links=(None if fl.as_bool("L") else
opts.ns.links if opts.ns is not None else None),
python/mirage/commands/spec/builtin_specs/listing.py
+1
View File
@@ -105,6 +105,7 @@ SPECS: dict[str, CommandSpec] = {
Option(short="-c"),
Option(short="-L"),
Option(short="-P"),
Option(short="-S", long="--separate-dirs"),
),
rest=Operand(type="path"),
),
python/mirage/workspace/executor/fanout.py
+7 -2
View File
@@ -48,6 +48,7 @@ class _DuFanFlags:
c: bool
human: bool
max_depth: int | None
separate_dirs: bool
def _path_segments(path: str) -> list[str]:
@@ -430,13 +431,16 @@ async def _fan_out_traversal(
c=flag_kwargs.get("c") is True,
human=flag_kwargs.get("h") is True,
max_depth=_depth_flag_value(
flag_kwargs.get("max_depth")))
flag_kwargs.get("max_depth")),
separate_dirs=flag_kwargs.get("separate_dirs")
is True)
if du_merge:
flag_kwargs = {
**flag_kwargs, "a": True,
"s": False,
"c": False,
"h": False
"h": False,
"separate_dirs": False
}
flag_kwargs.pop("max_depth", None)
@@ -529,6 +533,7 @@ async def _fan_out_traversal(
c=du_flags.c,
human=du_flags.human,
max_depth=du_flags.max_depth,
separate_dirs=du_flags.separate_dirs,
mount_roots=await
_mount_dirs(descendants, stat_path))
elif all_stdout and cmd_name == "find" and len(paths) == 1:
python/tests/commands/builtin/generic/test_du.py
+120 -1
View File
@@ -3,7 +3,8 @@ import pytest
from mirage import MountMode, Workspace
from mirage.commands.builtin.generic.du import (DuFlags, _depth, du,
parse_depth, parse_flags,
rollup, run_du, to_virtual)
rollup, run_du, separate_total,
to_virtual)
from mirage.commands.builtin.generic_bind import CommandIO, DuOps
from mirage.commands.errors import UsageError
from mirage.ops.types import LinkView, MountView
@@ -92,6 +93,87 @@ async def test_subdirectories_get_their_own_line():
assert out.stdout == b"1\t/dir/sub/deep\n3\t/dir/sub\n6\t/dir\n"
@pytest.mark.asyncio
async def test_separate_dirs_excludes_subdirectory_sizes():
"""GNU -S: parent totals omit children that are directories."""
tree = {"/dir/a.txt": 3, "/dir/sub/b.txt": 2, "/dir/sub/deep/c.txt": 1}
compute_size, compute_entries = _make_backend(tree)
out = await du([_spec("/dir", "dir")],
compute_size=compute_size,
compute_entries=compute_entries,
flags=DuFlags(S=True))
assert out.stdout == b"1\t/dir/sub/deep\n2\t/dir/sub\n3\t/dir\n"
@pytest.mark.asyncio
async def test_separate_dirs_with_summarize_uses_direct_files_only():
tree = {"/dir/a.txt": 3, "/dir/sub/b.txt": 2, "/dir/sub/deep/c.txt": 1}
compute_size, compute_entries = _make_backend(tree)
out = await du([_spec("/dir", "dir")],
compute_size=compute_size,
compute_entries=compute_entries,
flags=DuFlags(s=True, S=True))
assert out.stdout == b"3\t/dir\n"
@pytest.mark.asyncio
async def test_separate_dirs_with_all_lists_files():
tree = {"/dir/a.txt": 3, "/dir/sub/b.txt": 2, "/dir/sub/deep/c.txt": 1}
compute_size, compute_entries = _make_backend(tree)
out = await du([_spec("/dir", "dir")],
compute_size=compute_size,
compute_entries=compute_entries,
flags=DuFlags(a=True, S=True))
assert out.stdout == (b"3\t/dir/a.txt\n"
b"2\t/dir/sub/b.txt\n"
b"1\t/dir/sub/deep/c.txt\n"
b"1\t/dir/sub/deep\n"
b"2\t/dir/sub\n"
b"3\t/dir\n")
@pytest.mark.asyncio
async def test_separate_dirs_keeps_the_grand_total_recursive():
"""GNU -Sc: rows are separate, the total is not (coreutils 9.7)."""
tree = {"/dir/a.txt": 3, "/dir/sub/b.txt": 2, "/dir/sub/deep/c.txt": 1}
compute_size, compute_entries = _make_backend(tree)
out = await du([_spec("/dir", "dir")],
compute_size=compute_size,
compute_entries=compute_entries,
flags=DuFlags(c=True, S=True))
assert out.stdout == (b"1\t/dir/sub/deep\n"
b"2\t/dir/sub\n"
b"3\t/dir\n"
b"6\ttotal\n")
@pytest.mark.asyncio
async def test_separate_dirs_summarize_still_totals_recursively():
tree = {"/dir/a.txt": 3, "/dir/sub/b.txt": 2, "/dir/sub/deep/c.txt": 1}
compute_size, compute_entries = _make_backend(tree)
out = await du([_spec("/dir", "dir")],
compute_size=compute_size,
compute_entries=compute_entries,
flags=DuFlags(s=True, c=True, S=True))
assert out.stdout == b"3\t/dir\n6\ttotal\n"
@pytest.mark.asyncio
async def test_separate_dirs_keeps_a_file_operand_in_the_total():
"""GNU scopes -S to directories: a file operand counts itself."""
compute_size, compute_entries = _make_backend({"/f.txt": 7})
out = await du([_spec("/f.txt", "f.txt")],
compute_size=compute_size,
compute_entries=compute_entries,
flags=DuFlags(c=True, S=True))
assert out.stdout == b"7\t/f.txt\n7\ttotal\n"
def test_separate_total_sums_direct_children_only():
entries = [("/d/a.txt", 3), ("/d/sub/b.txt", 2), ("/d/sub/deep/c.txt", 1)]
assert separate_total(entries, "/d") == 3
@pytest.mark.asyncio
async def test_a_lists_every_file_then_every_directory():
"""Post-order: children before parents, exactly like GNU."""
@@ -436,6 +518,25 @@ def test_rollup_totals_are_recursive():
assert rows["/d/sub/deep"] == 1
def test_rollup_separate_dirs_counts_only_direct_files():
"""GNU -S: a directory omits subdirectory sizes (pinned coreutils 9.7)."""
entries = [("/d/a.txt", 3), ("/d/sub/b.txt", 2), ("/d/sub/deep/c.txt", 1)]
rows = dict(
rollup(entries, "/d", a=False, max_depth=None, separate_dirs=True))
assert rows["/d/sub/deep"] == 1
assert rows["/d/sub"] == 2
assert "/d/a.txt" not in rows
def test_rollup_separate_dirs_keeps_empty_ancestor_dirs():
"""A directory with only subdirs still prints, at size 0."""
entries = [("/d/sub/deep/c.txt", 4)]
rows = dict(
rollup(entries, "/d", a=False, max_depth=None, separate_dirs=True))
assert rows["/d/sub/deep"] == 4
assert rows["/d/sub"] == 0
def test_rollup_a_keeps_the_sum_over_a_directory_marker():
entries = [("/d/sub/deep/c.txt", 5), ("/d/sub/deep", 0)]
rows = dict(rollup(entries, "/d", a=True, max_depth=None))
@@ -581,6 +682,24 @@ async def test_fully_shadowed_operand_reports_zero():
assert out.stdout == b"0\t/base\n"
@pytest.mark.asyncio
@pytest.mark.parametrize("flag", ["-S", "--separate-dirs"])
async def test_du_separate_dirs_off_the_command_line(tmp_path, flag):
res = DiskResource(root=str(tmp_path))
ws = Workspace({"/d": res}, mode=MountMode.WRITE)
await ws.execute("mkdir -p /d/sub/deep")
await ws.execute("printf abc > /d/a.txt")
await ws.execute("printf de > /d/sub/b.txt")
await ws.execute("printf f > /d/sub/deep/c.txt")
result = await ws.execute(f"du {flag} -c /d")
assert result.exit_code == 0
assert await result.stdout_str() == ("1\t/d/sub/deep\n"
"2\t/d/sub\n"
"3\t/d\n"
"6\ttotal\n")
await ws.close()
@pytest.mark.asyncio
async def test_du_missing_operand_reports_and_exits_1(tmp_path):
# GNU: "du: cannot access 'X': No such file or directory", exit 1. Walking
python/tests/workspace/executor/test_fanout.py
+63
View File
@@ -3,6 +3,8 @@ from types import SimpleNamespace
import pytest
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import spec_flag_names
from mirage.io import IOResult
from mirage.ops.types import NamespaceView
from mirage.resource.ram import RAMResource
@@ -285,6 +287,37 @@ def test_du_c_fanout_prints_one_total_across_the_mounts():
assert _stdout(io) == "7\t/base/inner\n17\t/base\n17\ttotal\n"
def test_du_separate_dirs_fanout_scopes_only_the_rows():
"""``-S`` reaches the merge, and the ``-c`` total stays recursive.
Pinned on coreutils 9.7 over a tmpfs mounted at the same spot:
``du -bS base`` prints ``7 base/inner`` then ``10 base`` (the parent
counts only the file sitting in it), and ``du -bSc base`` still ends
``17 total``.
"""
ws = _shadowed_workspace()
assert _stdout(asyncio.run(
ws.execute("du -S /base"))) == "7\t/base/inner\n10\t/base\n"
assert _stdout(asyncio.run(ws.execute(
"du -Sc /base"))) == "7\t/base/inner\n10\t/base\n17\ttotal\n"
def test_du_separate_dirs_summarize_fanout():
ws = _shadowed_workspace()
assert _stdout(asyncio.run(ws.execute("du -Ss /base"))) == "10\t/base\n"
assert _stdout(asyncio.run(
ws.execute("du -Ssc /base"))) == "10\t/base\n17\ttotal\n"
def test_du_separate_dirs_all_fanout():
ws = _shadowed_workspace()
assert _stdout(asyncio.run(
ws.execute("du -Sa /base"))) == ("7\t/base/inner/real.txt\n"
"7\t/base/inner\n"
"10\t/base/top.txt\n"
"10\t/base\n")
def test_du_sc_fanout_prints_one_total():
ws = _shadowed_workspace()
io = asyncio.run(ws.execute("du -sc /base"))
@@ -395,6 +428,36 @@ def test_operands_spanning_mounts_still_fan_out_inside_each_operand():
"29\ttotal\n")
def test_du_fan_out_accounts_for_every_du_flag():
"""The du merge re-derives the whole tree centrally, so the sub-runs
are asked with the presentation flags stripped and each one is then
applied once, here. A flag nobody classified is neither stripped nor
re-applied, so it silently does nothing across a nested mount, which
is exactly how -S first shipped. Adding an option to du's spec fails
this until it is sorted into one of the two lists."""
# Applied centrally by _DuFanFlags, and neutralized in the sub-runs.
central = {"a", "s", "c", "h", "max_depth", "separate_dirs"}
# Chooses whether a run counts the symlinks on its own mount, which
# is a per-run question; the merge only ever sees the rows.
per_run = {"L", "P"}
assert spec_flag_names(SPECS["du"]) == central | per_run
def test_operands_spanning_mounts_separate_dirs():
"""-S has to survive both fan-outs at once: the per-operand one that
splits the operands across mounts, and the traversal one that folds
`/base/inner` into `/base`. GNU (coreutils 9.7, tmpfs at the nested
spot) scopes -S to each printed row and keeps the grand total
recursive, so `/base` reports only `top.txt` while the total still
covers every byte."""
ws = _spanning_workspace()
io = asyncio.run(ws.execute("du -Sc /base /other"))
assert _stdout(io) == ("9\t/base/inner\n"
"10\t/base\n"
"10\t/other\n"
"29\ttotal\n")
def test_operands_spanning_mounts_fan_out_for_find_and_grep():
ws = _spanning_workspace()
found = _stdout(asyncio.run(ws.execute("find /base /other")))
spec/python/general/du.json
+5
View File
@@ -290,6 +290,11 @@
{
"short": "-P",
"type": "bool"
},
{
"long": "--separate-dirs",
"short": "-S",
"type": "bool"
}
],
"rest": {
spec/typescript/browser/general/du.json
+5
View File
@@ -248,6 +248,11 @@
{
"short": "-P",
"type": "bool"
},
{
"long": "--separate-dirs",
"short": "-S",
"type": "bool"
}
],
"rest": {
spec/typescript/node/general/du.json
+5
View File
@@ -311,6 +311,11 @@
{
"short": "-P",
"type": "bool"
},
{
"long": "--separate-dirs",
"short": "-S",
"type": "bool"
}
],
"rest": {
typescript/packages/core/src/commands/builtin/generic/crossmount/fanout/du.ts
+11 -2
View File
@@ -13,7 +13,7 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { humanSize } from '../../../utils/formatting.ts'
import { rollup } from '../../du.ts'
import { rollup, separateTotal } from '../../du.ts'
import { respellRaw } from '../../../../../utils/path.ts'
import type { OperandRun } from '../types.ts'
@@ -121,18 +121,27 @@ export function mergeDuBlocks(
total: boolean
human: boolean
maxDepth: number | null
// -S, a directory counts only the files that sit directly in it. The
// per-mount runs are asked without it, because the merge needs their
// leaves and applies it here.
separateDirs?: boolean
mountRoots?: readonly string[]
},
): Uint8Array {
const mountRoots = opts.mountRoots ?? []
const leaves = leavesOf(parseRows(blocks), mountRoots)
const sum = leaves.reduce((acc, [, size]) => acc + size, 0)
// -S scopes to the operand's own row; GNU keeps the -c grand total
// recursive (coreutils 9.7 over a real mount: `du -bSc base` prints
// `3 base` then `10 total`).
const own = opts.separateDirs === true ? separateTotal(leaves, root) : sum
const lines: string[] = []
if (!opts.summarize) {
const rows = rollup(leaves, root, {
all: opts.all,
maxDepth: opts.maxDepth,
dirs: mountRoots,
separateDirs: opts.separateDirs === true,
})
const shown = respellRaw(
rows.map(([node]) => node),
@@ -143,7 +152,7 @@ export function mergeDuBlocks(
lines.push(`${formatSize(size, opts.human)}\t${shown[i] ?? ''}`)
})
}
lines.push(`${formatSize(sum, opts.human)}\t${label}`)
lines.push(`${formatSize(own, opts.human)}\t${label}`)
if (opts.total) lines.push(`${formatSize(sum, opts.human)}\ttotal`)
return ENC.encode(lines.join('\n') + '\n')
}
typescript/packages/core/src/commands/builtin/generic/du.test.ts
+98 -1
View File
@@ -22,6 +22,7 @@ import {
parseDuFlags,
rollup,
runDu,
separateTotal,
toVirtual,
} from './du.ts'
import { FileStat, FileType, PathSpec } from '../../../types.ts'
@@ -52,7 +53,7 @@ function opts(flags: Record<string, string | boolean> = {}): CommandOpts {
}
function flags(over: Partial<DuFlags> = {}): DuFlags {
return { s: false, a: false, h: false, c: false, maxDepth: null, ...over }
return { s: false, a: false, h: false, c: false, S: false, maxDepth: null, ...over }
}
/** Build (computeSize, computeEntries) over a mount-relative in-memory tree. */
@@ -102,6 +103,74 @@ describe('duGeneric', () => {
expect(DEC.decode(out.stdout)).toBe('1\t/dir/sub/deep\n3\t/dir/sub\n6\t/dir\n')
})
it('excludes subdirectory sizes under -S', async () => {
const [size, entries] = backend({
'/dir/a.txt': 3,
'/dir/sub/b.txt': 2,
'/dir/sub/deep/c.txt': 1,
})
const out = await duGeneric([spec('/dir', 'dir')], flags({ S: true }), size, entries)
expect(DEC.decode(out.stdout)).toBe('1\t/dir/sub/deep\n2\t/dir/sub\n3\t/dir\n')
})
it('summarises only direct files under -Ss', async () => {
const [size, entries] = backend({
'/dir/a.txt': 3,
'/dir/sub/b.txt': 2,
'/dir/sub/deep/c.txt': 1,
})
const out = await duGeneric([spec('/dir', 'dir')], flags({ s: true, S: true }), size, entries)
expect(DEC.decode(out.stdout)).toBe('3\t/dir\n')
})
it('lists files under -Sa with separate directory totals', async () => {
const [size, entries] = backend({
'/dir/a.txt': 3,
'/dir/sub/b.txt': 2,
'/dir/sub/deep/c.txt': 1,
})
const out = await duGeneric([spec('/dir', 'dir')], flags({ a: true, S: true }), size, entries)
expect(DEC.decode(out.stdout)).toBe(
'3\t/dir/a.txt\n2\t/dir/sub/b.txt\n1\t/dir/sub/deep/c.txt\n1\t/dir/sub/deep\n2\t/dir/sub\n3\t/dir\n',
)
})
it('keeps the -c grand total recursive under -S', async () => {
const [size, entries] = backend({
'/dir/a.txt': 3,
'/dir/sub/b.txt': 2,
'/dir/sub/deep/c.txt': 1,
})
const out = await duGeneric([spec('/dir', 'dir')], flags({ c: true, S: true }), size, entries)
expect(DEC.decode(out.stdout)).toBe('1\t/dir/sub/deep\n2\t/dir/sub\n3\t/dir\n6\ttotal\n')
})
it('keeps the -c grand total recursive under -Ss', async () => {
const [size, entries] = backend({
'/dir/a.txt': 3,
'/dir/sub/b.txt': 2,
'/dir/sub/deep/c.txt': 1,
})
const out = await duGeneric(
[spec('/dir', 'dir')],
flags({ s: true, c: true, S: true }),
size,
entries,
)
expect(DEC.decode(out.stdout)).toBe('3\t/dir\n6\ttotal\n')
})
it('keeps a file operand in the total under -S', async () => {
const [size, entries] = backend({ '/f.txt': 7 })
const out = await duGeneric(
[spec('/f.txt', 'f.txt')],
flags({ c: true, S: true }),
size,
entries,
)
expect(DEC.decode(out.stdout)).toBe('7\t/f.txt\n7\ttotal\n')
})
it('lists files then directories post-order under -a', async () => {
const [size, entries] = backend({
'/dir/a.txt': 3,
@@ -502,6 +571,34 @@ describe('rollup', () => {
expect(rows.get('/d/sub/deep')).toBe(1)
})
it('separateDirs counts only direct files', () => {
const entries: [string, number][] = [
['/d/a.txt', 3],
['/d/sub/b.txt', 2],
['/d/sub/deep/c.txt', 1],
]
const rows = new Map(rollup(entries, '/d', { all: false, maxDepth: null, separateDirs: true }))
expect(rows.get('/d/sub/deep')).toBe(1)
expect(rows.get('/d/sub')).toBe(2)
expect(rows.has('/d/a.txt')).toBe(false)
})
it('separateDirs keeps empty ancestor directories at size 0', () => {
const entries: [string, number][] = [['/d/sub/deep/c.txt', 4]]
const rows = new Map(rollup(entries, '/d', { all: false, maxDepth: null, separateDirs: true }))
expect(rows.get('/d/sub/deep')).toBe(4)
expect(rows.get('/d/sub')).toBe(0)
})
it('separateTotal sums only direct children', () => {
const entries: [string, number][] = [
['/d/a.txt', 3],
['/d/sub/b.txt', 2],
['/d/sub/deep/c.txt', 1],
]
expect(separateTotal(entries, '/d')).toBe(3)
})
it('keeps the sum over a directory marker under -a', () => {
const entries: [string, number][] = [
['/d/sub/deep/c.txt', 5],
typescript/packages/core/src/commands/builtin/generic/du.ts
+51 -9
View File
@@ -64,6 +64,8 @@ export interface DuFlags {
h: boolean
/** -c, append a grand total. */
c: boolean
/** -S/--separate-dirs, directories exclude subdirectory sizes. */
S: boolean
/** --max-depth/-d, deepest level to print. */
maxDepth: number | null
/** A non-fatal diagnostic GNU prints before the output. */
@@ -123,6 +125,7 @@ export function parseDuFlags(opts: CommandOpts): DuFlags {
a,
h: fl.asBool('h'),
c: fl.asBool('c'),
S: fl.asBool('separate_dirs'),
maxDepth,
...(warning === undefined ? {} : { warning }),
}
@@ -242,6 +245,18 @@ export function toVirtual(entries: [string, number][], path: PathSpec): [string,
return entries.map(([entry, size]) => [`${prefix}/${lstripSlash(entry)}`, size])
}
/**
* Sum of leaves whose parent is the operand (GNU `-S` total).
*/
export function separateTotal(entries: [string, number][], root: string): number {
const rootKey = norm(root)
let total = 0
for (const [leaf, size] of entries) {
if (parentOf(norm(leaf)) === rootKey) total += size
}
return total
}
/**
* Derive GNU's per-directory lines from a flat list of leaf files.
*
@@ -252,6 +267,9 @@ export function toVirtual(entries: [string, number][], path: PathSpec): [string,
* siblings sorted by name. GNU walks in readdir order, which is unspecified,
* so sorting is a deterministic choice within the same shape.
*
* With `-S`/`--separate-dirs` a directory only counts files that sit
* directly in it: a leaf still forces every ancestor directory to appear
* (possibly at size 0), but only the immediate parent gets its bytes.
* The operand's own line is not included; the caller renders it with the
* operand as typed.
*/
@@ -261,10 +279,16 @@ export function rollup(
// `dirs`: paths that are directories even though no leaf points at
// them. mirage cannot otherwise see an empty directory, so this is the
// one case it can: an empty mount still gets GNU's `0` row.
opts: { all: boolean; maxDepth: number | null; dirs?: readonly string[] },
opts: {
all: boolean
maxDepth: number | null
dirs?: readonly string[]
separateDirs?: boolean
},
): [string, number][] {
const rootKey = norm(root)
const prefix = rootKey.endsWith('/') ? rootKey : `${rootKey}/`
const separateDirs = opts.separateDirs === true
const sizes = new Map<string, number>()
const files = new Map<string, number>()
for (const [leaf, size] of entries) {
@@ -272,8 +296,17 @@ export function rollup(
if (node === rootKey || !node.startsWith(prefix)) continue
files.set(node, size)
let parent = parentOf(node)
let immediate = true
while (parent !== rootKey && parent.startsWith(prefix)) {
sizes.set(parent, (sizes.get(parent) ?? 0) + size)
if (separateDirs && !immediate) {
// -S: only the directory a file sits in counts its bytes. The
// ancestors still print, at 0 when they hold nothing but
// directories.
if (!sizes.has(parent)) sizes.set(parent, 0)
} else {
sizes.set(parent, (sizes.get(parent) ?? 0) + size)
}
immediate = false
parent = parentOf(parent)
}
}
@@ -378,7 +411,7 @@ async function duOne(
if (roots.length > 0) leaves = dropShadowed(leaves, roots)
const linkTotal = leaves.reduce((acc, [, size]) => acc + size, 0)
if (flags.s && roots.length === 0) {
if (flags.s && !flags.S && roots.length === 0) {
const total = (await computeSize(path)) + linkTotal
return [[`${fmt(total)}\t${label}`], total]
}
@@ -399,25 +432,34 @@ async function duOne(
entries = dropShadowed(entries, roots)
total = entries.reduce((acc, [, size]) => acc + size, 0)
}
if (flags.s) {
return [[`${fmt(total)}\t${label}`], total]
}
const rootKey = norm(path.virtual)
// A file operand walks to itself. GNU prints it once, with or without -a,
// never as a leaf line plus a roll-up line.
// never as a leaf line plus a roll-up line. GNU scopes -S to directories, so
// a file operand keeps its own size in both its row and the grand total.
const first = entries[0]
if (entries.length === 1 && first !== undefined && norm(first[0]) === rootKey) {
return [[`${fmt(first[1])}\t${label}`], total]
}
// -S changes what the operand's own row counts, not what the operand
// contributes to -c: GNU's grand total stays recursive (coreutils 9.7,
// `du -bSc dir` prints `3 dir` then `6 total`).
const own = flags.S ? separateTotal(entries, path.virtual) : total
if (flags.s) {
return [[`${fmt(own)}\t${label}`], total]
}
const rows = rollup(entries, path.virtual, { all: flags.a, maxDepth: flags.maxDepth })
const rows = rollup(entries, path.virtual, {
all: flags.a,
maxDepth: flags.maxDepth,
separateDirs: flags.S,
})
const shown = respellRaw(
rows.map(([p]) => p),
path.virtual,
label,
)
const lines = rows.map(([, size], i) => `${fmt(size)}\t${shown[i] ?? ''}`)
lines.push(`${fmt(total)}\t${label}`)
lines.push(`${fmt(own)}\t${label}`)
return [lines, total]
}
typescript/packages/core/src/commands/spec/builtin_specs/listing.ts
+1
View File
@@ -37,6 +37,7 @@ export const SPECS: Record<string, CommandSpec> = {
new Option({ short: '-c' }),
new Option({ short: '-L' }),
new Option({ short: '-P' }),
new Option({ short: '-S', long: '--separate-dirs' }),
],
rest: new Operand({ type: 'path' }),
}),
typescript/packages/core/src/workspace/executor/fanout.test.ts
+49
View File
@@ -27,6 +27,8 @@ import { basename } from '../../core/ram/utils.ts'
import { OpsRegistry } from '../../ops/registry.ts'
import { getTestParser, stdoutStr } from '../fixtures/workspace_fixture.ts'
import { Workspace } from '../workspace.ts'
import { specFlagNames } from '../../commands/spec/types.ts'
import { specOf } from '../../commands/spec/builtins.ts'
const NEVER_EXECUTE: ExecuteNodeFn = () => {
throw new Error('executeNode should not have been called')
@@ -295,6 +297,26 @@ describe('fanOutTraversal du at a descendant mount boundary', () => {
expect(await runLine('du -sc /base')).toBe('17\t/base\n17\ttotal\n')
})
// `-S` reaches the merge, and the `-c` total stays recursive. Pinned on
// coreutils 9.7 over a tmpfs mounted at the same spot: `du -bS base`
// prints `7 base/inner` then `10 base` (the parent counts only the file
// sitting in it), and `du -bSc base` still ends `17 total`.
it('scopes only the rows under -S', async () => {
expect(await runLine('du -S /base')).toBe('7\t/base/inner\n10\t/base\n')
expect(await runLine('du -Sc /base')).toBe('7\t/base/inner\n10\t/base\n17\ttotal\n')
})
it('summarises direct files only under -Ss', async () => {
expect(await runLine('du -Ss /base')).toBe('10\t/base\n')
expect(await runLine('du -Ssc /base')).toBe('10\t/base\n17\ttotal\n')
})
it('lists files under -Sa across the mounts', async () => {
expect(await runLine('du -Sa /base')).toBe(
'7\t/base/inner/real.txt\n7\t/base/inner\n10\t/base/top.txt\n10\t/base\n',
)
})
// Summing each mount's already-humanized total would round twice and
// report 3.0K; the sub-runs render exact bytes and only the merge
// humanizes.
@@ -376,6 +398,33 @@ describe('fanOutTraversal operands spanning mounts', () => {
)
})
// The du merge re-derives the whole tree centrally, so the sub-runs are
// asked with the presentation flags stripped and each one is then
// applied once, in the merge. A flag nobody classified is neither
// stripped nor re-applied, so it silently does nothing across a nested
// mount, which is exactly how -S first shipped. Adding an option to
// du's spec fails this until it is sorted into one of the two lists.
it('accounts for every du flag', () => {
// Applied centrally by the merge, and neutralized in the sub-runs.
const central = ['a', 'c', 'h', 'max_depth', 's', 'separate_dirs']
// Chooses whether a run counts the symlinks on its own mount, which
// is a per-run question; the merge only ever sees the rows.
const perRun = ['L', 'P']
expect([...specFlagNames(specOf('du'))].sort()).toEqual([...central, ...perRun].sort())
})
// -S has to survive both fan-outs at once: the per-operand one that
// splits the operands across mounts, and the traversal one that folds
// `/base/inner` into `/base`. GNU (coreutils 9.7, tmpfs at the nested
// spot) scopes -S to each printed row and keeps the grand total
// recursive, so `/base` reports only `top.txt` while the total still
// covers every byte.
it('keeps -S scoped to each row across both fan-outs', async () => {
expect(await runLine('du -Sc /base /other')).toBe(
'9\t/base/inner\n10\t/base\n10\t/other\n29\ttotal\n',
)
})
it('fans out inside each operand for find and grep -r', async () => {
const found = await runLine('find /base /other')
expect(found).toContain('/base/inner/real.txt')
typescript/packages/core/src/workspace/executor/fanout.ts
+2 -1
View File
@@ -342,12 +342,13 @@ export async function fanOutTraversal(
total: flagKwargs.c === true,
human: flagKwargs.h === true,
maxDepth: depthFlagValue(flagKwargs.max_depth ?? null),
separateDirs: flagKwargs.separate_dirs === true,
}
let flags = flagKwargs
if (duMerge) {
const rest = { ...flagKwargs }
delete rest.max_depth
flags = { ...rest, a: true, s: false, c: false, h: false }
flags = { ...rest, a: true, s: false, c: false, h: false, separate_dirs: false }
}
const allStdout: Uint8Array[] = []