diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 6ec8373ab..d48ce57e5 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -93,6 +93,14 @@ jobs: - name: Core barrel surface run: ./python/.venv/bin/python scripts/check_barrel_surface.py + # `normalizeFields` already maps every unlisted key through + # `snakeToCamel`, so a rename entry that restates that mapping is not + # configuration -- it is a second place for one fact to be wrong, and it + # buries the handful that are real overrides (`endpoint_url`, `timeout`, + # `aws_profile`) in a wall of noise. 95 had accumulated. + - name: Redundant config renames + run: ./python/.venv/bin/python scripts/check_normalize_renames.py + # The docs build parses frontmatter as YAML and stops on the first page # it cannot read, so an unquoted description holding ": " fails the # deploy with no local signal at all. diff --git a/python/mirage/accessor/github.py b/python/mirage/accessor/github.py index 2f378d6f2..5c33811fb 100644 --- a/python/mirage/accessor/github.py +++ b/python/mirage/accessor/github.py @@ -15,23 +15,27 @@ import asyncio from mirage.accessor.base import Accessor +from mirage.core.github.config import GitHubConfig from mirage.core.github.tree_entry import TreeEntry class GitHubAccessor(Accessor): def __init__(self, - config, - owner, - repo, - ref, + config: GitHubConfig, + owner: str, + repo: str, + ref: str | None = None, default_branch: str | None = None, tree: dict[str, TreeEntry] | None = None, - truncated=False): + truncated: bool = False) -> None: self.config = config self.owner = owner self.repo = repo - self.ref = ref + # None until resolved: an unpinned mount follows the repository's + # default branch, which costs a request to learn, so the ref is + # settled on the first read that needs one (`ensure_ref`). + self.ref: str | None = ref # None until hydrated: the mount is constructed without touching # the network, so the repo's default branch is fetched on the # first read that needs it (`ensure_default_branch`). diff --git a/python/mirage/commands/builtin/email/find.py b/python/mirage/commands/builtin/email/find.py index 11f6736a7..c1d797cfd 100644 --- a/python/mirage/commands/builtin/email/find.py +++ b/python/mirage/commands/builtin/email/find.py @@ -27,7 +27,7 @@ from mirage.commands.registry import command from mirage.commands.spec import SPECS from mirage.commands.spec.types import FlagView from mirage.core.email.client import fetch_headers -from mirage.core.email.readdir import _date_bucket, _sanitize +from mirage.core.email.readdir import _date_bucket, _msg_filename from mirage.core.email.readdir import readdir as _readdir from mirage.core.email.search import search_messages from mirage.core.email.stat import stat as _stat @@ -163,9 +163,12 @@ async def _find_server_side( results: list[str] = [] for h in headers: date_str = _date_bucket(h) - subject = _sanitize(h.get("subject", "No Subject")) uid = h.get("uid", "") - filename = f"{subject}__{uid}.email.json" + # The same builder readdir names the file with, not a second + # spelling of it: the subject's budget depends on the uid and the + # suffix, so a hit composed from a bare `_sanitize` pointed at a + # path that does not exist once a long subject was trimmed. + filename = _msg_filename(h.get("subject", "No Subject"), uid) if fnmatch(filename, name_pattern): vfs_path = "/".join(p for p in [prefix, folder, date_str, filename] diff --git a/python/mirage/commands/builtin/github/pushdown.py b/python/mirage/commands/builtin/github/pushdown.py index 185b3253a..cc6205a96 100644 --- a/python/mirage/commands/builtin/github/pushdown.py +++ b/python/mirage/commands/builtin/github/pushdown.py @@ -20,7 +20,7 @@ from mirage.commands.builtin.grep_helper import (is_literal_pattern, from mirage.core.github.constants import SCOPE_WARN from mirage.core.github.pushdown import (count_scope_files, scope_relative_key, should_use_search) -from mirage.core.github.repo import ensure_default_branch +from mirage.core.github.repo import ensure_default_branch, ensure_ref from mirage.core.github.search import narrow_paths from mirage.core.github.tree import ensure_tree from mirage.types import PathSpec @@ -87,7 +87,7 @@ async def narrow_scope( use_search = (query is not None and whole_word and literal and file_count > SCOPE_WARN and should_use_search( recursive=recursive, - on_default_branch=(accessor.ref == await + on_default_branch=(await ensure_ref(accessor) == await ensure_default_branch(accessor)), )) if use_search: diff --git a/python/mirage/core/email/readdir.py b/python/mirage/core/email/readdir.py index 3dfee2703..aa627113e 100644 --- a/python/mirage/core/email/readdir.py +++ b/python/mirage/core/email/readdir.py @@ -25,16 +25,22 @@ from mirage.core.email.render import message_json_bytes from mirage.types import PathSpec from mirage.utils.errors import enoent from mirage.utils.key_prefix import mount_key, mount_prefix_of -from mirage.utils.sanitize import sanitize_label +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label TITLE_MAX = 80 EPOCH_DATE = "1970-01-01" +MSG_SUFFIX = ".email.json" _sanitize = partial(sanitize_label, fallback="No_Subject", max_len=TITLE_MAX) def _msg_filename(subject: str, uid: str) -> str: - return f"{_sanitize(subject)}__{uid}.email.json" + # 80 characters is 240 bytes of CJK, which overflows the 255-byte + # NAME_MAX once the uid and `.email.json` are added, so the subject + # takes what they leave rather than a flat character count. + fixed = len("__") + byte_len(uid) + len(MSG_SUFFIX) + label = _sanitize(subject, max_bytes=NAME_MAX_BYTES - fixed) + return f"{label}__{uid}{MSG_SUFFIX}" def _parse_date(value: str) -> str | None: diff --git a/python/mirage/core/email/search.py b/python/mirage/core/email/search.py index 47f342d45..cc829fd20 100644 --- a/python/mirage/core/email/search.py +++ b/python/mirage/core/email/search.py @@ -16,7 +16,7 @@ from typing import Any from mirage.accessor.email import EmailAccessor from mirage.core.email.client import fetch_message, list_message_uids -from mirage.core.email.readdir import _date_bucket, _sanitize +from mirage.core.email.readdir import _date_bucket, _msg_filename from mirage.core.email.render import message_json_text from mirage.core.email.scope import EmailScope @@ -77,9 +77,12 @@ async def search_messages( def _build_vfs_path(prefix: str, folder: str, msg: dict[str, Any]) -> str: date_str = _date_bucket(msg) - subject = _sanitize(msg.get("subject", "No Subject")) uid = msg.get("uid", "") - filename = f"{subject}__{uid}.email.json" + # The same builder readdir names the file with, not a second spelling of + # it: the subject's budget depends on the uid and the suffix, so a hit + # composed here from a bare `_sanitize` pointed at a path that does not + # exist as soon as a long subject was trimmed differently. + filename = _msg_filename(msg.get("subject", "No Subject"), uid) parts = [prefix, folder, date_str, filename] return "/".join(p for p in parts if p) diff --git a/python/mirage/core/github/config.py b/python/mirage/core/github/config.py index 3e267d5f8..166930c86 100644 --- a/python/mirage/core/github/config.py +++ b/python/mirage/core/github/config.py @@ -19,7 +19,11 @@ class GitHubConfig(BaseModel): token: SecretStr owner: str | None = None repo: str | None = None - ref: str = "main" + # None means "whatever the repository's default branch is", resolved on + # the first read through ``ensure_ref``. It is not a synonym for "main": + # defaulting to that string mounted a nonexistent ref on every repo whose + # default is `master`, and the tree fetch 404s rather than falling back. + ref: str | None = None base_url: str | None = None diff --git a/python/mirage/core/github/readdir.py b/python/mirage/core/github/readdir.py index 268dad492..fcda79728 100644 --- a/python/mirage/core/github/readdir.py +++ b/python/mirage/core/github/readdir.py @@ -17,6 +17,7 @@ import logging from mirage.accessor.github import GitHubAccessor from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry, LookupStatus) +from mirage.core.github.repo import ensure_ref from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree, refill_index) from mirage.types import PathSpec @@ -120,7 +121,7 @@ async def _resolve_dir_sha( stem = prefix.rstrip("/") rest = norm[len(stem):] if stem and norm.startswith(stem) else norm parts = [p for p in rest.strip("/").split("/") if p] - current_sha = accessor.ref + current_sha = await ensure_ref(accessor) current_path = stem for part in parts: entries = await fetch_dir_tree(accessor.config, accessor.owner, diff --git a/python/mirage/core/github/repo.py b/python/mirage/core/github/repo.py index 04fcf98dc..92d03924a 100644 --- a/python/mirage/core/github/repo.py +++ b/python/mirage/core/github/repo.py @@ -61,6 +61,34 @@ async def ensure_default_branch(accessor: GitHubAccessor, ) -> str: return accessor.default_branch +async def ensure_ref(accessor: GitHubAccessor) -> str: + """Settle which ref this mount reads, fetching the default branch once. + + A mount that named no ref follows the repository's default branch, and + learning that costs a request the constructor cannot make. Every reader + that needs a concrete ref -- the tree fetches, the watch walk, readdir's + per-directory descent -- goes through here instead of reading + ``accessor.ref`` directly, so an unpinned mount resolves exactly once and + then behaves like a pinned one. + + Defaulting to the string ``"main"`` instead was the bug this replaces: a + repository whose default branch is ``master`` (or anything else) 404s on + every tree fetch, so the whole mount reads as empty. + + Args: + accessor (GitHubAccessor): the mount's accessor. + + Returns: + str: the ref to read, as named by the mount or as resolved from the + repository's default branch. + """ + if accessor.ref is not None: + return accessor.ref + resolved = await ensure_default_branch(accessor) + accessor.ref = resolved + return resolved + + def parse_repo(spec: str) -> RepoRef: """Split gh's `[HOST/]OWNER/REPO`. diff --git a/python/mirage/core/github/tree.py b/python/mirage/core/github/tree.py index 03a38e465..e6753d798 100644 --- a/python/mirage/core/github/tree.py +++ b/python/mirage/core/github/tree.py @@ -22,6 +22,7 @@ from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry, LookupStatus) from mirage.core.github.client import github_get from mirage.core.github.config import GitHubConfig +from mirage.core.github.repo import ensure_ref from mirage.core.github.tree_entry import TreeEntry log = logging.getLogger(__name__) @@ -201,8 +202,9 @@ async def refill_index( """ if index is NULL_INDEX: return False + ref = await ensure_ref(accessor) tree, truncated = await fetch_tree(accessor.config, accessor.owner, - accessor.repo, accessor.ref) + accessor.repo, ref) accessor.truncated = truncated accessor.tree = tree accessor.tree_loaded = True @@ -301,8 +303,9 @@ async def ensure_tree( await ensure_live_index(accessor, index, prefix) if accessor.tree_loaded: return + ref = await ensure_ref(accessor) tree, truncated = await fetch_tree(accessor.config, accessor.owner, - accessor.repo, accessor.ref) + accessor.repo, ref) accessor.truncated = truncated accessor.tree = tree accessor.tree_loaded = True diff --git a/python/mirage/core/github/watch.py b/python/mirage/core/github/watch.py index 340f065c4..b959bd229 100644 --- a/python/mirage/core/github/watch.py +++ b/python/mirage/core/github/watch.py @@ -15,6 +15,7 @@ from collections.abc import AsyncIterator from mirage.accessor.github import GitHubAccessor +from mirage.core.github.repo import ensure_ref from mirage.core.github.tree import fetch_tree from mirage.types import PathSpec, WalkEntry from mirage.utils.key_prefix import mount_prefix_of @@ -58,12 +59,13 @@ class GitHubWalk: """ accessor = self._accessor prefix = mount_prefix_of(root.virtual, root.resource_path) + ref = await ensure_ref(accessor) tree, truncated = await fetch_tree(accessor.config, accessor.owner, - accessor.repo, accessor.ref) + accessor.repo, ref) if truncated: raise IncompleteWalkError( f"github tree for {accessor.owner}/{accessor.repo}" - f"@{accessor.ref} was truncated; cannot diff a partial tree") + f"@{ref} was truncated; cannot diff a partial tree") # A complete tree for the ref is exactly what the accessor holds, # and find/du/grep's scope counter read it directly. Discarding it # here left them answering from the tree the mount was built with diff --git a/python/mirage/core/gmail/readdir.py b/python/mirage/core/gmail/readdir.py index 4eb7a7f03..36f7202a9 100644 --- a/python/mirage/core/gmail/readdir.py +++ b/python/mirage/core/gmail/readdir.py @@ -27,21 +27,43 @@ from mirage.core.gmail.messages import (_extract_attachments, _extract_header, from mirage.types import PathSpec from mirage.utils.errors import enoent from mirage.utils.key_prefix import mount_key, mount_prefix_of -from mirage.utils.sanitize import sanitize_label +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label logger = logging.getLogger(__name__) TITLE_MAX = 80 +MSG_SUFFIX = ".gmail.json" _sanitize = partial(sanitize_label, fallback="No_Subject", max_len=TITLE_MAX) +def _subject(subject: str, msg_id: str) -> str: + """Sanitize a subject to fit what the id and suffix leave of NAME_MAX. + + 80 characters is 240 bytes of CJK, which overflows the 255-byte + NAME_MAX once the id and `.gmail.json` are added; the filesystem + rejects the name outright. Both the message file and its attachment + directory take the file's (stricter) budget so one subject renders + the same in both, rather than the directory getting the eleven bytes + the suffix would have used. + + Args: + subject (str): raw subject header. + msg_id (str): the Gmail message id the name embeds. + + Returns: + str: the sanitized subject segment. + """ + fixed = len("__") + byte_len(msg_id) + len(MSG_SUFFIX) + return _sanitize(subject, max_bytes=NAME_MAX_BYTES - fixed) + + def _msg_filename(subject: str, msg_id: str) -> str: - return f"{_sanitize(subject)}__{msg_id}.gmail.json" + return f"{_subject(subject, msg_id)}__{msg_id}{MSG_SUFFIX}" def _attach_dir_name(subject: str, msg_id: str) -> str: - return f"{_sanitize(subject)}__{msg_id}" + return f"{_subject(subject, msg_id)}__{msg_id}" def _attachment_filename(_attachment_id: str, filename: str) -> str: diff --git a/python/mirage/core/gmail/search.py b/python/mirage/core/gmail/search.py index 1d58adbcb..1674fe69d 100644 --- a/python/mirage/core/gmail/search.py +++ b/python/mirage/core/gmail/search.py @@ -18,7 +18,7 @@ from typing import Any from mirage.core.gmail.messages import (_decode_body, _extract_header, get_message_processed, get_message_raw, list_messages) -from mirage.core.gmail.readdir import _sanitize +from mirage.core.gmail.readdir import _msg_filename from mirage.core.gmail.scope import GmailScope from mirage.core.google.client import TokenManager @@ -116,8 +116,11 @@ def format_grep_results( label = row.get("label") or scope.label_name or "INBOX" date = row.get("date", "") mid = row.get("id", "") - subject_clean = _sanitize(row.get("subject") or "No Subject") - filename = f"{subject_clean}__{mid}.gmail.json" + # The same builder readdir names the file with, not a second + # spelling of it: the subject's budget depends on the id and the + # suffix, so a hit composed from a bare `_sanitize` pointed at a + # path that does not exist once a long subject was trimmed. + filename = _msg_filename(row.get("subject") or "No Subject", mid) sender = row.get("sender", "?") haystack = f"{row.get('subject', '')}\n{row.get('body_text', '')}" excerpt = _extract_excerpt(haystack, pattern) if pattern else "" diff --git a/python/mirage/resource/gcal/event_entry.py b/python/mirage/resource/gcal/event_entry.py index e90d71cc0..161f4bab6 100644 --- a/python/mirage/resource/gcal/event_entry.py +++ b/python/mirage/resource/gcal/event_entry.py @@ -13,7 +13,8 @@ # ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= from mirage.utils.naming import make_id_name -from mirage.utils.sanitize import NAME_MAX_BYTES, sanitize_name, truncate_bytes +from mirage.utils.sanitize import (NAME_MAX_BYTES, byte_len, sanitize_name, + truncate_bytes) EVENT_SUFFIX = ".gcal.json" CALENDAR_FILE = "calendar.json" @@ -58,7 +59,7 @@ def make_event_filename(event_id: str, hhmm: str, title: str) -> str: Returns: str: e.g. ``la9i1t9...__0900-1030_PhD_Defense.gcal.json``. """ - fixed = len(event_id.encode()) + len("__") + len(hhmm) + len("_") + len( + fixed = byte_len(event_id) + len("__") + len(hhmm) + len("_") + len( EVENT_SUFFIX) trimmed = truncate_bytes(title, NAME_MAX_BYTES - fixed).rstrip("_") if not trimmed: diff --git a/python/mirage/resource/gdocs/doc_entry.py b/python/mirage/resource/gdocs/doc_entry.py index 215e5a277..f6a9cfa80 100644 --- a/python/mirage/resource/gdocs/doc_entry.py +++ b/python/mirage/resource/gdocs/doc_entry.py @@ -15,9 +15,11 @@ from dataclasses import dataclass from functools import partial -from mirage.utils.sanitize import sanitize_label +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label TITLE_MAX_CHARS = 100 +SUFFIX = ".gdoc.json" +DATE_LEN = 10 @dataclass @@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label, def make_filename(title: str, doc_id: str, modified_time: str = "") -> str: """Build a filename from title, doc ID, and modified date. + The title takes whatever of the 255-byte NAME_MAX the date, the id and + the suffix leave, rather than a flat character count: those are the same + number only for ASCII, and a 100-character CJK title rendered a name ext4 + and APFS reject outright. The id never gives, so the name keeps + addressing the document -- same rule as gcal's event filenames. + Args: title (str): raw document title. doc_id (str): Google Docs document ID. @@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str: Returns: str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json". """ - date_prefix = modified_time[:10] if len(modified_time) >= 10 else "" - if date_prefix: - return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gdoc.json" - return f"{sanitize_title(title)}__{doc_id}.gdoc.json" + lead = (f"{modified_time[:DATE_LEN]}_" + if len(modified_time) >= DATE_LEN else "") + fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX) + label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed) + return f"{lead}{label}__{doc_id}{SUFFIX}" diff --git a/python/mirage/resource/github/github.py b/python/mirage/resource/github/github.py index 65a7e1c0d..6afae5a4a 100644 --- a/python/mirage/resource/github/github.py +++ b/python/mirage/resource/github/github.py @@ -81,7 +81,9 @@ class GitHubResource(BaseResource): repo (str | None): repository name; falls back to ``config.repo``. ref (str | None): branch, tag or commit the mount is pinned - to; falls back to ``config.ref``. + to; falls back to ``config.ref``, and when neither names + one the mount follows the repository's default branch, + resolved on first read by ``ensure_ref``. default_branch (str | None): the repo's default branch, for ``is_default_branch``. Fetched on first use when None. tree (dict[str, TreeEntry] | None): the recursive git tree, @@ -130,8 +132,12 @@ class GitHubResource(BaseResource): def is_default_branch(self) -> bool | None: """Whether the mount is pinned to the repo's default branch. - ``None`` means not known yet, not "no": the default branch is - fetched on first use, and until something calls + An unpinned mount answers True without a request: naming no ref + *means* following the default branch, so the two agree whatever + that branch turns out to be. + + Otherwise ``None`` means not known yet, not "no": the default + branch is fetched on first use, and until something calls :func:`mirage.core.github.repo.ensure_default_branch` there is nothing to compare ``ref`` against. Answering ``False`` there would be a wrong answer rather than an absent one, and an @@ -146,6 +152,8 @@ class GitHubResource(BaseResource): Returns: bool | None: the comparison, or None if not yet hydrated. """ + if self.accessor.ref is None: + return True if self.accessor.default_branch is None: return None return self.accessor.ref == self.accessor.default_branch diff --git a/python/mirage/resource/gsheets/sheet_entry.py b/python/mirage/resource/gsheets/sheet_entry.py index 36ce14498..cccf68349 100644 --- a/python/mirage/resource/gsheets/sheet_entry.py +++ b/python/mirage/resource/gsheets/sheet_entry.py @@ -15,9 +15,11 @@ from dataclasses import dataclass from functools import partial -from mirage.utils.sanitize import sanitize_label +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label TITLE_MAX_CHARS = 100 +SUFFIX = ".gsheet.json" +DATE_LEN = 10 @dataclass @@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label, def make_filename(title: str, doc_id: str, modified_time: str = "") -> str: """Build a filename from title, doc ID, and modified date. + The title takes whatever of the 255-byte NAME_MAX the date, the id and + the suffix leave, rather than a flat character count: those are the same + number only for ASCII, and a 100-character CJK title rendered a name ext4 + and APFS reject outright. The id never gives, so the name keeps + addressing the document -- same rule as gcal's event filenames. + Args: title (str): raw document title. doc_id (str): Google Sheets spreadsheet ID. @@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str: Returns: str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json". """ - date_prefix = modified_time[:10] if len(modified_time) >= 10 else "" - if date_prefix: - return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gsheet.json" - return f"{sanitize_title(title)}__{doc_id}.gsheet.json" + lead = (f"{modified_time[:DATE_LEN]}_" + if len(modified_time) >= DATE_LEN else "") + fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX) + label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed) + return f"{lead}{label}__{doc_id}{SUFFIX}" diff --git a/python/mirage/resource/gslides/slide_entry.py b/python/mirage/resource/gslides/slide_entry.py index 9ea5e02a4..8138183b4 100644 --- a/python/mirage/resource/gslides/slide_entry.py +++ b/python/mirage/resource/gslides/slide_entry.py @@ -15,9 +15,11 @@ from dataclasses import dataclass from functools import partial -from mirage.utils.sanitize import sanitize_label +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label TITLE_MAX_CHARS = 100 +SUFFIX = ".gslide.json" +DATE_LEN = 10 @dataclass @@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label, def make_filename(title: str, doc_id: str, modified_time: str = "") -> str: """Build a filename from title, doc ID, and modified date. + The title takes whatever of the 255-byte NAME_MAX the date, the id and + the suffix leave, rather than a flat character count: those are the same + number only for ASCII, and a 100-character CJK title rendered a name ext4 + and APFS reject outright. The id never gives, so the name keeps + addressing the document -- same rule as gcal's event filenames. + Args: title (str): raw document title. doc_id (str): Google Slides presentation ID. @@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str: Returns: str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json". """ - date_prefix = modified_time[:10] if len(modified_time) >= 10 else "" - if date_prefix: - return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gslide.json" - return f"{sanitize_title(title)}__{doc_id}.gslide.json" + lead = (f"{modified_time[:DATE_LEN]}_" + if len(modified_time) >= DATE_LEN else "") + fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX) + label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed) + return f"{lead}{label}__{doc_id}{SUFFIX}" diff --git a/python/mirage/utils/sanitize.py b/python/mirage/utils/sanitize.py index 3f49818e9..c0e065f93 100644 --- a/python/mirage/utils/sanitize.py +++ b/python/mirage/utils/sanitize.py @@ -21,6 +21,19 @@ MAX_LEN = 100 # characters is the same number only for ASCII: a 100-character CJK title is # 300 bytes. NAME_MAX_BYTES = 255 +ELLIPSIS = "..." + + +def byte_len(text: str) -> int: + """Measure a string the way the filesystem does. + + Args: + text (str): the string to measure. + + Returns: + int: the length of ``text`` in UTF-8 bytes. + """ + return len(text.encode("utf-8")) def truncate_bytes(text: str, budget: int) -> str: @@ -88,7 +101,11 @@ def path_safe_name(name: str) -> str: return name.replace("/", "∕") -def sanitize_label(text: str, *, fallback: str, max_len: int) -> str: +def sanitize_label(text: str, + *, + fallback: str, + max_len: int, + max_bytes: int = NAME_MAX_BYTES) -> str: """Sanitize an API-supplied label for use inside a filename. The shared body behind every backend's title/subject sanitizer: @@ -100,11 +117,24 @@ def sanitize_label(text: str, *, fallback: str, max_len: int) -> str: Unlike ``sanitize_name`` this ellipsizes rather than hard-cutting, so a truncated name reads as truncated. + Two budgets apply, and both have to: ``max_len`` is the readable + length a backend wants, while ``max_bytes`` is what the filesystem + will actually accept. They are the same number only for ASCII, so a + 100-character CJK title passed a 100-character budget untouched and + rendered a 300-byte filename, which ext4 and APFS reject with + ENAMETOOLONG. Pass the bytes the *rest* of the filename does not + already use -- see ``make_filename`` in the gdocs/gsheets/gslides + entries and ``make_event_filename`` in gcal, which is where the + fixed overhead is known. + Args: text (str): raw label from the API. fallback (str): what an empty or whitespace-only label becomes. max_len (int): budget in characters; a longer label keeps its first ``max_len - 3`` characters plus an ellipsis. + max_bytes (int): budget in UTF-8 bytes for the label alone. A + budget too small to hold even the ellipsis yields a bare + truncation rather than three dots and nothing. Returns: str: the sanitized label. @@ -114,5 +144,12 @@ def sanitize_label(text: str, *, fallback: str, max_len: int) -> str: cleaned = UNSAFE_CHARS.sub("_", text).replace(" ", "_") cleaned = MULTI_UNDERSCORE.sub("_", cleaned).strip("_") if len(cleaned) > max_len: - cleaned = cleaned[:max_len - 3] + "..." + cleaned = cleaned[:max_len - len(ELLIPSIS)] + ELLIPSIS + if byte_len(cleaned) > max_bytes: + head = truncate_bytes(cleaned, max(max_bytes - len(ELLIPSIS), 0)) + trimmed = head.rstrip("_.") + if trimmed: + cleaned = trimmed + ELLIPSIS + else: + cleaned = truncate_bytes(cleaned, max_bytes) return cleaned diff --git a/python/pyproject.toml b/python/pyproject.toml index 10f1c8ebe..29e7a93a6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -311,7 +311,6 @@ module = [ "mirage.accessor._hf", "mirage.accessor.gdocs", "mirage.accessor.gdrive", - "mirage.accessor.github", "mirage.accessor.gmail", "mirage.accessor.gsheets", "mirage.accessor.gslides", diff --git a/python/tests/core/email/test_search.py b/python/tests/core/email/test_search.py new file mode 100644 index 000000000..609f88b68 --- /dev/null +++ b/python/tests/core/email/test_search.py @@ -0,0 +1,44 @@ +# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +from mirage.core.email.readdir import _msg_filename +from mirage.core.email.search import _build_vfs_path +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len + +CJK_SUBJECT = "会議の記録" * 40 + + +def test_a_hit_names_the_file_readdir_created(): + # Composed here from a bare `_sanitize`, a hit pointed at a path that + # does not exist as soon as the subject was long enough to be trimmed: + # readdir budgets the subject against the uid and the suffix, and this + # did not, so the two names differed. + msg = { + "subject": CJK_SUBJECT, + "uid": "7", + "date": "Mon, 5 Jan 2026 10:00:00 +0000" + } + path = _build_vfs_path("/mail", "INBOX", msg) + assert path.endswith("/" + _msg_filename(CJK_SUBJECT, "7")) + + +def test_a_hits_filename_fits_name_max(): + msg = { + "subject": CJK_SUBJECT, + "uid": "7", + "date": "Mon, 5 Jan 2026 10:00:00 +0000" + } + name = _build_vfs_path("/mail", "INBOX", msg).rsplit("/", 1)[-1] + assert byte_len(name) <= NAME_MAX_BYTES + assert "\ufffd" not in name diff --git a/python/tests/core/github/test_repo.py b/python/tests/core/github/test_repo.py index df7ded6b4..565cebef7 100644 --- a/python/tests/core/github/test_repo.py +++ b/python/tests/core/github/test_repo.py @@ -12,12 +12,15 @@ # limitations under the License. # ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +import asyncio from unittest.mock import patch import pytest +from mirage.accessor.github import GitHubAccessor from mirage.core.github.config import GitHubConfig -from mirage.core.github.repo import fetch_default_branch, parse_repo +from mirage.core.github.repo import (ensure_ref, fetch_default_branch, + parse_repo) @pytest.fixture @@ -64,3 +67,35 @@ def test_parse_repo_drops_the_optional_host(): def test_parse_repo_refuses_a_spec_that_is_not_the_format(spec): with pytest.raises(ValueError, match="OWNER/REPO"): parse_repo(spec) + + +@pytest.mark.asyncio +@patch("mirage.core.github.repo.github_get") +async def test_ensure_ref_resolves_the_default_branch_when_none_was_named( + mock_get, config): + mock_get.return_value = {"default_branch": "master"} + accessor = GitHubAccessor(config, "acme", "proj") + assert await ensure_ref(accessor) == "master" + # Settled on the accessor, so the next reader neither refetches nor + # disagrees with the ref this one read. + assert accessor.ref == "master" + + +@pytest.mark.asyncio +@patch("mirage.core.github.repo.github_get") +async def test_ensure_ref_keeps_a_pinned_ref_without_a_request( + mock_get, config): + accessor = GitHubAccessor(config, "acme", "proj", "release-2") + assert await ensure_ref(accessor) == "release-2" + mock_get.assert_not_awaited() + + +@pytest.mark.asyncio +@patch("mirage.core.github.repo.github_get") +async def test_ensure_ref_resolves_once_for_concurrent_readers( + mock_get, config): + mock_get.return_value = {"default_branch": "trunk"} + accessor = GitHubAccessor(config, "acme", "proj") + refs = await asyncio.gather(*(ensure_ref(accessor) for _ in range(4))) + assert refs == ["trunk"] * 4 + assert mock_get.await_count == 1 diff --git a/python/tests/core/github/test_tree.py b/python/tests/core/github/test_tree.py index 15190a56d..a53587586 100644 --- a/python/tests/core/github/test_tree.py +++ b/python/tests/core/github/test_tree.py @@ -21,8 +21,8 @@ from mirage.accessor.github import GitHubAccessor from mirage.cache.index import NULL_INDEX from mirage.cache.index.ram import RAMIndexCacheStore from mirage.core.github.config import GitHubConfig -from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree, - fetch_tree, index_rows) +from mirage.core.github.tree import (ensure_live_index, ensure_tree, + fetch_dir_tree, fetch_tree, index_rows) from mirage.core.github.tree_entry import TreeEntry @@ -251,3 +251,35 @@ def test_index_rows_root_mount_keeps_bare_paths(): def test_index_rows_gives_an_empty_repo_a_root_row(): _entries, children = index_rows({}, "/gh") assert children == {"/gh": []} + + +@pytest.mark.asyncio +@patch("mirage.core.github.repo.github_get") +@patch("mirage.core.github.tree.github_get") +async def test_an_unpinned_mount_reads_the_repos_default_branch( + mock_tree_get, mock_repo_get, config): + """An unresolved ref must be settled before the tree is fetched. + + ``accessor.ref`` is None until something resolves it, so reading it + straight sends `ref=None` to the one request the whole mount is built + on. This pins the resolution, not the config default -- the mount that + supplies the default lives a layer up, in + tests/resource/github/test_lazy_hydration.py. + """ + mock_repo_get.return_value = {"default_branch": "master"} + mock_tree_get.return_value = {"truncated": False, "tree": []} + accessor = GitHubAccessor(config, "acme", "proj") + await ensure_tree(accessor) + assert mock_tree_get.await_args.kwargs["ref"] == "master" + + +@pytest.mark.asyncio +@patch("mirage.core.github.repo.github_get") +@patch("mirage.core.github.tree.github_get") +async def test_a_pinned_mount_reads_its_ref_and_never_asks_for_the_branch( + mock_tree_get, mock_repo_get, config): + mock_tree_get.return_value = {"truncated": False, "tree": []} + accessor = GitHubAccessor(config, "acme", "proj", "release-2") + await ensure_tree(accessor) + assert mock_tree_get.await_args.kwargs["ref"] == "release-2" + mock_repo_get.assert_not_awaited() diff --git a/python/tests/resource/gcal/test_event_entry.py b/python/tests/resource/gcal/test_event_entry.py index c17fd556e..6cfdf4c3f 100644 --- a/python/tests/resource/gcal/test_event_entry.py +++ b/python/tests/resource/gcal/test_event_entry.py @@ -52,7 +52,8 @@ def test_long_ascii_title_is_trimmed_to_name_max(): def test_long_cjk_title_is_trimmed_by_bytes_not_characters(): # 3 bytes per character: a character-counted budget would overflow - # NAME_MAX, which is the bug gdocs' sanitize_title still has. + # NAME_MAX. gdocs/gsheets/gslides had exactly that bug until + # sanitize_label grew a byte budget. name = make_event_filename(EVENT_ID, "0900-1030", "会" * 200) raw = name.encode() assert len(raw) <= NAME_MAX_BYTES diff --git a/python/tests/resource/gdocs/test_doc_entry.py b/python/tests/resource/gdocs/test_doc_entry.py index 819247341..79a3ae252 100644 --- a/python/tests/resource/gdocs/test_doc_entry.py +++ b/python/tests/resource/gdocs/test_doc_entry.py @@ -14,6 +14,7 @@ from mirage.resource.gdocs.doc_entry import (DocEntry, make_filename, sanitize_title) +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len TITLE_MAX = 100 @@ -81,3 +82,34 @@ def test_make_filename_duplicate_titles_different_dates(): assert f1 != f2 assert f1 == "2026-04-01_My_Doc__abc123.gdoc.json" assert f2 == "2026-03-15_My_Doc__def456.gdoc.json" + + +# A real Google file id is 44 characters, so this is the fixed overhead a +# title actually has to fit inside. +DOC_ID = "1" * 44 + + +def test_make_filename_fits_name_max_for_a_cjk_title(): + # 100 characters of CJK is 300 bytes, which the character budget passed + # untouched: with the date, the id and the suffix the name came to 367 + # bytes and ext4/APFS reject it with ENAMETOOLONG. + name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z") + assert byte_len(name) <= NAME_MAX_BYTES + assert name.startswith("2026-08-20_") + assert name.endswith(f"__{DOC_ID}.gdoc.json") + # The cut lands on a character boundary, never mid-sequence. + assert "\ufffd" not in name + + +def test_make_filename_leaves_an_ascii_title_on_the_char_budget(): + name = make_filename("a" * 400, DOC_ID, "") + assert byte_len(name) <= NAME_MAX_BYTES + assert name == f"{'a' * 97}...__{DOC_ID}.gdoc.json" + + +def test_make_filename_keeps_the_id_when_it_leaves_no_room(): + # The title is what gives, never the id: a trimmed id would stop + # addressing the document. Same rule as gcal's event filenames. + long_id = "v" * (NAME_MAX_BYTES - 4) + name = make_filename("Some Title", long_id, "") + assert f"__{long_id}.gdoc.json" in name diff --git a/python/tests/resource/github/test_config.py b/python/tests/resource/github/test_config.py index 3834f6239..8a01b2fdd 100644 --- a/python/tests/resource/github/test_config.py +++ b/python/tests/resource/github/test_config.py @@ -34,7 +34,10 @@ def test_github_config_owner_repo_ref_default(): cfg = GitHubConfig(token="ghp_abc123") assert cfg.owner is None assert cfg.repo is None - assert cfg.ref == "main" + # Not "main": that guess 404s the tree fetch on every repository whose + # default branch is something else. None means "resolve the default + # branch", which `ensure_ref` does on the first read. + assert cfg.ref is None def test_github_config_accepts_owner_repo_ref(): diff --git a/python/tests/resource/github/test_lazy_hydration.py b/python/tests/resource/github/test_lazy_hydration.py index 9b681f091..e93a2a43d 100644 --- a/python/tests/resource/github/test_lazy_hydration.py +++ b/python/tests/resource/github/test_lazy_hydration.py @@ -85,3 +85,37 @@ async def test_a_tree_passed_to_the_constructor_counts_as_hydrated(tree_calls): resource = GitHubResource(CONFIG, "o", "r", "main", tree=dict(TREE)) await ensure_tree(resource.accessor) assert tree_calls == [] + + +@pytest.fixture +def default_branch(monkeypatch): + + async def _fetch(config, owner, repo): + return "master" + + monkeypatch.setattr("mirage.core.github.repo.fetch_default_branch", _fetch) + + +@pytest.mark.asyncio +async def test_a_mount_naming_no_ref_reads_the_repos_default_branch( + tree_calls, default_branch): + # The config used to default `ref` to the literal string "main", so a + # repository whose default branch is anything else 404d on the one + # request the whole mount is built on and the mount read as empty. + resource = GitHubResource( + GitHubConfig(token="ghp_test", owner="o", repo="r")) + assert resource.accessor.ref is None + await ensure_tree(resource.accessor) + assert tree_calls == [("o", "r", "master")] + + +@pytest.mark.asyncio +async def test_an_unpinned_mount_is_on_the_default_branch_before_any_fetch( + tree_calls): + # Naming no ref *means* following the default branch, so the two agree + # whatever it turns out to be -- no request, and not the "not known + # yet" None a pinned mount answers. + resource = GitHubResource( + GitHubConfig(token="ghp_test", owner="o", repo="r")) + assert resource.is_default_branch is True + assert GitHubResource(CONFIG, "o", "r", "dev").is_default_branch is None diff --git a/python/tests/resource/gsheets/test_sheet_entry.py b/python/tests/resource/gsheets/test_sheet_entry.py new file mode 100644 index 000000000..76d1f9cec --- /dev/null +++ b/python/tests/resource/gsheets/test_sheet_entry.py @@ -0,0 +1,77 @@ +# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +from mirage.resource.gsheets.sheet_entry import (SheetEntry, make_filename, + sanitize_title) +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len + +# A real Google file id is 44 characters, so this is the fixed overhead a +# title actually has to fit inside. +DOC_ID = "1" * 44 + + +def test_sheet_entry_creation(): + entry = SheetEntry( + id="abc123", + name="My Spreadsheet", + modified_time="2026-04-01T12:00:00.000Z", + created_time="2026-03-01T12:00:00.000Z", + owner="user@gmail.com", + owned_by_me=True, + can_edit=True, + filename="My_Spreadsheet__abc123.gsheet.json", + ) + assert entry.id == "abc123" + assert entry.owned_by_me is True + assert entry.can_edit is True + + +def test_sanitize_title_basic(): + assert sanitize_title("Hello World") == "Hello_World" + assert sanitize_title("My/Doc: A\\Test") == "My_Doc_A_Test" + assert sanitize_title("") == "Untitled" + + +def test_make_filename_with_and_without_a_date(): + assert make_filename("My Spreadsheet", "abc123", + "2026-03-15T10:00:00Z") == \ + "2026-03-15_My_Spreadsheet__abc123.gsheet.json" + assert make_filename("My Spreadsheet", + "abc123") == "My_Spreadsheet__abc123.gsheet.json" + + +def test_make_filename_fits_name_max_for_a_cjk_title(): + # 100 characters of CJK is 300 bytes, which the character budget passed + # untouched: with the date, the id and the suffix the name came to 367 + # bytes and ext4/APFS reject it with ENAMETOOLONG. + name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z") + assert byte_len(name) <= NAME_MAX_BYTES + assert name.startswith("2026-08-20_") + assert name.endswith(f"__{DOC_ID}.gsheet.json") + # The cut lands on a character boundary, never mid-sequence. + assert "\ufffd" not in name + + +def test_make_filename_leaves_an_ascii_title_on_the_char_budget(): + name = make_filename("a" * 400, DOC_ID, "") + assert byte_len(name) <= NAME_MAX_BYTES + assert name == f"{'a' * 97}...__{DOC_ID}.gsheet.json" + + +def test_make_filename_keeps_the_id_when_it_leaves_no_room(): + # The title is what gives, never the id: a trimmed id would stop + # addressing the spreadsheet. Same rule as gcal's event filenames. + long_id = "v" * (NAME_MAX_BYTES - 4) + name = make_filename("Some Title", long_id, "") + assert f"__{long_id}.gsheet.json" in name diff --git a/python/tests/resource/gslides/test_slide_entry.py b/python/tests/resource/gslides/test_slide_entry.py new file mode 100644 index 000000000..0feca1dd8 --- /dev/null +++ b/python/tests/resource/gslides/test_slide_entry.py @@ -0,0 +1,77 @@ +# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +from mirage.resource.gslides.slide_entry import (SlideEntry, make_filename, + sanitize_title) +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len + +# A real Google file id is 44 characters, so this is the fixed overhead a +# title actually has to fit inside. +DOC_ID = "1" * 44 + + +def test_slide_entry_creation(): + entry = SlideEntry( + id="abc123", + name="My Presentation", + modified_time="2026-04-01T12:00:00.000Z", + created_time="2026-03-01T12:00:00.000Z", + owner="user@gmail.com", + owned_by_me=True, + can_edit=True, + filename="My_Presentation__abc123.gslide.json", + ) + assert entry.id == "abc123" + assert entry.owned_by_me is True + assert entry.can_edit is True + + +def test_sanitize_title_basic(): + assert sanitize_title("Hello World") == "Hello_World" + assert sanitize_title("My/Doc: A\\Test") == "My_Doc_A_Test" + assert sanitize_title("") == "Untitled" + + +def test_make_filename_with_and_without_a_date(): + assert make_filename("My Presentation", "abc123", + "2026-03-15T10:00:00Z") == \ + "2026-03-15_My_Presentation__abc123.gslide.json" + assert make_filename("My Presentation", + "abc123") == "My_Presentation__abc123.gslide.json" + + +def test_make_filename_fits_name_max_for_a_cjk_title(): + # 100 characters of CJK is 300 bytes, which the character budget passed + # untouched: with the date, the id and the suffix the name came to 367 + # bytes and ext4/APFS reject it with ENAMETOOLONG. + name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z") + assert byte_len(name) <= NAME_MAX_BYTES + assert name.startswith("2026-08-20_") + assert name.endswith(f"__{DOC_ID}.gslide.json") + # The cut lands on a character boundary, never mid-sequence. + assert "\ufffd" not in name + + +def test_make_filename_leaves_an_ascii_title_on_the_char_budget(): + name = make_filename("a" * 400, DOC_ID, "") + assert byte_len(name) <= NAME_MAX_BYTES + assert name == f"{'a' * 97}...__{DOC_ID}.gslide.json" + + +def test_make_filename_keeps_the_id_when_it_leaves_no_room(): + # The title is what gives, never the id: a trimmed id would stop + # addressing the presentation. Same rule as gcal's event filenames. + long_id = "v" * (NAME_MAX_BYTES - 4) + name = make_filename("Some Title", long_id, "") + assert f"__{long_id}.gslide.json" in name diff --git a/python/tests/test_tests_mirror_sources.py b/python/tests/test_tests_mirror_sources.py index 0d71a36a4..fc601bfee 100644 --- a/python/tests/test_tests_mirror_sources.py +++ b/python/tests/test_tests_mirror_sources.py @@ -61,7 +61,7 @@ UNMIRRORED_DIRS = { # would count 816 today. What the ratchet buys is narrower than it looks: # a module whose name appears nowhere in the suite cannot be added # silently. -MIRROR_BASELINE = 186 +MIRROR_BASELINE = 184 def _test_dirs() -> list[pathlib.Path]: diff --git a/python/tests/utils/test_sanitize.py b/python/tests/utils/test_sanitize.py index 4e2506735..5de1f1201 100644 --- a/python/tests/utils/test_sanitize.py +++ b/python/tests/utils/test_sanitize.py @@ -12,7 +12,7 @@ # limitations under the License. # ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= -from mirage.utils.sanitize import sanitize_label +from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label def test_sanitize_label_replaces_unsafe_and_spaces(): @@ -60,8 +60,38 @@ def test_sanitize_label_budget_counts_code_points(): def test_sanitize_label_ellipsizes_on_code_point_boundary(): + # A byte budget wide enough to stay out of the way, so this pins the + # character budget alone. label = "\U00010400" * 120 - result = sanitize_label(label, fallback="X", max_len=100) + result = sanitize_label(label, fallback="X", max_len=100, max_bytes=10_000) assert len(result) == 100 assert result.endswith("...") assert "\ufffd" not in result + + +def test_sanitize_label_honors_the_byte_ceiling_within_the_char_budget(): + # 100 astral code points is 400 bytes, so a name the character budget + # accepts is one ext4 and APFS reject with ENAMETOOLONG. The default + # budget is NAME_MAX, and the cut still lands on a code-point boundary. + label = "\U00010400" * 120 + result = sanitize_label(label, fallback="X", max_len=100) + assert len(result) < 100 + assert byte_len(result) <= NAME_MAX_BYTES + assert result.endswith("...") + assert "\ufffd" not in result + + +def test_sanitize_label_byte_budget_is_the_callers_remaining_room(): + # What the gdocs/gmail filenames pass: NAME_MAX minus the id, the + # separators and the suffix. + result = sanitize_label("会" * 200, fallback="X", max_len=100, max_bytes=60) + assert byte_len(result) <= 60 + assert result.endswith("...") + assert "\ufffd" not in result + + +def test_sanitize_label_drops_the_ellipsis_when_it_cannot_fit(): + # Three dots and nothing is not a name; a budget this small yields + # whatever of the label actually fits. + assert sanitize_label("abcdef", fallback="X", max_len=100, + max_bytes=2) == "ab" diff --git a/scripts/check_normalize_renames.py b/scripts/check_normalize_renames.py new file mode 100644 index 000000000..19dd0c076 --- /dev/null +++ b/scripts/check_normalize_renames.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +TS_SRC = [ + REPO / "typescript/packages/core/src", + REPO / "typescript/packages/node/src", + REPO / "typescript/packages/browser/src", +] +# `normalizeFields(input, {rename: {...}})` and the shared `const RENAME` +# maps the S3-alias families feed it. +RENAME_PROP_RE = re.compile(r"rename:\s*\{([^{}]*)\}", re.S) +RENAME_CONST_RE = re.compile(r"^const \w*RENAME\w*[^=\n]*=\s*\{([^{}]*)\}", + re.M | re.S) +PAIR_RE = re.compile(r"([A-Za-z0-9_]+)\s*:\s*'([A-Za-z0-9_]+)'") +SNAKE_RE = re.compile(r"_([a-z0-9])") + + +def snake_to_camel(snake: str) -> str: + """Reimplement `snakeToCamel` from utils/normalize.ts. + + Args: + snake: The python-side field name. + + Returns: + The camelCase spelling `normalizeFields` produces by default. + """ + return SNAKE_RE.sub(lambda m: m.group(1).upper(), snake) + + +def redundant_pairs(text: str) -> list[tuple[str, str]]: + """Find rename entries that only restate the default mapping. + + Args: + text: One TypeScript source file. + + Returns: + The `(source, target)` pairs whose target is exactly what + `snakeToCamel` would have produced anyway. + """ + found: list[tuple[str, str]] = [] + for match in RENAME_PROP_RE.finditer(text): + found.extend(PAIR_RE.findall(match.group(1))) + for match in RENAME_CONST_RE.finditer(text): + found.extend(PAIR_RE.findall(match.group(1))) + return [(k, v) for k, v in found if snake_to_camel(k) == v] + + +def source_files() -> list[Path]: + """Every TypeScript source file that could hold a rename map. + + Returns: + The files to scan, build output and node_modules excluded. + """ + out: list[Path] = [] + for root in TS_SRC: + if not root.is_dir(): + continue + out.extend(p for p in root.rglob("*.ts") + if "node_modules" not in p.parts and "dist" not in p.parts) + return sorted(out) + + +def main() -> int: + """Fail when a rename map restates what `snakeToCamel` already does. + + `normalizeFields` falls back to `snakeToCamel(key)` for every key no + rename names, so `api_key: 'apiKey'` is not configuration, it is a + second place for the same fact to be wrong -- and it hides the entries + that *are* load-bearing (`endpoint_url: 'endpoint'`, `timeout: + 'timeoutMs'`, `aws_profile: 'profile'`) in a wall of noise. + + Returns: + 0 when every surviving rename entry is a real override. + """ + offenders: list[tuple[Path, list[tuple[str, str]]]] = [] + for path in source_files(): + if path.name == "normalize.test.ts": + continue + pairs = redundant_pairs(path.read_text()) + if pairs: + offenders.append((path, pairs)) + if not offenders: + return 0 + total = sum(len(p) for _, p in offenders) + print(f"{total} rename entries only restate snakeToCamel:\n") + for path, pairs in offenders: + print(f" {path.relative_to(REPO)}") + for key, value in pairs: + print(f" {key}: '{value}'") + print("\nDelete them: normalizeFields already maps these by default.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/typescript/packages/browser/src/index.ts b/typescript/packages/browser/src/index.ts index 11afccc93..f351b7852 100644 --- a/typescript/packages/browser/src/index.ts +++ b/typescript/packages/browser/src/index.ts @@ -31,6 +31,7 @@ export { OPFSAccessor } from './accessor/opfs.ts' export { OPFS_COMMANDS } from './commands/builtin/opfs/index.ts' export { S3Resource, S3_BROWSER_PROMPT, type S3ResourceState } from './resource/s3/s3.ts' export { + normalizeS3Config, redactConfig as redactS3Config, type S3BrowserOperation, type S3BrowserPresignedUrlProvider, @@ -165,12 +166,14 @@ export { } from './resource/qingstor/config.ts' export { SlackResource, type SlackResourceState } from './resource/slack/slack.ts' export { + normalizeSlackConfig, redactSlackConfig, type SlackConfig, type SlackConfigRedacted, } from './resource/slack/config.ts' export { DiscordResource, type DiscordResourceState } from './resource/discord/discord.ts' export { + normalizeDiscordConfig, redactDiscordConfig, type DiscordConfig, type DiscordConfigRedacted, @@ -181,6 +184,7 @@ export { MongoDBResource, type MongoDBResourceOptions } from './resource/mongodb export { HttpMongoDriver, type HttpMongoDriverOptions } from './resource/mongodb/http_driver.ts' export { TrelloResource, type TrelloResourceState } from './resource/trello/trello.ts' export { + normalizeTrelloConfig, redactTrelloConfig, type TrelloConfig, type TrelloConfigRedacted, @@ -193,12 +197,14 @@ export { export type { LinearConfig, LinearConfigRedacted } from '@struktoai/mirage-core/core/linear/config' export { NotionResource, type NotionResourceState } from './resource/notion/notion.ts' export { + normalizeNotionConfig, redactNotionConfig, type NotionConfig, type NotionConfigRedacted, } from './resource/notion/config.ts' export { LangfuseResource, type LangfuseResourceState } from './resource/langfuse/langfuse.ts' export { + normalizeLangfuseConfig, redactLangfuseConfig, type LangfuseConfig, type LangfuseConfigRedacted, diff --git a/typescript/packages/browser/src/resource/aliyun/config.ts b/typescript/packages/browser/src/resource/aliyun/config.ts index 07399ba3a..706335206 100644 --- a/typescript/packages/browser/src/resource/aliyun/config.ts +++ b/typescript/packages/browser/src/resource/aliyun/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias({ export const resolvedAliyunEndpoint = alias.resolvedEndpoint export const aliyunToS3Config = alias.toS3Config export const redactAliyunConfig = alias.redact +export const normalizeAliyunConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/backblaze/config.ts b/typescript/packages/browser/src/resource/backblaze/config.ts index 58abdc3af..e809f41e4 100644 --- a/typescript/packages/browser/src/resource/backblaze/config.ts +++ b/typescript/packages/browser/src/resource/backblaze/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias({ export const resolvedBackblazeEndpoint = alias.resolvedEndpoint export const backblazeToS3Config = alias.toS3Config export const redactBackblazeConfig = alias.redact +export const normalizeBackblazeConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/box/config.ts b/typescript/packages/browser/src/resource/box/config.ts index 723eb85d6..f2c2a35a6 100644 --- a/typescript/packages/browser/src/resource/box/config.ts +++ b/typescript/packages/browser/src/resource/box/config.ts @@ -62,12 +62,6 @@ export function redactBoxConfig(config: BoxConfig): BoxConfigRedacted { export function normalizeBoxConfig(input: Record): BoxConfig { return normalizeFields(input, { rename: { - root_folder_id: 'rootFolderId', - content_search: 'contentSearch', - client_id: 'clientId', - client_secret: 'clientSecret', - refresh_token: 'refreshToken', - access_token: 'accessToken', developer_token: 'accessToken', }, }) as unknown as BoxConfig diff --git a/typescript/packages/browser/src/resource/ceph/config.ts b/typescript/packages/browser/src/resource/ceph/config.ts index de3edc86b..8369f1858 100644 --- a/typescript/packages/browser/src/resource/ceph/config.ts +++ b/typescript/packages/browser/src/resource/ceph/config.ts @@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias({ export const cephToS3Config = alias.toS3Config export const redactCephConfig = alias.redact +export const normalizeCephConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/digitalocean/config.ts b/typescript/packages/browser/src/resource/digitalocean/config.ts index 13557096b..6ff9f7b38 100644 --- a/typescript/packages/browser/src/resource/digitalocean/config.ts +++ b/typescript/packages/browser/src/resource/digitalocean/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias export const resolvedDigitalOceanEndpoint = alias.resolvedEndpoint export const digitalOceanToS3Config = alias.toS3Config export const redactDigitalOceanConfig = alias.redact +export const normalizeDigitalOceanConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/discord/config.ts b/typescript/packages/browser/src/resource/discord/config.ts index ed19b48e4..bddcad031 100644 --- a/typescript/packages/browser/src/resource/discord/config.ts +++ b/typescript/packages/browser/src/resource/discord/config.ts @@ -14,6 +14,7 @@ import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets' import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' type HeaderProvider = () => Promise> | Record @@ -31,3 +32,14 @@ export type DiscordConfigRedacted = RedactedConfig export function redactDiscordConfig(config: DiscordConfig): DiscordConfigRedacted { return redactConfigWithSchema(DiscordConfigSchema, config) as unknown as DiscordConfigRedacted } + +/** + * Translate a python-style config blob to this one's camelCase. + * + * No rename map: every field's camelCase spelling is what `snakeToCamel` + * already produces, and restating those only creates a second place to be + * wrong. Mirrors node's `normalizeDiscordConfig`. + */ +export function normalizeDiscordConfig(input: Record): DiscordConfig { + return normalizeFields(input) as unknown as DiscordConfig +} diff --git a/typescript/packages/browser/src/resource/dropbox/config.ts b/typescript/packages/browser/src/resource/dropbox/config.ts index 114dff7a0..b12c32528 100644 --- a/typescript/packages/browser/src/resource/dropbox/config.ts +++ b/typescript/packages/browser/src/resource/dropbox/config.ts @@ -47,13 +47,5 @@ export function redactDropboxConfig(config: DropboxConfig): DropboxConfigRedacte } export function normalizeDropboxConfig(input: Record): DropboxConfig { - return normalizeFields(input, { - rename: { - client_id: 'clientId', - client_secret: 'clientSecret', - refresh_token: 'refreshToken', - root_path: 'rootPath', - content_search: 'contentSearch', - }, - }) as unknown as DropboxConfig + return normalizeFields(input) as unknown as DropboxConfig } diff --git a/typescript/packages/browser/src/resource/gcs/config.ts b/typescript/packages/browser/src/resource/gcs/config.ts index 173823acf..6de704251 100644 --- a/typescript/packages/browser/src/resource/gcs/config.ts +++ b/typescript/packages/browser/src/resource/gcs/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias({ export const gcsToS3Config = alias.toS3Config export const redactGcsConfig = alias.redact +export const normalizeGCSConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/langfuse/config.ts b/typescript/packages/browser/src/resource/langfuse/config.ts index 6ed5101c4..3e3bc183d 100644 --- a/typescript/packages/browser/src/resource/langfuse/config.ts +++ b/typescript/packages/browser/src/resource/langfuse/config.ts @@ -14,6 +14,7 @@ import { redactConfigWithSchema, secretStr, z } from '@struktoai/mirage-core/resource/secrets' import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' const LangfuseConfigSchema = z.object({ publicKey: z.string(), @@ -31,3 +32,14 @@ export type LangfuseConfigRedacted = RedactedConfig export function redactLangfuseConfig(config: LangfuseConfig): LangfuseConfigRedacted { return redactConfigWithSchema(LangfuseConfigSchema, config) as unknown as LangfuseConfigRedacted } + +/** + * Translate a python-style config blob to this one's camelCase. + * + * No rename map: every field's camelCase spelling is what `snakeToCamel` + * already produces, and restating those only creates a second place to be + * wrong. Mirrors node's `normalizeLangfuseConfig`. + */ +export function normalizeLangfuseConfig(input: Record): LangfuseConfig { + return normalizeFields(input) as unknown as LangfuseConfig +} diff --git a/typescript/packages/browser/src/resource/minio/config.ts b/typescript/packages/browser/src/resource/minio/config.ts index d68f212f8..ef880d51a 100644 --- a/typescript/packages/browser/src/resource/minio/config.ts +++ b/typescript/packages/browser/src/resource/minio/config.ts @@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias({ export const minioToS3Config = alias.toS3Config export const redactMinIOConfig = alias.redact +export const normalizeMinIOConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/notion/config.ts b/typescript/packages/browser/src/resource/notion/config.ts index ce502b388..b2f1a0f88 100644 --- a/typescript/packages/browser/src/resource/notion/config.ts +++ b/typescript/packages/browser/src/resource/notion/config.ts @@ -15,6 +15,7 @@ import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js' import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets' import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' const NotionConfigSchema = z.object({ authProvider: secretSchema( @@ -30,3 +31,14 @@ export type NotionConfigRedacted = RedactedConfig export function redactNotionConfig(config: NotionConfig): NotionConfigRedacted { return redactConfigWithSchema(NotionConfigSchema, config) as unknown as NotionConfigRedacted } + +/** + * Translate a python-style config blob to this one's camelCase. + * + * No rename map: every field's camelCase spelling is what `snakeToCamel` + * already produces, and restating those only creates a second place to be + * wrong. Mirrors node's `normalizeNotionConfig`. + */ +export function normalizeNotionConfig(input: Record): NotionConfig { + return normalizeFields(input) as unknown as NotionConfig +} diff --git a/typescript/packages/browser/src/resource/oci/config.ts b/typescript/packages/browser/src/resource/oci/config.ts index 52ddfcbde..be74a5a39 100644 --- a/typescript/packages/browser/src/resource/oci/config.ts +++ b/typescript/packages/browser/src/resource/oci/config.ts @@ -36,3 +36,4 @@ const alias = makeBrowserS3Alias({ export const resolvedOciEndpoint = alias.resolvedEndpoint export const ociToS3Config = alias.toS3Config export const redactOciConfig = alias.redact +export const normalizeOCIConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/qingstor/config.ts b/typescript/packages/browser/src/resource/qingstor/config.ts index 0937add08..52138c032 100644 --- a/typescript/packages/browser/src/resource/qingstor/config.ts +++ b/typescript/packages/browser/src/resource/qingstor/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias({ export const resolvedQingStorEndpoint = alias.resolvedEndpoint export const qingStorToS3Config = alias.toS3Config export const redactQingStorConfig = alias.redact +export const normalizeQingStorConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/r2/config.ts b/typescript/packages/browser/src/resource/r2/config.ts index 95b43c13a..67f483d6d 100644 --- a/typescript/packages/browser/src/resource/r2/config.ts +++ b/typescript/packages/browser/src/resource/r2/config.ts @@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias({ export const resolvedR2Endpoint = alias.resolvedEndpoint export const r2ToS3Config = alias.toS3Config export const redactR2Config = alias.redact +export const normalizeR2Config = alias.normalize diff --git a/typescript/packages/browser/src/resource/registry.test.ts b/typescript/packages/browser/src/resource/registry.test.ts index 3563fc628..291c6e517 100644 --- a/typescript/packages/browser/src/resource/registry.test.ts +++ b/typescript/packages/browser/src/resource/registry.test.ts @@ -36,6 +36,50 @@ describe('browser resource registry', () => { } }) + // Every entry used to hand-roll `normalizeFields` with a rename map that + // mostly restated what `snakeToCamel` already does, then cast the result + // through a config interface written a second time in the registry. The + // casts hid a mismatch: nothing checked that the shape the resource wants + // is the shape the entry produces. + it('normalizes snake_case config for every hand-wired backend', async () => { + const provider = (): Promise => Promise.resolve('https://example.com/signed') + const cases: [string, Record, Record][] = [ + [ + 'trello', + { api_key: 'k', api_token: 't', workspace_id: 'w', board_ids: ['b'], base_url: 'u' }, + { workspaceId: 'w', boardIds: ['b'], baseUrl: 'u' }, + ], + [ + 'langfuse', + { public_key: 'p', secret_key: 's', default_trace_limit: 5, default_search_limit: 6 }, + { publicKey: 'p', defaultTraceLimit: 5, defaultSearchLimit: 6 }, + ], + ['slack', { proxy_url: 'http://x' }, { proxyUrl: 'http://x' }], + ['discord', { proxy_url: 'http://x' }, { proxyUrl: 'http://x' }], + [ + 's3', + { bucket: 'b', presignedUrlProvider: provider, endpoint_url: 'http://e', key_prefix: 'p/' }, + { bucket: 'b', endpoint: 'http://e', keyPrefix: 'p/' }, + ], + [ + 'minio', + { bucket: 'b', presignedUrlProvider: provider, endpoint_url: 'http://e' }, + { bucket: 'b', endpoint: 'http://e' }, + ], + ] + for (const [name, input, expected] of cases) { + const state = (await (await buildResource(name, input)).getState()) as { + config: Record + } + expect(state.config, name).toMatchObject(expected) + // `endpoint_url` is the one rename that is not mechanical; a leftover + // snake_case key means the entry skipped normalization entirely. + for (const key of Object.keys(state.config)) { + expect(key, `${name}.${key}`).not.toContain('_') + } + } + }) + it('lists known resources sorted', () => { const names = knownResources() expect(names).toContain('ram') diff --git a/typescript/packages/browser/src/resource/registry.ts b/typescript/packages/browser/src/resource/registry.ts index 74ab5afbf..07c6cd1c5 100644 --- a/typescript/packages/browser/src/resource/registry.ts +++ b/typescript/packages/browser/src/resource/registry.ts @@ -18,7 +18,6 @@ import type { DifyConfig } from '@struktoai/mirage-core/resource/dify/config' import type { QdrantConfig } from '@struktoai/mirage-core/resource/qdrant/config' import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' import { compareCodePoints } from '@struktoai/mirage-core/utils/sort' -import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js' /** * Construct a resource by registry name in the browser runtime. @@ -35,78 +34,6 @@ import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth. */ export type ResourceFactory = (config: Record) => Promise -interface S3BrowserCtorConfig { - bucket: string - presignedUrlProvider: ( - path: string, - op: 'GET' | 'PUT' | 'HEAD' | 'DELETE' | 'LIST' | 'COPY', - opts?: { - contentType?: string - ttlSec?: number - listPrefix?: string - listDelimiter?: string - listContinuationToken?: string - copySource?: string - }, - ) => Promise -} - -type GCSBrowserCtorConfig = S3BrowserCtorConfig & { region?: string; endpoint?: string } -type R2BrowserCtorConfig = S3BrowserCtorConfig & { - accountId?: string - region?: string - endpoint?: string -} -type OCIBrowserCtorConfig = S3BrowserCtorConfig & { - namespace?: string - region?: string - endpoint?: string -} -type SupabaseBrowserCtorConfig = S3BrowserCtorConfig & { - projectRef?: string - region?: string - endpoint?: string -} -type S3AliasBrowserCtorConfig = S3BrowserCtorConfig & { - region?: string - endpoint?: string -} -interface SlackBrowserCtorConfig { - proxyUrl: string - getHeaders?: () => Promise> | Record -} -interface DiscordBrowserCtorConfig { - proxyUrl: string - getHeaders?: () => Promise> | Record -} -interface NotionBrowserCtorConfig { - authProvider: OAuthClientProvider - serverUrl?: string -} -interface TrelloBrowserCtorConfig { - apiKey: string - apiToken: string - workspaceId?: string - boardIds?: readonly string[] - baseUrl?: string -} -interface LangfuseBrowserCtorConfig { - publicKey: string - secretKey: string - host?: string - defaultTraceLimit?: number - defaultSearchLimit?: number - defaultFromTimestamp?: string -} -interface PostgresBrowserCtorConfig { - dsn: string - schemas?: readonly string[] - defaultRowLimit?: number - maxReadRows?: number - maxReadBytes?: number - defaultSearchLimit?: number -} - const REGISTRY: Record = { ram: async (_config) => { const { RAMResource } = await import('@struktoai/mirage-core/resource/ram/ram') @@ -119,135 +46,93 @@ const REGISTRY: Record = { }, s3: async (config) => { const { S3Resource } = await import('./s3/s3.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new S3Resource(norm as unknown as S3BrowserCtorConfig) + const { normalizeS3Config } = await import('./s3/config.ts') + return new S3Resource(normalizeS3Config(config)) }, gcs: async (config) => { const { GCSResource } = await import('./gcs/gcs.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new GCSResource(norm as unknown as GCSBrowserCtorConfig) + const { normalizeGCSConfig } = await import('./gcs/config.ts') + return new GCSResource(normalizeGCSConfig(config)) }, r2: async (config) => { const { R2Resource } = await import('./r2/r2.ts') - const norm = normalizeFields(config, { - rename: { account_id: 'accountId', endpoint_url: 'endpoint' }, - }) - return new R2Resource(norm as unknown as R2BrowserCtorConfig) + const { normalizeR2Config } = await import('./r2/config.ts') + return new R2Resource(normalizeR2Config(config)) }, oci: async (config) => { const { OCIResource } = await import('./oci/oci.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new OCIResource(norm as unknown as OCIBrowserCtorConfig) + const { normalizeOCIConfig } = await import('./oci/config.ts') + return new OCIResource(normalizeOCIConfig(config)) }, supabase: async (config) => { const { SupabaseResource } = await import('./supabase/supabase.ts') - const norm = normalizeFields(config, { - rename: { project_ref: 'projectRef', endpoint_url: 'endpoint' }, - }) - return new SupabaseResource(norm as unknown as SupabaseBrowserCtorConfig) + const { normalizeSupabaseConfig } = await import('./supabase/config.ts') + return new SupabaseResource(normalizeSupabaseConfig(config)) }, minio: async (config) => { const { MinIOResource } = await import('./minio/minio.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new MinIOResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeMinIOConfig } = await import('./minio/config.ts') + return new MinIOResource(normalizeMinIOConfig(config)) }, ceph: async (config) => { const { CephResource } = await import('./ceph/ceph.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new CephResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeCephConfig } = await import('./ceph/config.ts') + return new CephResource(normalizeCephConfig(config)) }, seaweedfs: async (config) => { const { SeaweedFSResource } = await import('./seaweedfs/seaweedfs.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new SeaweedFSResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeSeaweedFSConfig } = await import('./seaweedfs/config.ts') + return new SeaweedFSResource(normalizeSeaweedFSConfig(config)) }, wasabi: async (config) => { const { WasabiResource } = await import('./wasabi/wasabi.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new WasabiResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeWasabiConfig } = await import('./wasabi/config.ts') + return new WasabiResource(normalizeWasabiConfig(config)) }, backblaze: async (config) => { const { BackblazeResource } = await import('./backblaze/backblaze.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new BackblazeResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeBackblazeConfig } = await import('./backblaze/config.ts') + return new BackblazeResource(normalizeBackblazeConfig(config)) }, digitalocean: async (config) => { const { DigitalOceanResource } = await import('./digitalocean/digitalocean.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new DigitalOceanResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeDigitalOceanConfig } = await import('./digitalocean/config.ts') + return new DigitalOceanResource(normalizeDigitalOceanConfig(config)) }, tencent: async (config) => { const { TencentResource } = await import('./tencent/tencent.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new TencentResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeTencentConfig } = await import('./tencent/config.ts') + return new TencentResource(normalizeTencentConfig(config)) }, aliyun: async (config) => { const { AliyunResource } = await import('./aliyun/aliyun.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new AliyunResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeAliyunConfig } = await import('./aliyun/config.ts') + return new AliyunResource(normalizeAliyunConfig(config)) }, scaleway: async (config) => { const { ScalewayResource } = await import('./scaleway/scaleway.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new ScalewayResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeScalewayConfig } = await import('./scaleway/config.ts') + return new ScalewayResource(normalizeScalewayConfig(config)) }, qingstor: async (config) => { const { QingStorResource } = await import('./qingstor/qingstor.ts') - const norm = normalizeFields(config, { - rename: { endpoint_url: 'endpoint' }, - }) - return new QingStorResource(norm as unknown as S3AliasBrowserCtorConfig) + const { normalizeQingStorConfig } = await import('./qingstor/config.ts') + return new QingStorResource(normalizeQingStorConfig(config)) }, slack: async (config) => { const { SlackResource } = await import('./slack/slack.ts') - const norm = normalizeFields(config, { - rename: { proxy_url: 'proxyUrl', get_headers: 'getHeaders' }, - }) - return new SlackResource(norm as unknown as SlackBrowserCtorConfig) + const { normalizeSlackConfig } = await import('./slack/config.ts') + return new SlackResource(normalizeSlackConfig(config)) }, discord: async (config) => { const { DiscordResource } = await import('./discord/discord.ts') - const norm = normalizeFields(config, { - rename: { proxy_url: 'proxyUrl', get_headers: 'getHeaders' }, - }) - return new DiscordResource(norm as unknown as DiscordBrowserCtorConfig) + const { normalizeDiscordConfig } = await import('./discord/config.ts') + return new DiscordResource(normalizeDiscordConfig(config)) }, trello: async (config) => { const { TrelloResource } = await import('./trello/trello.ts') - const norm = normalizeFields(config, { - rename: { - api_key: 'apiKey', - api_token: 'apiToken', - workspace_id: 'workspaceId', - board_ids: 'boardIds', - base_url: 'baseUrl', - }, - }) - return new TrelloResource(norm as unknown as TrelloBrowserCtorConfig) + const { normalizeTrelloConfig } = await import('./trello/config.ts') + return new TrelloResource(normalizeTrelloConfig(config)) }, linear: async (config) => { const { LinearResource } = await import('./linear/linear.ts') @@ -256,15 +141,9 @@ const REGISTRY: Record = { }, postgres: async (config) => { const { PostgresResource } = await import('./postgres/postgres.ts') - const norm = normalizeFields(config, { - rename: { - default_row_limit: 'defaultRowLimit', - max_read_rows: 'maxReadRows', - max_read_bytes: 'maxReadBytes', - default_search_limit: 'defaultSearchLimit', - }, - }) - return new PostgresResource(norm as unknown as PostgresBrowserCtorConfig) + const { normalizePostgresConfig } = + await import('@struktoai/mirage-core/resource/postgres/config') + return new PostgresResource(normalizePostgresConfig(config)) }, mongodb: async (config) => { const { MongoDBResource } = await import('./mongodb/mongodb.ts') @@ -294,23 +173,13 @@ const REGISTRY: Record = { }, notion: async (config) => { const { NotionResource } = await import('./notion/notion.ts') - const norm = normalizeFields(config, { - rename: { auth_provider: 'authProvider', server_url: 'serverUrl' }, - }) - return new NotionResource(norm as unknown as NotionBrowserCtorConfig) + const { normalizeNotionConfig } = await import('./notion/config.ts') + return new NotionResource(normalizeNotionConfig(config)) }, langfuse: async (config) => { const { LangfuseResource } = await import('./langfuse/langfuse.ts') - const norm = normalizeFields(config, { - rename: { - public_key: 'publicKey', - secret_key: 'secretKey', - default_trace_limit: 'defaultTraceLimit', - default_search_limit: 'defaultSearchLimit', - default_from_timestamp: 'defaultFromTimestamp', - }, - }) - return new LangfuseResource(norm as unknown as LangfuseBrowserCtorConfig) + const { normalizeLangfuseConfig } = await import('./langfuse/config.ts') + return new LangfuseResource(normalizeLangfuseConfig(config)) }, github: async (config) => { const { GitHubResource } = await import('./github/github.ts') diff --git a/typescript/packages/browser/src/resource/s3/config.ts b/typescript/packages/browser/src/resource/s3/config.ts index 3faae773b..906a4e11a 100644 --- a/typescript/packages/browser/src/resource/s3/config.ts +++ b/typescript/packages/browser/src/resource/s3/config.ts @@ -12,6 +12,20 @@ // limitations under the License. // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +import type { S3Config } from '@struktoai/mirage-core/resource/s3/config' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' + +// The one field whose camelCase spelling is not what `snakeToCamel` would +// produce, so it is the whole rename map for the browser's S3 family. Every +// other field (`key_prefix`, `default_content_type`, ...) round-trips through +// the default, and restating those only creates a second place to be wrong. +export const S3_BROWSER_RENAME: Record = { endpoint_url: 'endpoint' } + +/** Translate a python-style S3 config blob to the browser's camelCase one. */ +export function normalizeS3Config(input: Record): S3Config { + return normalizeFields(input, { rename: S3_BROWSER_RENAME }) as unknown as S3Config +} + export { redactConfig } from '@struktoai/mirage-core/resource/s3/config' export type { S3BrowserOperation, diff --git a/typescript/packages/browser/src/resource/s3_alias.ts b/typescript/packages/browser/src/resource/s3_alias.ts index 36dbf0b0e..ff2f09972 100644 --- a/typescript/packages/browser/src/resource/s3_alias.ts +++ b/typescript/packages/browser/src/resource/s3_alias.ts @@ -17,6 +17,8 @@ import type { RegisteredCommand } from '@struktoai/mirage-core/commands/config' import type { RegisteredOp } from '@struktoai/mirage-core/ops/registry' import { remapCommandsResource, remapOpsResource } from '@struktoai/mirage-core/resource/s3/remap' import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' +import { S3_BROWSER_RENAME } from './s3/config.ts' import type { S3BrowserPresignedUrlProvider, S3Config, S3ConfigRedacted } from './s3/config.ts' /** @@ -92,6 +94,7 @@ export interface BrowserAlias { resolvedEndpoint: (config: C) => string | E toS3Config: (config: C) => S3Config redact: (config: C) => R + normalize: (input: Record) => C } /** @@ -137,6 +140,7 @@ export function makeBrowserS3Alias< } }, redact: (config) => redactConfigWithSchema(schema, config) as unknown as R, + normalize: (input) => normalizeFields(input, { rename: S3_BROWSER_RENAME }) as unknown as C, } } diff --git a/typescript/packages/browser/src/resource/scaleway/config.ts b/typescript/packages/browser/src/resource/scaleway/config.ts index 3d0496aad..984177b39 100644 --- a/typescript/packages/browser/src/resource/scaleway/config.ts +++ b/typescript/packages/browser/src/resource/scaleway/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias({ export const resolvedScalewayEndpoint = alias.resolvedEndpoint export const scalewayToS3Config = alias.toS3Config export const redactScalewayConfig = alias.redact +export const normalizeScalewayConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/seaweedfs/config.ts b/typescript/packages/browser/src/resource/seaweedfs/config.ts index bec39f21b..8b995837d 100644 --- a/typescript/packages/browser/src/resource/seaweedfs/config.ts +++ b/typescript/packages/browser/src/resource/seaweedfs/config.ts @@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias({ export const seaweedfsToS3Config = alias.toS3Config export const redactSeaweedFSConfig = alias.redact +export const normalizeSeaweedFSConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/slack/config.ts b/typescript/packages/browser/src/resource/slack/config.ts index 13b793644..6c34e54aa 100644 --- a/typescript/packages/browser/src/resource/slack/config.ts +++ b/typescript/packages/browser/src/resource/slack/config.ts @@ -14,6 +14,7 @@ import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets' import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' type HeaderProvider = () => Promise> | Record @@ -31,3 +32,14 @@ export type SlackConfigRedacted = RedactedConfig export function redactSlackConfig(config: SlackConfig): SlackConfigRedacted { return redactConfigWithSchema(SlackConfigSchema, config) as unknown as SlackConfigRedacted } + +/** + * Translate a python-style config blob to this one's camelCase. + * + * No rename map: every field's camelCase spelling is what `snakeToCamel` + * already produces, and restating those only creates a second place to be + * wrong. Mirrors node's `normalizeSlackConfig`. + */ +export function normalizeSlackConfig(input: Record): SlackConfig { + return normalizeFields(input) as unknown as SlackConfig +} diff --git a/typescript/packages/browser/src/resource/supabase/config.ts b/typescript/packages/browser/src/resource/supabase/config.ts index 6e4326ecc..adbea2af6 100644 --- a/typescript/packages/browser/src/resource/supabase/config.ts +++ b/typescript/packages/browser/src/resource/supabase/config.ts @@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias({ export const resolvedSupabaseEndpoint = alias.resolvedEndpoint export const supabaseToS3Config = alias.toS3Config export const redactSupabaseConfig = alias.redact +export const normalizeSupabaseConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/tencent/config.ts b/typescript/packages/browser/src/resource/tencent/config.ts index 840f3cdf6..c3a5e460b 100644 --- a/typescript/packages/browser/src/resource/tencent/config.ts +++ b/typescript/packages/browser/src/resource/tencent/config.ts @@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias({ export const resolvedTencentEndpoint = alias.resolvedEndpoint export const tencentToS3Config = alias.toS3Config export const redactTencentConfig = alias.redact +export const normalizeTencentConfig = alias.normalize diff --git a/typescript/packages/browser/src/resource/trello/config.ts b/typescript/packages/browser/src/resource/trello/config.ts index aa9890dcf..f704ee406 100644 --- a/typescript/packages/browser/src/resource/trello/config.ts +++ b/typescript/packages/browser/src/resource/trello/config.ts @@ -14,6 +14,7 @@ import { redactConfigWithSchema, secretStr, z } from '@struktoai/mirage-core/resource/secrets' import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets' +import { normalizeFields } from '@struktoai/mirage-core/utils/normalize' const TrelloConfigSchema = z.object({ apiKey: secretStr(), @@ -30,3 +31,14 @@ export type TrelloConfigRedacted = RedactedConfig): TrelloConfig { + return normalizeFields(input) as unknown as TrelloConfig +} diff --git a/typescript/packages/browser/src/resource/wasabi/config.ts b/typescript/packages/browser/src/resource/wasabi/config.ts index 52b247576..4010956db 100644 --- a/typescript/packages/browser/src/resource/wasabi/config.ts +++ b/typescript/packages/browser/src/resource/wasabi/config.ts @@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias({ export const resolvedWasabiEndpoint = alias.resolvedEndpoint export const wasabiToS3Config = alias.toS3Config export const redactWasabiConfig = alias.redact +export const normalizeWasabiConfig = alias.normalize diff --git a/typescript/packages/core/src/core/discord/config.ts b/typescript/packages/core/src/core/discord/config.ts index 7d36d645d..5cec0c7a5 100644 --- a/typescript/packages/core/src/core/discord/config.ts +++ b/typescript/packages/core/src/core/discord/config.ts @@ -35,5 +35,5 @@ export function redactDiscordConfig(config: DiscordConfig): DiscordConfigRedacte } export function normalizeDiscordConfig(input: Record): DiscordConfig { - return normalizeFields(input, {}) as unknown as DiscordConfig + return normalizeFields(input) as unknown as DiscordConfig } diff --git a/typescript/packages/core/src/core/github/config.ts b/typescript/packages/core/src/core/github/config.ts index f547668c5..0e8f48c5b 100644 --- a/typescript/packages/core/src/core/github/config.ts +++ b/typescript/packages/core/src/core/github/config.ts @@ -34,9 +34,7 @@ export function redactGitHubConfig(config: GitHubConfig): GitHubConfigRedacted { } export function normalizeGitHubConfig(input: Record): GitHubConfig { - return normalizeFields(input, { - rename: { base_url: 'baseUrl' }, - }) as unknown as GitHubConfig + return normalizeFields(input) as unknown as GitHubConfig } export const GhConfigSchema = z.object({ diff --git a/typescript/packages/core/src/core/gmail/readdir.ts b/typescript/packages/core/src/core/gmail/readdir.ts index 95842cc87..8f33ec99f 100644 --- a/typescript/packages/core/src/core/gmail/readdir.ts +++ b/typescript/packages/core/src/core/gmail/readdir.ts @@ -27,16 +27,27 @@ import { messageJsonBytes, } from './messages.ts' import { enoent } from '../../utils/errors.ts' -import { sanitizeLabel } from '../../utils/sanitize.ts' +import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts' import { compareCodePoints } from '../../utils/sort.ts' const TITLE_MAX = 80 +const MSG_SUFFIX = '.gmail.json' -export const sanitize = (text: string): string => - sanitizeLabel(text, { fallback: 'No_Subject', maxLen: TITLE_MAX }) +export const sanitize = (text: string, maxBytes?: number): string => + sanitizeLabel(text, { + fallback: 'No_Subject', + maxLen: TITLE_MAX, + ...(maxBytes !== undefined ? { maxBytes } : {}), + }) -function msgFilename(subject: string, msgId: string): string { - return `${sanitize(subject)}__${msgId}.gmail.json` +/** + * 80 characters is 240 bytes of CJK, which overflows the 255-byte NAME_MAX + * once the id and `.gmail.json` are added; the filesystem rejects the name + * outright. So the subject takes what the id and the suffix leave. + */ +export function msgFilename(subject: string, msgId: string): string { + const fixed = 2 + byteLength(msgId) + MSG_SUFFIX.length + return `${sanitize(subject, NAME_MAX_BYTES - fixed)}__${msgId}${MSG_SUFFIX}` } function dateFromInternal(internalDate: string | undefined): string { diff --git a/typescript/packages/core/src/core/gmail/search.ts b/typescript/packages/core/src/core/gmail/search.ts index e3425de0a..29b88c66c 100644 --- a/typescript/packages/core/src/core/gmail/search.ts +++ b/typescript/packages/core/src/core/gmail/search.ts @@ -14,7 +14,7 @@ import type { TokenManager } from '../google/client.ts' import { decodeBody, extractHeader, getMessageRaw, listMessages } from './messages.ts' -import { sanitize } from './readdir.ts' +import { msgFilename } from './readdir.ts' import type { GmailScope } from './scope.ts' const EXCERPT_WINDOW = 120 @@ -115,7 +115,11 @@ export function formatGrepResults( const label = row.label !== '' ? row.label : (scope.labelName ?? 'INBOX') const date = row.date const mid = row.id - const filename = `${sanitize(row.subject || 'No Subject')}__${mid}.gmail.json` + // The same builder readdir names the file with, not a second spelling of + // it: the subject's budget depends on the id and the suffix, so a hit + // composed from a bare `sanitize` pointed at a path that does not exist + // once a long subject was trimmed. + const filename = msgFilename(row.subject || 'No Subject', mid) const sender = row.sender !== '' ? row.sender : '?' const haystack = `${row.subject}\n${row.bodyText}` let excerpt = pattern !== '' ? extractExcerpt(haystack, pattern) : '' diff --git a/typescript/packages/core/src/core/google/config.ts b/typescript/packages/core/src/core/google/config.ts index e67365b1f..0d752c138 100644 --- a/typescript/packages/core/src/core/google/config.ts +++ b/typescript/packages/core/src/core/google/config.ts @@ -75,15 +75,5 @@ export function redactGoogleConfig(config: GoogleConfig): GoogleConfigRedacted { } export function normalizeGoogleConfig(input: Record): GoogleConfig { - return normalizeFields(input, { - rename: { - client_id: 'clientId', - client_secret: 'clientSecret', - refresh_token: 'refreshToken', - api_base: 'apiBase', - folder_id: 'folderId', - time_zone: 'timeZone', - min_access_role: 'minAccessRole', - }, - }) as unknown as GoogleConfig + return normalizeFields(input) as unknown as GoogleConfig } diff --git a/typescript/packages/core/src/core/linear/config.ts b/typescript/packages/core/src/core/linear/config.ts index 9f8c6548a..97426993b 100644 --- a/typescript/packages/core/src/core/linear/config.ts +++ b/typescript/packages/core/src/core/linear/config.ts @@ -37,11 +37,5 @@ export function redactLinearConfig(config: LinearConfig): LinearConfigRedacted { } export function normalizeLinearConfig(input: Record): LinearConfig { - return normalizeFields(input, { - rename: { - api_key: 'apiKey', - team_ids: 'teamIds', - base_url: 'baseUrl', - }, - }) as unknown as LinearConfig + return normalizeFields(input) as unknown as LinearConfig } diff --git a/typescript/packages/core/src/core/notion/config.ts b/typescript/packages/core/src/core/notion/config.ts index e828ce7b9..42c7b7e34 100644 --- a/typescript/packages/core/src/core/notion/config.ts +++ b/typescript/packages/core/src/core/notion/config.ts @@ -35,10 +35,5 @@ export function redactNotionConfig(config: NotionConfig): NotionConfigRedacted { } export function normalizeNotionConfig(input: Record): NotionConfig { - return normalizeFields(input, { - rename: { - api_key: 'apiKey', - base_url: 'baseUrl', - }, - }) as unknown as NotionConfig + return normalizeFields(input) as unknown as NotionConfig } diff --git a/typescript/packages/core/src/core/slack/config.ts b/typescript/packages/core/src/core/slack/config.ts index 6e1913c33..cb7385fab 100644 --- a/typescript/packages/core/src/core/slack/config.ts +++ b/typescript/packages/core/src/core/slack/config.ts @@ -36,7 +36,5 @@ export function redactSlackConfig(config: SlackConfig): SlackConfigRedacted { } export function normalizeSlackConfig(input: Record): SlackConfig { - return normalizeFields(input, { - rename: { search_token: 'searchToken', base_url: 'baseUrl' }, - }) as unknown as SlackConfig + return normalizeFields(input) as unknown as SlackConfig } diff --git a/typescript/packages/core/src/resource/databricks_volume/config.ts b/typescript/packages/core/src/resource/databricks_volume/config.ts index fe2a55010..66f7b2b8f 100644 --- a/typescript/packages/core/src/resource/databricks_volume/config.ts +++ b/typescript/packages/core/src/resource/databricks_volume/config.ts @@ -61,8 +61,6 @@ export function redactDatabricksVolumeConfig( export function normalizeDatabricksVolumeConfig( input: Record, ): DatabricksVolumeConfig { - const renamed = normalizeFields(input, { - rename: { root_path: 'rootPath' }, - }) + const renamed = normalizeFields(input) return DatabricksVolumeConfigSchema.parse(renamed) as DatabricksVolumeConfig } diff --git a/typescript/packages/core/src/resource/gcal/event_entry.test.ts b/typescript/packages/core/src/resource/gcal/event_entry.test.ts index 6fd141fc8..055cf549c 100644 --- a/typescript/packages/core/src/resource/gcal/event_entry.test.ts +++ b/typescript/packages/core/src/resource/gcal/event_entry.test.ts @@ -56,7 +56,8 @@ describe('gcal event entry naming', () => { it('trims a long CJK title by bytes, not characters', () => { // 3 bytes per character: a character-counted budget would overflow - // NAME_MAX, which is the bug gdocs' sanitizeTitle still has. + // NAME_MAX. gdocs/gsheets/gslides had exactly that bug until + // sanitizeLabel grew a byte budget. const name = makeEventFilename(EVENT_ID, '0900-1030', '会'.repeat(200)) expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) expect(name).not.toContain('�') diff --git a/typescript/packages/core/src/resource/gcal/event_entry.ts b/typescript/packages/core/src/resource/gcal/event_entry.ts index 2daa5b2d9..9cb19c5cc 100644 --- a/typescript/packages/core/src/resource/gcal/event_entry.ts +++ b/typescript/packages/core/src/resource/gcal/event_entry.ts @@ -16,6 +16,7 @@ import { enoent } from '../../utils/errors.ts' import { makeIdName } from '../../utils/naming.ts' import { NAME_MAX_BYTES, + byteLength, sanitizeName, stripTrailingUnderscores, truncateBytes, @@ -31,8 +32,6 @@ const UNTITLED = 'untitled' // there is no title to sanitize and "busy" is the honest rendering. const BUSY = 'busy' -const UTF8 = new TextEncoder() - /** Pick the title segment for an event filename. */ export function eventTitle(summary: string | null, freeBusy = false): string { if (summary !== null && summary.trim() !== '') return sanitizeName(summary) @@ -48,7 +47,7 @@ export function eventTitle(summary: string | null, freeBusy = false): string { * 255-byte NAME_MAX is left rather than a fixed character count. */ export function makeEventFilename(eventId: string, hhmm: string, title: string): string { - const fixed = UTF8.encode(eventId).length + 2 + hhmm.length + 1 + EVENT_SUFFIX.length + const fixed = byteLength(eventId) + 2 + hhmm.length + 1 + EVENT_SUFFIX.length const trimmed = stripTrailingUnderscores(truncateBytes(title, NAME_MAX_BYTES - fixed)) if (trimmed === '') { // The title is what gives, never the id: trimming the id would make the diff --git a/typescript/packages/core/src/resource/gdocs/doc_entry.test.ts b/typescript/packages/core/src/resource/gdocs/doc_entry.test.ts new file mode 100644 index 000000000..7e96b6eda --- /dev/null +++ b/typescript/packages/core/src/resource/gdocs/doc_entry.test.ts @@ -0,0 +1,55 @@ +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +import { describe, expect, it } from 'vitest' +import { makeFilename } from './doc_entry.ts' +import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts' + +// A real Google file id is 44 characters, so this is the fixed overhead a +// title actually has to fit inside. +const DOC_ID = '1'.repeat(44) + +describe('gdocs document filenames', () => { + it('leads with the date when there is one', () => { + expect(makeFilename('My Document', 'abc123', '2026-03-15T10:00:00Z')).toBe( + '2026-03-15_My_Document__abc123.gdoc.json', + ) + expect(makeFilename('My Document', 'abc123')).toBe('My_Document__abc123.gdoc.json') + }) + + it('fits NAME_MAX for a CJK title', () => { + // 100 characters of CJK is 300 bytes, which the character budget passed + // untouched: with the date, the id and the suffix the name came to 367 + // bytes and ext4/APFS reject it with ENAMETOOLONG. + const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z') + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name.startsWith('2026-08-20_')).toBe(true) + expect(name.endsWith(`__${DOC_ID}.gdoc.json`)).toBe(true) + // The cut lands on a character boundary, never mid-sequence. + expect(name).not.toContain('\uFFFD') + }) + + it('leaves an ascii title on the character budget', () => { + const name = makeFilename('a'.repeat(400), DOC_ID, '') + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gdoc.json`) + }) + + it('keeps the id when it leaves no room', () => { + // The title is what gives, never the id: a trimmed id would stop + // addressing the document. Same rule as gcal's event filenames. + const longId = 'v'.repeat(NAME_MAX_BYTES - 4) + expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gdoc.json`) + }) +}) diff --git a/typescript/packages/core/src/resource/gdocs/doc_entry.ts b/typescript/packages/core/src/resource/gdocs/doc_entry.ts index d8d78661a..e17ed9ec2 100644 --- a/typescript/packages/core/src/resource/gdocs/doc_entry.ts +++ b/typescript/packages/core/src/resource/gdocs/doc_entry.ts @@ -12,17 +12,26 @@ // limitations under the License. // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= -import { sanitizeLabel } from '../../utils/sanitize.ts' +import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts' const TITLE_MAX_CHARS = 100 +const SUFFIX = '.gdoc.json' +const DATE_LEN = 10 -const sanitizeTitle = (title: string): string => - sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS }) +const sanitizeTitle = (title: string, maxBytes: number): string => + sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes }) +/** + * Build a filename from title, doc ID, and modified date. + * + * The title takes whatever of the 255-byte NAME_MAX the date, the id and the + * suffix leave, rather than a flat character count: those are the same number + * only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS + * reject outright. The id never gives, so the name keeps addressing the + * document -- same rule as gcal's event filenames. + */ export function makeFilename(title: string, docId: string, modifiedTime = ''): string { - const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : '' - if (datePrefix !== '') { - return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gdoc.json` - } - return `${sanitizeTitle(title)}__${docId}.gdoc.json` + const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : '' + const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length + return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}` } diff --git a/typescript/packages/core/src/resource/gsheets/sheet_entry.test.ts b/typescript/packages/core/src/resource/gsheets/sheet_entry.test.ts new file mode 100644 index 000000000..5f572732c --- /dev/null +++ b/typescript/packages/core/src/resource/gsheets/sheet_entry.test.ts @@ -0,0 +1,55 @@ +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +import { describe, expect, it } from 'vitest' +import { makeFilename } from './sheet_entry.ts' +import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts' + +// A real Google file id is 44 characters, so this is the fixed overhead a +// title actually has to fit inside. +const DOC_ID = '1'.repeat(44) + +describe('gsheets spreadsheet filenames', () => { + it('leads with the date when there is one', () => { + expect(makeFilename('My Spreadsheet', 'abc123', '2026-03-15T10:00:00Z')).toBe( + '2026-03-15_My_Spreadsheet__abc123.gsheet.json', + ) + expect(makeFilename('My Spreadsheet', 'abc123')).toBe('My_Spreadsheet__abc123.gsheet.json') + }) + + it('fits NAME_MAX for a CJK title', () => { + // 100 characters of CJK is 300 bytes, which the character budget passed + // untouched: with the date, the id and the suffix the name came to 367 + // bytes and ext4/APFS reject it with ENAMETOOLONG. + const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z') + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name.startsWith('2026-08-20_')).toBe(true) + expect(name.endsWith(`__${DOC_ID}.gsheet.json`)).toBe(true) + // The cut lands on a character boundary, never mid-sequence. + expect(name).not.toContain('\uFFFD') + }) + + it('leaves an ascii title on the character budget', () => { + const name = makeFilename('a'.repeat(400), DOC_ID, '') + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gsheet.json`) + }) + + it('keeps the id when it leaves no room', () => { + // The title is what gives, never the id: a trimmed id would stop + // addressing the spreadsheet. Same rule as gcal's event filenames. + const longId = 'v'.repeat(NAME_MAX_BYTES - 4) + expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gsheet.json`) + }) +}) diff --git a/typescript/packages/core/src/resource/gsheets/sheet_entry.ts b/typescript/packages/core/src/resource/gsheets/sheet_entry.ts index 4f014f823..d77386c08 100644 --- a/typescript/packages/core/src/resource/gsheets/sheet_entry.ts +++ b/typescript/packages/core/src/resource/gsheets/sheet_entry.ts @@ -12,17 +12,26 @@ // limitations under the License. // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= -import { sanitizeLabel } from '../../utils/sanitize.ts' +import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts' const TITLE_MAX_CHARS = 100 +const SUFFIX = '.gsheet.json' +const DATE_LEN = 10 -const sanitizeTitle = (title: string): string => - sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS }) +const sanitizeTitle = (title: string, maxBytes: number): string => + sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes }) +/** + * Build a filename from title, doc ID, and modified date. + * + * The title takes whatever of the 255-byte NAME_MAX the date, the id and the + * suffix leave, rather than a flat character count: those are the same number + * only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS + * reject outright. The id never gives, so the name keeps addressing the + * document -- same rule as gcal's event filenames. + */ export function makeFilename(title: string, docId: string, modifiedTime = ''): string { - const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : '' - if (datePrefix !== '') { - return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gsheet.json` - } - return `${sanitizeTitle(title)}__${docId}.gsheet.json` + const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : '' + const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length + return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}` } diff --git a/typescript/packages/core/src/resource/gslides/slide_entry.test.ts b/typescript/packages/core/src/resource/gslides/slide_entry.test.ts new file mode 100644 index 000000000..1d8719cc1 --- /dev/null +++ b/typescript/packages/core/src/resource/gslides/slide_entry.test.ts @@ -0,0 +1,55 @@ +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +import { describe, expect, it } from 'vitest' +import { makeFilename } from './slide_entry.ts' +import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts' + +// A real Google file id is 44 characters, so this is the fixed overhead a +// title actually has to fit inside. +const DOC_ID = '1'.repeat(44) + +describe('gslides presentation filenames', () => { + it('leads with the date when there is one', () => { + expect(makeFilename('My Presentation', 'abc123', '2026-03-15T10:00:00Z')).toBe( + '2026-03-15_My_Presentation__abc123.gslide.json', + ) + expect(makeFilename('My Presentation', 'abc123')).toBe('My_Presentation__abc123.gslide.json') + }) + + it('fits NAME_MAX for a CJK title', () => { + // 100 characters of CJK is 300 bytes, which the character budget passed + // untouched: with the date, the id and the suffix the name came to 367 + // bytes and ext4/APFS reject it with ENAMETOOLONG. + const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z') + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name.startsWith('2026-08-20_')).toBe(true) + expect(name.endsWith(`__${DOC_ID}.gslide.json`)).toBe(true) + // The cut lands on a character boundary, never mid-sequence. + expect(name).not.toContain('\uFFFD') + }) + + it('leaves an ascii title on the character budget', () => { + const name = makeFilename('a'.repeat(400), DOC_ID, '') + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gslide.json`) + }) + + it('keeps the id when it leaves no room', () => { + // The title is what gives, never the id: a trimmed id would stop + // addressing the presentation. Same rule as gcal's event filenames. + const longId = 'v'.repeat(NAME_MAX_BYTES - 4) + expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gslide.json`) + }) +}) diff --git a/typescript/packages/core/src/resource/gslides/slide_entry.ts b/typescript/packages/core/src/resource/gslides/slide_entry.ts index e375a021c..7dabb78cb 100644 --- a/typescript/packages/core/src/resource/gslides/slide_entry.ts +++ b/typescript/packages/core/src/resource/gslides/slide_entry.ts @@ -12,17 +12,26 @@ // limitations under the License. // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= -import { sanitizeLabel } from '../../utils/sanitize.ts' +import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts' const TITLE_MAX_CHARS = 100 +const SUFFIX = '.gslide.json' +const DATE_LEN = 10 -const sanitizeTitle = (title: string): string => - sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS }) +const sanitizeTitle = (title: string, maxBytes: number): string => + sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes }) +/** + * Build a filename from title, doc ID, and modified date. + * + * The title takes whatever of the 255-byte NAME_MAX the date, the id and the + * suffix leave, rather than a flat character count: those are the same number + * only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS + * reject outright. The id never gives, so the name keeps addressing the + * document -- same rule as gcal's event filenames. + */ export function makeFilename(title: string, docId: string, modifiedTime = ''): string { - const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : '' - if (datePrefix !== '') { - return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gslide.json` - } - return `${sanitizeTitle(title)}__${docId}.gslide.json` + const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : '' + const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length + return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}` } diff --git a/typescript/packages/core/src/utils/sanitize.test.ts b/typescript/packages/core/src/utils/sanitize.test.ts index 31fc8977c..f63b891f4 100644 --- a/typescript/packages/core/src/utils/sanitize.test.ts +++ b/typescript/packages/core/src/utils/sanitize.test.ts @@ -13,7 +13,13 @@ // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= import { describe, expect, it } from 'vitest' -import { pathSafeName, sanitizeLabel, sanitizeName } from './sanitize.ts' +import { + NAME_MAX_BYTES, + byteLength, + pathSafeName, + sanitizeLabel, + sanitizeName, +} from './sanitize.ts' describe('sanitizeName', () => { it('returns "unknown" for empty/whitespace input', () => { @@ -98,13 +104,42 @@ describe('sanitizeLabel', () => { }) it('ellipsizes on a code-point boundary', () => { + // A byte budget wide enough to stay out of the way, so this pins the + // character budget alone. const label = '\u{10400}'.repeat(120) - const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100 }) + const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100, maxBytes: 10_000 }) expect(Array.from(out)).toHaveLength(100) expect(out.endsWith('...')).toBe(true) expect(out).not.toContain('\uFFFD') }) + it('honors the byte ceiling within the character budget', () => { + // 100 astral code points is 400 bytes, so a name the character budget + // accepts is one ext4 and APFS reject with ENAMETOOLONG. The default + // budget is NAME_MAX, and the cut still lands on a code-point boundary. + const label = '\u{10400}'.repeat(120) + const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100 }) + expect(Array.from(out).length).toBeLessThan(100) + expect(byteLength(out)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(out.endsWith('...')).toBe(true) + expect(out).not.toContain('\uFFFD') + }) + + it('takes the byte budget as the caller remaining room', () => { + // What the gdocs/gmail filenames pass: NAME_MAX minus the id, the + // separators and the suffix. + const out = sanitizeLabel('会'.repeat(200), { fallback: 'X', maxLen: 100, maxBytes: 60 }) + expect(byteLength(out)).toBeLessThanOrEqual(60) + expect(out.endsWith('...')).toBe(true) + expect(out).not.toContain('\uFFFD') + }) + + it('drops the ellipsis when it cannot fit', () => { + // Three dots and nothing is not a name; a budget this small yields + // whatever of the label actually fits. + expect(sanitizeLabel('abcdef', { fallback: 'X', maxLen: 100, maxBytes: 2 })).toBe('ab') + }) + it('keeps non-ascii letters, matching python', () => { // The per-backend copies this replaced used `\w`, which is ascii-only in // javascript, so a CJK title became a row of underscores while python -- diff --git a/typescript/packages/core/src/utils/sanitize.ts b/typescript/packages/core/src/utils/sanitize.ts index a53019295..48099ea22 100644 --- a/typescript/packages/core/src/utils/sanitize.ts +++ b/typescript/packages/core/src/utils/sanitize.ts @@ -20,10 +20,16 @@ const MAX_LEN = 100 // characters is the same number only for ASCII: a 100-character CJK title is // 300 bytes. export const NAME_MAX_BYTES = 255 +const ELLIPSIS = '...' const UTF8 = new TextEncoder() const UTF8_DECODER = new TextDecoder('utf-8') +/** Measure a string the way the filesystem does: in UTF-8 bytes. */ +export function byteLength(text: string): number { + return UTF8.encode(text).length +} + /** * Trim a string to fit a byte budget without splitting a character. * @@ -51,6 +57,13 @@ export function stripTrailingUnderscores(value: string): string { return value.slice(0, end) } +/** python's `str.rstrip("_.")`, so a byte cut cannot leave `Foo.....`. */ +function stripTrailingUnderscoresAndDots(value: string): string { + let end = value.length + while (end > 0 && (value[end - 1] === '_' || value[end - 1] === '.')) end -= 1 + return value.slice(0, end) +} + export function stripUnderscores(value: string): string { let start = 0 let end = value.length @@ -101,8 +114,20 @@ export function pathSafeName(name: string): string { * * Unlike `sanitizeName` this ellipsizes rather than hard-cutting, so a * truncated name reads as truncated. + * + * Two budgets apply, and both have to: `maxLen` is the readable length a + * backend wants, while `maxBytes` is what the filesystem will actually + * accept. They are the same number only for ASCII, so a 100-character CJK + * title passed a 100-character budget untouched and rendered a 300-byte + * filename, which ext4 and APFS reject with ENAMETOOLONG. Pass the bytes the + * *rest* of the filename does not already use -- see `makeFilename` in the + * gdocs/gsheets/gslides entries and `makeEventFilename` in gcal, which is + * where the fixed overhead is known. */ -export function sanitizeLabel(text: string, options: { fallback: string; maxLen: number }): string { +export function sanitizeLabel( + text: string, + options: { fallback: string; maxLen: number; maxBytes?: number }, +): string { if (text.trim() === '') return options.fallback let cleaned = text.replace(UNSAFE_CHARS, '_').replace(/ /g, '_').replace(MULTI_UNDERSCORE, '_') cleaned = stripUnderscores(cleaned) @@ -112,7 +137,14 @@ export function sanitizeLabel(text: string, options: { fallback: string; maxLen: // encodes as U+FFFD in the filename. const points = Array.from(cleaned) if (points.length > options.maxLen) { - cleaned = `${points.slice(0, options.maxLen - 3).join('')}...` + cleaned = `${points.slice(0, options.maxLen - ELLIPSIS.length).join('')}${ELLIPSIS}` + } + const maxBytes = options.maxBytes ?? NAME_MAX_BYTES + if (byteLength(cleaned) > maxBytes) { + const head = stripTrailingUnderscoresAndDots( + truncateBytes(cleaned, Math.max(maxBytes - ELLIPSIS.length, 0)), + ) + cleaned = head !== '' ? `${head}${ELLIPSIS}` : truncateBytes(cleaned, maxBytes) } return cleaned } diff --git a/typescript/packages/node/src/core/email/config.ts b/typescript/packages/node/src/core/email/config.ts index 43eacab5e..9c1807f89 100644 --- a/typescript/packages/node/src/core/email/config.ts +++ b/typescript/packages/node/src/core/email/config.ts @@ -68,18 +68,7 @@ export function buildEmailConfig(input: EmailConfigInput): EmailConfig { } export function normalizeEmailConfig(input: Record): EmailConfig { - const norm = normalizeFields(input, { - rename: { - imap_host: 'imapHost', - imap_port: 'imapPort', - smtp_host: 'smtpHost', - smtp_port: 'smtpPort', - use_ssl: 'useSsl', - max_messages: 'maxMessages', - save_copy: 'saveCopy', - sent_folder: 'sentFolder', - }, - }) + const norm = normalizeFields(input) const built: EmailConfigInput = { imapHost: asString(norm.imapHost), smtpHost: asString(norm.smtpHost), diff --git a/typescript/packages/node/src/core/email/readdir.ts b/typescript/packages/node/src/core/email/readdir.ts index c28081216..49bd1f747 100644 --- a/typescript/packages/node/src/core/email/readdir.ts +++ b/typescript/packages/node/src/core/email/readdir.ts @@ -18,6 +18,7 @@ import { PathSpec as PathSpecCtor } from '@struktoai/mirage-core/types' import type { PathSpec } from '@struktoai/mirage-core/types' import { enoent } from '@struktoai/mirage-core/utils/errors' import { mountKey, mountPrefixOf } from '@struktoai/mirage-core/utils/key_prefix' +import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '@struktoai/mirage-core/utils/sanitize' import { compareCodePoints } from '@struktoai/mirage-core/utils/sort' import type { EmailAccessor } from '../../accessor/email.ts' import { fetchHeaders, listMessageUids, type FetchedMessage } from './client.ts' @@ -26,20 +27,26 @@ import { messageJsonBytes } from './render.ts' import type { ParsedAttachment } from './_parse.ts' const TITLE_MAX = 80 -const UNSAFE = /[^\w\s\-.]/g -const MULTI_UNDERSCORE = /_+/g const EPOCH_DATE = '1970-01-01' +const MSG_SUFFIX = '.email.json' -export function sanitize(text: string): string { - if (text.trim() === '') return 'No_Subject' - let cleaned = text.replace(UNSAFE, '_').replace(/ /g, '_') - cleaned = cleaned.replace(MULTI_UNDERSCORE, '_').replace(/^_+|_+$/g, '') - if (cleaned.length > TITLE_MAX) cleaned = `${cleaned.slice(0, TITLE_MAX - 3)}...` - return cleaned -} +// Routed through the shared sanitizer rather than a local copy of it. The +// copy's `\w` was JS's ASCII-only one where python's is unicode, so every +// accented or CJK subject came back as a row of underscores here and intact +// there; it also measured the budget in UTF-16 units instead of code points. +const sanitize = (text: string, maxBytes?: number): string => + sanitizeLabel(text, { + fallback: 'No_Subject', + maxLen: TITLE_MAX, + ...(maxBytes !== undefined ? { maxBytes } : {}), + }) -function msgFilename(subject: string, uid: string): string { - return `${sanitize(subject)}__${uid}.email.json` +// 80 characters is 240 bytes of CJK, which overflows the 255-byte NAME_MAX +// once the uid and `.email.json` are added, so the subject takes what they +// leave rather than a flat character count. +export function msgFilename(subject: string, uid: string): string { + const fixed = 2 + byteLength(uid) + MSG_SUFFIX.length + return `${sanitize(subject, NAME_MAX_BYTES - fixed)}__${uid}${MSG_SUFFIX}` } // RFC 5322's obsolete zone names, the set `parsedate_to_datetime` knows. diff --git a/typescript/packages/node/src/core/email/search.test.ts b/typescript/packages/node/src/core/email/search.test.ts new file mode 100644 index 000000000..01c85af70 --- /dev/null +++ b/typescript/packages/node/src/core/email/search.test.ts @@ -0,0 +1,41 @@ +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= + +import { NAME_MAX_BYTES, byteLength } from '@struktoai/mirage-core/utils/sanitize' +import { describe, expect, it } from 'vitest' +import { msgFilename } from './readdir.ts' +import { buildVfsPath } from './search.ts' + +const CJK_SUBJECT = '会議の記録'.repeat(40) +const MSG = { subject: CJK_SUBJECT, uid: '7', date: 'Mon, 5 Jan 2026 10:00:00 +0000' } + +describe('email search paths', () => { + it('names the file readdir created', () => { + // Composed here from a bare `sanitize`, a hit pointed at a path that does + // not exist as soon as the subject was long enough to be trimmed: readdir + // budgets the subject against the uid and the suffix, and this did not, + // so the two names differed. + const path = buildVfsPath('/mail', 'INBOX', MSG as never) + expect(path.endsWith(`/${msgFilename(CJK_SUBJECT, '7')}`)).toBe(true) + }) + + it('fits NAME_MAX', () => { + const name = + buildVfsPath('/mail', 'INBOX', MSG as never) + .split('/') + .pop() ?? '' + expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES) + expect(name).not.toContain('\uFFFD') + }) +}) diff --git a/typescript/packages/node/src/core/email/search.ts b/typescript/packages/node/src/core/email/search.ts index be882e11a..295c5bded 100644 --- a/typescript/packages/node/src/core/email/search.ts +++ b/typescript/packages/node/src/core/email/search.ts @@ -14,7 +14,7 @@ import type { EmailAccessor } from '../../accessor/email.ts' import { fetchMessage, listMessageUids, type FetchedMessage } from './client.ts' -import { dateBucket, sanitize } from './readdir.ts' +import { dateBucket, msgFilename } from './readdir.ts' import { messageJsonText } from './render.ts' import type { EmailScope } from './scope.ts' @@ -62,10 +62,13 @@ async function searchMessages( return listMessageUids(accessor, folder, criteria, maxResults) } -function buildVfsPath(prefix: string, folder: string, msg: FetchedMessage): string { +export function buildVfsPath(prefix: string, folder: string, msg: FetchedMessage): string { const dateStr = dateBucket(msg) - const subject = sanitize(msg.subject !== '' ? msg.subject : 'No Subject') - const filename = `${subject}__${msg.uid}.email.json` + // The same builder readdir names the file with, not a second spelling of + // it: the subject's budget depends on the uid and the suffix, so a hit + // composed here from a bare `sanitize` pointed at a path that does not + // exist as soon as a long subject was trimmed differently. + const filename = msgFilename(msg.subject !== '' ? msg.subject : 'No Subject', msg.uid) return [prefix, folder, dateStr, filename].filter((p) => p !== '').join('/') } diff --git a/typescript/packages/node/src/resource/box/config.ts b/typescript/packages/node/src/resource/box/config.ts index 0b94a41df..699bfac49 100644 --- a/typescript/packages/node/src/resource/box/config.ts +++ b/typescript/packages/node/src/resource/box/config.ts @@ -70,13 +70,6 @@ export function redactBoxConfig(config: BoxConfig): BoxConfigRedacted { export function normalizeBoxConfig(input: Record): BoxConfig { return normalizeFields(input, { rename: { - root_folder_id: 'rootFolderId', - content_search: 'contentSearch', - client_id: 'clientId', - client_secret: 'clientSecret', - refresh_token: 'refreshToken', - enterprise_id: 'enterpriseId', - access_token: 'accessToken', developer_token: 'accessToken', }, }) as unknown as BoxConfig diff --git a/typescript/packages/node/src/resource/dropbox/config.ts b/typescript/packages/node/src/resource/dropbox/config.ts index 0ae52de35..c110e2948 100644 --- a/typescript/packages/node/src/resource/dropbox/config.ts +++ b/typescript/packages/node/src/resource/dropbox/config.ts @@ -47,13 +47,5 @@ export function redactDropboxConfig(config: DropboxConfig): DropboxConfigRedacte } export function normalizeDropboxConfig(input: Record): DropboxConfig { - return normalizeFields(input, { - rename: { - client_id: 'clientId', - client_secret: 'clientSecret', - refresh_token: 'refreshToken', - root_path: 'rootPath', - content_search: 'contentSearch', - }, - }) as unknown as DropboxConfig + return normalizeFields(input) as unknown as DropboxConfig } diff --git a/typescript/packages/node/src/resource/gcs/config.ts b/typescript/packages/node/src/resource/gcs/config.ts index 39d1a0304..e4fe35876 100644 --- a/typescript/packages/node/src/resource/gcs/config.ts +++ b/typescript/packages/node/src/resource/gcs/config.ts @@ -81,13 +81,9 @@ export function redactGcsConfig(config: GCSConfig): GCSConfigRedacted { export function normalizeGcsConfig(input: Record): GCSConfig { return normalizeFields(input, { rename: { - access_key_id: 'accessKeyId', - secret_access_key: 'secretAccessKey', - session_token: 'sessionToken', aws_profile: 'profile', endpoint_url: 'endpoint', path_style: 'forcePathStyle', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: { diff --git a/typescript/packages/node/src/resource/gridfs/config.ts b/typescript/packages/node/src/resource/gridfs/config.ts index 9d6e691a5..c47ba4005 100644 --- a/typescript/packages/node/src/resource/gridfs/config.ts +++ b/typescript/packages/node/src/resource/gridfs/config.ts @@ -39,12 +39,7 @@ export function redactConfig(config: GridFSConfig): GridFSConfigRedacted { * normalize the key prefix the way the accessor expects. */ export function normalizeGridFSConfig(input: Record): GridFSConfig { - const norm = normalizeFields(input, { - rename: { - key_prefix: 'keyPrefix', - chunk_size_bytes: 'chunkSizeBytes', - }, - }) as unknown as GridFSConfig + const norm = normalizeFields(input) as unknown as GridFSConfig const prefix = normalizeKeyPrefix(norm.keyPrefix) if (prefix !== undefined) { norm.keyPrefix = prefix diff --git a/typescript/packages/node/src/resource/hf_buckets/config.ts b/typescript/packages/node/src/resource/hf_buckets/config.ts index 708cbf120..73b966018 100644 --- a/typescript/packages/node/src/resource/hf_buckets/config.ts +++ b/typescript/packages/node/src/resource/hf_buckets/config.ts @@ -59,7 +59,6 @@ export function redactHfBucketsConfig(config: HfBucketsConfig): HfBucketsConfigR export function normalizeHfBucketsConfig(input: Record): HfBucketsConfig { const config = normalizeFields(input, { rename: { - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: { @@ -102,8 +101,6 @@ export function redactHfRepoConfig(config: HfRepoConfig): HfRepoConfigRedacted { export function normalizeHfRepoConfig(input: Record): HfRepoConfig { const config = normalizeFields(input, { rename: { - repo_id: 'repoId', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: { diff --git a/typescript/packages/node/src/resource/jaeger/config.ts b/typescript/packages/node/src/resource/jaeger/config.ts index 331898e38..370bf2585 100644 --- a/typescript/packages/node/src/resource/jaeger/config.ts +++ b/typescript/packages/node/src/resource/jaeger/config.ts @@ -34,12 +34,5 @@ export function redactJaegerConfig(config: JaegerConfig): JaegerConfigRedacted { } export function normalizeJaegerConfig(input: Record): JaegerConfig { - return normalizeFields(input, { - rename: { - default_trace_limit: 'defaultTraceLimit', - default_from_timestamp: 'defaultFromTimestamp', - default_to_timestamp: 'defaultToTimestamp', - request_timeout: 'requestTimeout', - }, - }) as unknown as JaegerConfig + return normalizeFields(input) as unknown as JaegerConfig } diff --git a/typescript/packages/node/src/resource/langfuse/config.ts b/typescript/packages/node/src/resource/langfuse/config.ts index 580c82d8f..d497c6f5c 100644 --- a/typescript/packages/node/src/resource/langfuse/config.ts +++ b/typescript/packages/node/src/resource/langfuse/config.ts @@ -34,13 +34,5 @@ export function redactLangfuseConfig(config: LangfuseConfig): LangfuseConfigReda } export function normalizeLangfuseConfig(input: Record): LangfuseConfig { - return normalizeFields(input, { - rename: { - public_key: 'publicKey', - secret_key: 'secretKey', - default_trace_limit: 'defaultTraceLimit', - default_search_limit: 'defaultSearchLimit', - default_from_timestamp: 'defaultFromTimestamp', - }, - }) as unknown as LangfuseConfig + return normalizeFields(input) as unknown as LangfuseConfig } diff --git a/typescript/packages/node/src/resource/oci/config.ts b/typescript/packages/node/src/resource/oci/config.ts index d4a60fd04..9aa7ca97d 100644 --- a/typescript/packages/node/src/resource/oci/config.ts +++ b/typescript/packages/node/src/resource/oci/config.ts @@ -83,12 +83,8 @@ export function redactOciConfig(config: OCIConfig): OCIConfigRedacted { export function normalizeOciConfig(input: Record): OCIConfig { return normalizeFields(input, { rename: { - access_key_id: 'accessKeyId', - secret_access_key: 'secretAccessKey', - session_token: 'sessionToken', aws_profile: 'profile', endpoint_url: 'endpoint', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: { diff --git a/typescript/packages/node/src/resource/r2/config.ts b/typescript/packages/node/src/resource/r2/config.ts index 036046df6..6e4741697 100644 --- a/typescript/packages/node/src/resource/r2/config.ts +++ b/typescript/packages/node/src/resource/r2/config.ts @@ -89,14 +89,9 @@ export function redactR2Config(config: R2Config): R2ConfigRedacted { export function normalizeR2Config(input: Record): R2Config { return normalizeFields(input, { rename: { - account_id: 'accountId', - access_key_id: 'accessKeyId', - secret_access_key: 'secretAccessKey', - session_token: 'sessionToken', aws_profile: 'profile', endpoint_url: 'endpoint', path_style: 'forcePathStyle', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: { diff --git a/typescript/packages/node/src/resource/s3/config.ts b/typescript/packages/node/src/resource/s3/config.ts index 3a3ef0074..d891ff783 100644 --- a/typescript/packages/node/src/resource/s3/config.ts +++ b/typescript/packages/node/src/resource/s3/config.ts @@ -68,7 +68,6 @@ export function normalizeS3Config(input: Record): S3Config { endpoint_url: 'endpoint', path_style: 'forcePathStyle', timeout: 'timeoutMs', - key_prefix: 'keyPrefix', }, transform: { timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v), diff --git a/typescript/packages/node/src/resource/s3_alias.ts b/typescript/packages/node/src/resource/s3_alias.ts index c9faae70f..4149601f5 100644 --- a/typescript/packages/node/src/resource/s3_alias.ts +++ b/typescript/packages/node/src/resource/s3_alias.ts @@ -52,16 +52,14 @@ export interface S3AliasConfig { proxy?: string } -// Every provider accepts the same snake_case spelling of its fields, and -// Python states timeouts in seconds where TypeScript uses milliseconds. +// Only the entries `snakeToCamel` would get wrong: python spells the profile +// `aws_profile`, the endpoint `endpoint_url`, path style `path_style`, and +// states timeouts in seconds where TypeScript uses milliseconds. Every other +// field maps by default, so it does not belong here. const RENAME: Record = { - access_key_id: 'accessKeyId', - secret_access_key: 'secretAccessKey', - session_token: 'sessionToken', aws_profile: 'profile', endpoint_url: 'endpoint', path_style: 'forcePathStyle', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', } diff --git a/typescript/packages/node/src/resource/ssh/config.ts b/typescript/packages/node/src/resource/ssh/config.ts index 66a313465..ff5eeb2d9 100644 --- a/typescript/packages/node/src/resource/ssh/config.ts +++ b/typescript/packages/node/src/resource/ssh/config.ts @@ -38,10 +38,5 @@ export function redactSshConfig(config: SSHConfig): SSHConfigRedacted { } export function normalizeSshConfig(input: Record): SSHConfig { - return normalizeFields(input, { - rename: { - identity_file: 'identityFile', - known_hosts: 'knownHosts', - }, - }) as unknown as SSHConfig + return normalizeFields(input) as unknown as SSHConfig } diff --git a/typescript/packages/node/src/resource/supabase/config.ts b/typescript/packages/node/src/resource/supabase/config.ts index fe253e88f..2ecec632e 100644 --- a/typescript/packages/node/src/resource/supabase/config.ts +++ b/typescript/packages/node/src/resource/supabase/config.ts @@ -86,13 +86,8 @@ export function redactSupabaseConfig(config: SupabaseConfig): SupabaseConfigReda export function normalizeSupabaseConfig(input: Record): SupabaseConfig { return normalizeFields(input, { rename: { - project_ref: 'projectRef', - access_key_id: 'accessKeyId', - secret_access_key: 'secretAccessKey', - session_token: 'sessionToken', aws_profile: 'profile', endpoint_url: 'endpoint', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: { diff --git a/typescript/packages/node/src/resource/trello/config.ts b/typescript/packages/node/src/resource/trello/config.ts index 918d38ec6..ee72e639a 100644 --- a/typescript/packages/node/src/resource/trello/config.ts +++ b/typescript/packages/node/src/resource/trello/config.ts @@ -33,13 +33,5 @@ export function redactTrelloConfig(config: TrelloConfig): TrelloConfigRedacted { } export function normalizeTrelloConfig(input: Record): TrelloConfig { - return normalizeFields(input, { - rename: { - api_key: 'apiKey', - api_token: 'apiToken', - workspace_id: 'workspaceId', - board_ids: 'boardIds', - base_url: 'baseUrl', - }, - }) as unknown as TrelloConfig + return normalizeFields(input) as unknown as TrelloConfig } diff --git a/typescript/packages/node/src/resource/wasabi/config.ts b/typescript/packages/node/src/resource/wasabi/config.ts index ee35e4686..4129fdce3 100644 --- a/typescript/packages/node/src/resource/wasabi/config.ts +++ b/typescript/packages/node/src/resource/wasabi/config.ts @@ -85,13 +85,9 @@ export function redactWasabiConfig(config: WasabiConfig): WasabiConfigRedacted { export function normalizeWasabiConfig(input: Record): WasabiConfig { return normalizeFields(input, { rename: { - access_key_id: 'accessKeyId', - secret_access_key: 'secretAccessKey', - session_token: 'sessionToken', aws_profile: 'profile', endpoint_url: 'endpoint', path_style: 'forcePathStyle', - key_prefix: 'keyPrefix', timeout: 'timeoutMs', }, transform: {