fix(parity): github ref default, NAME_MAX byte budget, browser registry onto node's config shape

Two py<->ts divergences, one structural alignment, one swept-and-gated class
of config noise.

1. GitHub `ref` defaulted to the literal "main" in python where TypeScript
   resolves `config.ref ?? repoInfo.default_branch`, so every repository whose
   default branch is something else 404d on the one tree fetch the mount is
   built on and read as empty. `GitHubConfig.ref` is now optional, and
   `ensure_ref` resolves it lazily beside the existing `ensure_default_branch`
   -- python's constructor deliberately does no network. Every reader that
   needs a concrete ref goes through it. `mirage.accessor.github` is fully
   annotated and leaves mypy's untyped-defs allowlist.

2. `sanitize_label` capped characters where NAME_MAX counts bytes, which the
   constant's own comment already spelled out. gdocs/gsheets/gslides rendered
   ~367-byte filenames for a CJK title and gmail/email ~269; ext4 and APFS
   reject those. `sanitize_label`/`sanitizeLabel` grew a byte budget and every
   compose site passes what the date, id and suffix leave. Found alongside: TS
   email's hand-rolled copy of the sanitizer used JS's ASCII-only `\w` (so
   non-ASCII subjects became underscores there and stayed intact in python),
   and three sites re-composed `<subject>__<id>.<ext>` independently of
   readdir, so a search hit could name a path that does not exist.

3. The browser registry hand-rolled `normalizeFields` per entry and cast
   through 7 `*BrowserCtorConfig` interfaces -- hand-written twins of the real
   config types. It now delegates to a `normalizeXConfig` beside each config,
   as node's registry does; the interfaces and 22 double casts are gone.

4. 95 rename entries that only restated `snakeToCamel` deleted across 24
   files, with `scripts/check_normalize_renames.py` and a CI step to keep them
   from returning.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
bytecii
2026-08-20 02:34:03 -07:00
parent f5b2e1e677
commit ca89dc5970
96 changed files with 1265 additions and 434 deletions
+8
View File
@@ -93,6 +93,14 @@ jobs:
- name: Core barrel surface
run: ./python/.venv/bin/python scripts/check_barrel_surface.py
# `normalizeFields` already maps every unlisted key through
# `snakeToCamel`, so a rename entry that restates that mapping is not
# configuration -- it is a second place for one fact to be wrong, and it
# buries the handful that are real overrides (`endpoint_url`, `timeout`,
# `aws_profile`) in a wall of noise. 95 had accumulated.
- name: Redundant config renames
run: ./python/.venv/bin/python scripts/check_normalize_renames.py
# The docs build parses frontmatter as YAML and stops on the first page
# it cannot read, so an unquoted description holding ": " fails the
# deploy with no local signal at all.
+10 -6
View File
@@ -15,23 +15,27 @@
import asyncio
from mirage.accessor.base import Accessor
from mirage.core.github.config import GitHubConfig
from mirage.core.github.tree_entry import TreeEntry
class GitHubAccessor(Accessor):
def __init__(self,
config,
owner,
repo,
ref,
config: GitHubConfig,
owner: str,
repo: str,
ref: str | None = None,
default_branch: str | None = None,
tree: dict[str, TreeEntry] | None = None,
truncated=False):
truncated: bool = False) -> None:
self.config = config
self.owner = owner
self.repo = repo
self.ref = ref
# None until resolved: an unpinned mount follows the repository's
# default branch, which costs a request to learn, so the ref is
# settled on the first read that needs one (`ensure_ref`).
self.ref: str | None = ref
# None until hydrated: the mount is constructed without touching
# the network, so the repo's default branch is fetched on the
# first read that needs it (`ensure_default_branch`).
+6 -3
View File
@@ -27,7 +27,7 @@ from mirage.commands.registry import command
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagView
from mirage.core.email.client import fetch_headers
from mirage.core.email.readdir import _date_bucket, _sanitize
from mirage.core.email.readdir import _date_bucket, _msg_filename
from mirage.core.email.readdir import readdir as _readdir
from mirage.core.email.search import search_messages
from mirage.core.email.stat import stat as _stat
@@ -163,9 +163,12 @@ async def _find_server_side(
results: list[str] = []
for h in headers:
date_str = _date_bucket(h)
subject = _sanitize(h.get("subject", "No Subject"))
uid = h.get("uid", "")
filename = f"{subject}__{uid}.email.json"
# The same builder readdir names the file with, not a second
# spelling of it: the subject's budget depends on the uid and the
# suffix, so a hit composed from a bare `_sanitize` pointed at a
# path that does not exist once a long subject was trimmed.
filename = _msg_filename(h.get("subject", "No Subject"), uid)
if fnmatch(filename, name_pattern):
vfs_path = "/".join(p
for p in [prefix, folder, date_str, filename]
@@ -20,7 +20,7 @@ from mirage.commands.builtin.grep_helper import (is_literal_pattern,
from mirage.core.github.constants import SCOPE_WARN
from mirage.core.github.pushdown import (count_scope_files, scope_relative_key,
should_use_search)
from mirage.core.github.repo import ensure_default_branch
from mirage.core.github.repo import ensure_default_branch, ensure_ref
from mirage.core.github.search import narrow_paths
from mirage.core.github.tree import ensure_tree
from mirage.types import PathSpec
@@ -87,7 +87,7 @@ async def narrow_scope(
use_search = (query is not None and whole_word and literal
and file_count > SCOPE_WARN and should_use_search(
recursive=recursive,
on_default_branch=(accessor.ref == await
on_default_branch=(await ensure_ref(accessor) == await
ensure_default_branch(accessor)),
))
if use_search:
+8 -2
View File
@@ -25,16 +25,22 @@ from mirage.core.email.render import message_json_bytes
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
from mirage.utils.sanitize import sanitize_label
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
TITLE_MAX = 80
EPOCH_DATE = "1970-01-01"
MSG_SUFFIX = ".email.json"
_sanitize = partial(sanitize_label, fallback="No_Subject", max_len=TITLE_MAX)
def _msg_filename(subject: str, uid: str) -> str:
return f"{_sanitize(subject)}__{uid}.email.json"
# 80 characters is 240 bytes of CJK, which overflows the 255-byte
# NAME_MAX once the uid and `.email.json` are added, so the subject
# takes what they leave rather than a flat character count.
fixed = len("__") + byte_len(uid) + len(MSG_SUFFIX)
label = _sanitize(subject, max_bytes=NAME_MAX_BYTES - fixed)
return f"{label}__{uid}{MSG_SUFFIX}"
def _parse_date(value: str) -> str | None:
+6 -3
View File
@@ -16,7 +16,7 @@ from typing import Any
from mirage.accessor.email import EmailAccessor
from mirage.core.email.client import fetch_message, list_message_uids
from mirage.core.email.readdir import _date_bucket, _sanitize
from mirage.core.email.readdir import _date_bucket, _msg_filename
from mirage.core.email.render import message_json_text
from mirage.core.email.scope import EmailScope
@@ -77,9 +77,12 @@ async def search_messages(
def _build_vfs_path(prefix: str, folder: str, msg: dict[str, Any]) -> str:
date_str = _date_bucket(msg)
subject = _sanitize(msg.get("subject", "No Subject"))
uid = msg.get("uid", "")
filename = f"{subject}__{uid}.email.json"
# The same builder readdir names the file with, not a second spelling of
# it: the subject's budget depends on the uid and the suffix, so a hit
# composed here from a bare `_sanitize` pointed at a path that does not
# exist as soon as a long subject was trimmed differently.
filename = _msg_filename(msg.get("subject", "No Subject"), uid)
parts = [prefix, folder, date_str, filename]
return "/".join(p for p in parts if p)
+5 -1
View File
@@ -19,7 +19,11 @@ class GitHubConfig(BaseModel):
token: SecretStr
owner: str | None = None
repo: str | None = None
ref: str = "main"
# None means "whatever the repository's default branch is", resolved on
# the first read through ``ensure_ref``. It is not a synonym for "main":
# defaulting to that string mounted a nonexistent ref on every repo whose
# default is `master`, and the tree fetch 404s rather than falling back.
ref: str | None = None
base_url: str | None = None
+2 -1
View File
@@ -17,6 +17,7 @@ import logging
from mirage.accessor.github import GitHubAccessor
from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
LookupStatus)
from mirage.core.github.repo import ensure_ref
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
refill_index)
from mirage.types import PathSpec
@@ -120,7 +121,7 @@ async def _resolve_dir_sha(
stem = prefix.rstrip("/")
rest = norm[len(stem):] if stem and norm.startswith(stem) else norm
parts = [p for p in rest.strip("/").split("/") if p]
current_sha = accessor.ref
current_sha = await ensure_ref(accessor)
current_path = stem
for part in parts:
entries = await fetch_dir_tree(accessor.config, accessor.owner,
+28
View File
@@ -61,6 +61,34 @@ async def ensure_default_branch(accessor: GitHubAccessor, ) -> str:
return accessor.default_branch
async def ensure_ref(accessor: GitHubAccessor) -> str:
"""Settle which ref this mount reads, fetching the default branch once.
A mount that named no ref follows the repository's default branch, and
learning that costs a request the constructor cannot make. Every reader
that needs a concrete ref -- the tree fetches, the watch walk, readdir's
per-directory descent -- goes through here instead of reading
``accessor.ref`` directly, so an unpinned mount resolves exactly once and
then behaves like a pinned one.
Defaulting to the string ``"main"`` instead was the bug this replaces: a
repository whose default branch is ``master`` (or anything else) 404s on
every tree fetch, so the whole mount reads as empty.
Args:
accessor (GitHubAccessor): the mount's accessor.
Returns:
str: the ref to read, as named by the mount or as resolved from the
repository's default branch.
"""
if accessor.ref is not None:
return accessor.ref
resolved = await ensure_default_branch(accessor)
accessor.ref = resolved
return resolved
def parse_repo(spec: str) -> RepoRef:
"""Split gh's `[HOST/]OWNER/REPO`.
+5 -2
View File
@@ -22,6 +22,7 @@ from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
LookupStatus)
from mirage.core.github.client import github_get
from mirage.core.github.config import GitHubConfig
from mirage.core.github.repo import ensure_ref
from mirage.core.github.tree_entry import TreeEntry
log = logging.getLogger(__name__)
@@ -201,8 +202,9 @@ async def refill_index(
"""
if index is NULL_INDEX:
return False
ref = await ensure_ref(accessor)
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
accessor.repo, accessor.ref)
accessor.repo, ref)
accessor.truncated = truncated
accessor.tree = tree
accessor.tree_loaded = True
@@ -301,8 +303,9 @@ async def ensure_tree(
await ensure_live_index(accessor, index, prefix)
if accessor.tree_loaded:
return
ref = await ensure_ref(accessor)
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
accessor.repo, accessor.ref)
accessor.repo, ref)
accessor.truncated = truncated
accessor.tree = tree
accessor.tree_loaded = True
+4 -2
View File
@@ -15,6 +15,7 @@
from collections.abc import AsyncIterator
from mirage.accessor.github import GitHubAccessor
from mirage.core.github.repo import ensure_ref
from mirage.core.github.tree import fetch_tree
from mirage.types import PathSpec, WalkEntry
from mirage.utils.key_prefix import mount_prefix_of
@@ -58,12 +59,13 @@ class GitHubWalk:
"""
accessor = self._accessor
prefix = mount_prefix_of(root.virtual, root.resource_path)
ref = await ensure_ref(accessor)
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
accessor.repo, accessor.ref)
accessor.repo, ref)
if truncated:
raise IncompleteWalkError(
f"github tree for {accessor.owner}/{accessor.repo}"
f"@{accessor.ref} was truncated; cannot diff a partial tree")
f"@{ref} was truncated; cannot diff a partial tree")
# A complete tree for the ref is exactly what the accessor holds,
# and find/du/grep's scope counter read it directly. Discarding it
# here left them answering from the tree the mount was built with
+25 -3
View File
@@ -27,21 +27,43 @@ from mirage.core.gmail.messages import (_extract_attachments, _extract_header,
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
from mirage.utils.sanitize import sanitize_label
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
logger = logging.getLogger(__name__)
TITLE_MAX = 80
MSG_SUFFIX = ".gmail.json"
_sanitize = partial(sanitize_label, fallback="No_Subject", max_len=TITLE_MAX)
def _subject(subject: str, msg_id: str) -> str:
"""Sanitize a subject to fit what the id and suffix leave of NAME_MAX.
80 characters is 240 bytes of CJK, which overflows the 255-byte
NAME_MAX once the id and `.gmail.json` are added; the filesystem
rejects the name outright. Both the message file and its attachment
directory take the file's (stricter) budget so one subject renders
the same in both, rather than the directory getting the eleven bytes
the suffix would have used.
Args:
subject (str): raw subject header.
msg_id (str): the Gmail message id the name embeds.
Returns:
str: the sanitized subject segment.
"""
fixed = len("__") + byte_len(msg_id) + len(MSG_SUFFIX)
return _sanitize(subject, max_bytes=NAME_MAX_BYTES - fixed)
def _msg_filename(subject: str, msg_id: str) -> str:
return f"{_sanitize(subject)}__{msg_id}.gmail.json"
return f"{_subject(subject, msg_id)}__{msg_id}{MSG_SUFFIX}"
def _attach_dir_name(subject: str, msg_id: str) -> str:
return f"{_sanitize(subject)}__{msg_id}"
return f"{_subject(subject, msg_id)}__{msg_id}"
def _attachment_filename(_attachment_id: str, filename: str) -> str:
+6 -3
View File
@@ -18,7 +18,7 @@ from typing import Any
from mirage.core.gmail.messages import (_decode_body, _extract_header,
get_message_processed, get_message_raw,
list_messages)
from mirage.core.gmail.readdir import _sanitize
from mirage.core.gmail.readdir import _msg_filename
from mirage.core.gmail.scope import GmailScope
from mirage.core.google.client import TokenManager
@@ -116,8 +116,11 @@ def format_grep_results(
label = row.get("label") or scope.label_name or "INBOX"
date = row.get("date", "")
mid = row.get("id", "")
subject_clean = _sanitize(row.get("subject") or "No Subject")
filename = f"{subject_clean}__{mid}.gmail.json"
# The same builder readdir names the file with, not a second
# spelling of it: the subject's budget depends on the id and the
# suffix, so a hit composed from a bare `_sanitize` pointed at a
# path that does not exist once a long subject was trimmed.
filename = _msg_filename(row.get("subject") or "No Subject", mid)
sender = row.get("sender", "?")
haystack = f"{row.get('subject', '')}\n{row.get('body_text', '')}"
excerpt = _extract_excerpt(haystack, pattern) if pattern else ""
+3 -2
View File
@@ -13,7 +13,8 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.utils.naming import make_id_name
from mirage.utils.sanitize import NAME_MAX_BYTES, sanitize_name, truncate_bytes
from mirage.utils.sanitize import (NAME_MAX_BYTES, byte_len, sanitize_name,
truncate_bytes)
EVENT_SUFFIX = ".gcal.json"
CALENDAR_FILE = "calendar.json"
@@ -58,7 +59,7 @@ def make_event_filename(event_id: str, hhmm: str, title: str) -> str:
Returns:
str: e.g. ``la9i1t9...__0900-1030_PhD_Defense.gcal.json``.
"""
fixed = len(event_id.encode()) + len("__") + len(hhmm) + len("_") + len(
fixed = byte_len(event_id) + len("__") + len(hhmm) + len("_") + len(
EVENT_SUFFIX)
trimmed = truncate_bytes(title, NAME_MAX_BYTES - fixed).rstrip("_")
if not trimmed:
+14 -5
View File
@@ -15,9 +15,11 @@
from dataclasses import dataclass
from functools import partial
from mirage.utils.sanitize import sanitize_label
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
TITLE_MAX_CHARS = 100
SUFFIX = ".gdoc.json"
DATE_LEN = 10
@dataclass
@@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label,
def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
"""Build a filename from title, doc ID, and modified date.
The title takes whatever of the 255-byte NAME_MAX the date, the id and
the suffix leave, rather than a flat character count: those are the same
number only for ASCII, and a 100-character CJK title rendered a name ext4
and APFS reject outright. The id never gives, so the name keeps
addressing the document -- same rule as gcal's event filenames.
Args:
title (str): raw document title.
doc_id (str): Google Docs document ID.
@@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
Returns:
str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json".
"""
date_prefix = modified_time[:10] if len(modified_time) >= 10 else ""
if date_prefix:
return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gdoc.json"
return f"{sanitize_title(title)}__{doc_id}.gdoc.json"
lead = (f"{modified_time[:DATE_LEN]}_"
if len(modified_time) >= DATE_LEN else "")
fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX)
label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed)
return f"{lead}{label}__{doc_id}{SUFFIX}"
+11 -3
View File
@@ -81,7 +81,9 @@ class GitHubResource(BaseResource):
repo (str | None): repository name; falls back to
``config.repo``.
ref (str | None): branch, tag or commit the mount is pinned
to; falls back to ``config.ref``.
to; falls back to ``config.ref``, and when neither names
one the mount follows the repository's default branch,
resolved on first read by ``ensure_ref``.
default_branch (str | None): the repo's default branch, for
``is_default_branch``. Fetched on first use when None.
tree (dict[str, TreeEntry] | None): the recursive git tree,
@@ -130,8 +132,12 @@ class GitHubResource(BaseResource):
def is_default_branch(self) -> bool | None:
"""Whether the mount is pinned to the repo's default branch.
``None`` means not known yet, not "no": the default branch is
fetched on first use, and until something calls
An unpinned mount answers True without a request: naming no ref
*means* following the default branch, so the two agree whatever
that branch turns out to be.
Otherwise ``None`` means not known yet, not "no": the default
branch is fetched on first use, and until something calls
:func:`mirage.core.github.repo.ensure_default_branch` there is
nothing to compare ``ref`` against. Answering ``False`` there
would be a wrong answer rather than an absent one, and an
@@ -146,6 +152,8 @@ class GitHubResource(BaseResource):
Returns:
bool | None: the comparison, or None if not yet hydrated.
"""
if self.accessor.ref is None:
return True
if self.accessor.default_branch is None:
return None
return self.accessor.ref == self.accessor.default_branch
+14 -5
View File
@@ -15,9 +15,11 @@
from dataclasses import dataclass
from functools import partial
from mirage.utils.sanitize import sanitize_label
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
TITLE_MAX_CHARS = 100
SUFFIX = ".gsheet.json"
DATE_LEN = 10
@dataclass
@@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label,
def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
"""Build a filename from title, doc ID, and modified date.
The title takes whatever of the 255-byte NAME_MAX the date, the id and
the suffix leave, rather than a flat character count: those are the same
number only for ASCII, and a 100-character CJK title rendered a name ext4
and APFS reject outright. The id never gives, so the name keeps
addressing the document -- same rule as gcal's event filenames.
Args:
title (str): raw document title.
doc_id (str): Google Sheets spreadsheet ID.
@@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
Returns:
str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json".
"""
date_prefix = modified_time[:10] if len(modified_time) >= 10 else ""
if date_prefix:
return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gsheet.json"
return f"{sanitize_title(title)}__{doc_id}.gsheet.json"
lead = (f"{modified_time[:DATE_LEN]}_"
if len(modified_time) >= DATE_LEN else "")
fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX)
label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed)
return f"{lead}{label}__{doc_id}{SUFFIX}"
+14 -5
View File
@@ -15,9 +15,11 @@
from dataclasses import dataclass
from functools import partial
from mirage.utils.sanitize import sanitize_label
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
TITLE_MAX_CHARS = 100
SUFFIX = ".gslide.json"
DATE_LEN = 10
@dataclass
@@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label,
def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
"""Build a filename from title, doc ID, and modified date.
The title takes whatever of the 255-byte NAME_MAX the date, the id and
the suffix leave, rather than a flat character count: those are the same
number only for ASCII, and a 100-character CJK title rendered a name ext4
and APFS reject outright. The id never gives, so the name keeps
addressing the document -- same rule as gcal's event filenames.
Args:
title (str): raw document title.
doc_id (str): Google Slides presentation ID.
@@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
Returns:
str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json".
"""
date_prefix = modified_time[:10] if len(modified_time) >= 10 else ""
if date_prefix:
return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gslide.json"
return f"{sanitize_title(title)}__{doc_id}.gslide.json"
lead = (f"{modified_time[:DATE_LEN]}_"
if len(modified_time) >= DATE_LEN else "")
fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX)
label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed)
return f"{lead}{label}__{doc_id}{SUFFIX}"
+39 -2
View File
@@ -21,6 +21,19 @@ MAX_LEN = 100
# characters is the same number only for ASCII: a 100-character CJK title is
# 300 bytes.
NAME_MAX_BYTES = 255
ELLIPSIS = "..."
def byte_len(text: str) -> int:
"""Measure a string the way the filesystem does.
Args:
text (str): the string to measure.
Returns:
int: the length of ``text`` in UTF-8 bytes.
"""
return len(text.encode("utf-8"))
def truncate_bytes(text: str, budget: int) -> str:
@@ -88,7 +101,11 @@ def path_safe_name(name: str) -> str:
return name.replace("/", "")
def sanitize_label(text: str, *, fallback: str, max_len: int) -> str:
def sanitize_label(text: str,
*,
fallback: str,
max_len: int,
max_bytes: int = NAME_MAX_BYTES) -> str:
"""Sanitize an API-supplied label for use inside a filename.
The shared body behind every backend's title/subject sanitizer:
@@ -100,11 +117,24 @@ def sanitize_label(text: str, *, fallback: str, max_len: int) -> str:
Unlike ``sanitize_name`` this ellipsizes rather than hard-cutting, so
a truncated name reads as truncated.
Two budgets apply, and both have to: ``max_len`` is the readable
length a backend wants, while ``max_bytes`` is what the filesystem
will actually accept. They are the same number only for ASCII, so a
100-character CJK title passed a 100-character budget untouched and
rendered a 300-byte filename, which ext4 and APFS reject with
ENAMETOOLONG. Pass the bytes the *rest* of the filename does not
already use -- see ``make_filename`` in the gdocs/gsheets/gslides
entries and ``make_event_filename`` in gcal, which is where the
fixed overhead is known.
Args:
text (str): raw label from the API.
fallback (str): what an empty or whitespace-only label becomes.
max_len (int): budget in characters; a longer label keeps its
first ``max_len - 3`` characters plus an ellipsis.
max_bytes (int): budget in UTF-8 bytes for the label alone. A
budget too small to hold even the ellipsis yields a bare
truncation rather than three dots and nothing.
Returns:
str: the sanitized label.
@@ -114,5 +144,12 @@ def sanitize_label(text: str, *, fallback: str, max_len: int) -> str:
cleaned = UNSAFE_CHARS.sub("_", text).replace(" ", "_")
cleaned = MULTI_UNDERSCORE.sub("_", cleaned).strip("_")
if len(cleaned) > max_len:
cleaned = cleaned[:max_len - 3] + "..."
cleaned = cleaned[:max_len - len(ELLIPSIS)] + ELLIPSIS
if byte_len(cleaned) > max_bytes:
head = truncate_bytes(cleaned, max(max_bytes - len(ELLIPSIS), 0))
trimmed = head.rstrip("_.")
if trimmed:
cleaned = trimmed + ELLIPSIS
else:
cleaned = truncate_bytes(cleaned, max_bytes)
return cleaned
-1
View File
@@ -311,7 +311,6 @@ module = [
"mirage.accessor._hf",
"mirage.accessor.gdocs",
"mirage.accessor.gdrive",
"mirage.accessor.github",
"mirage.accessor.gmail",
"mirage.accessor.gsheets",
"mirage.accessor.gslides",
+44
View File
@@ -0,0 +1,44 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.core.email.readdir import _msg_filename
from mirage.core.email.search import _build_vfs_path
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
CJK_SUBJECT = "会議の記録" * 40
def test_a_hit_names_the_file_readdir_created():
# Composed here from a bare `_sanitize`, a hit pointed at a path that
# does not exist as soon as the subject was long enough to be trimmed:
# readdir budgets the subject against the uid and the suffix, and this
# did not, so the two names differed.
msg = {
"subject": CJK_SUBJECT,
"uid": "7",
"date": "Mon, 5 Jan 2026 10:00:00 +0000"
}
path = _build_vfs_path("/mail", "INBOX", msg)
assert path.endswith("/" + _msg_filename(CJK_SUBJECT, "7"))
def test_a_hits_filename_fits_name_max():
msg = {
"subject": CJK_SUBJECT,
"uid": "7",
"date": "Mon, 5 Jan 2026 10:00:00 +0000"
}
name = _build_vfs_path("/mail", "INBOX", msg).rsplit("/", 1)[-1]
assert byte_len(name) <= NAME_MAX_BYTES
assert "\ufffd" not in name
+36 -1
View File
@@ -12,12 +12,15 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
from unittest.mock import patch
import pytest
from mirage.accessor.github import GitHubAccessor
from mirage.core.github.config import GitHubConfig
from mirage.core.github.repo import fetch_default_branch, parse_repo
from mirage.core.github.repo import (ensure_ref, fetch_default_branch,
parse_repo)
@pytest.fixture
@@ -64,3 +67,35 @@ def test_parse_repo_drops_the_optional_host():
def test_parse_repo_refuses_a_spec_that_is_not_the_format(spec):
with pytest.raises(ValueError, match="OWNER/REPO"):
parse_repo(spec)
@pytest.mark.asyncio
@patch("mirage.core.github.repo.github_get")
async def test_ensure_ref_resolves_the_default_branch_when_none_was_named(
mock_get, config):
mock_get.return_value = {"default_branch": "master"}
accessor = GitHubAccessor(config, "acme", "proj")
assert await ensure_ref(accessor) == "master"
# Settled on the accessor, so the next reader neither refetches nor
# disagrees with the ref this one read.
assert accessor.ref == "master"
@pytest.mark.asyncio
@patch("mirage.core.github.repo.github_get")
async def test_ensure_ref_keeps_a_pinned_ref_without_a_request(
mock_get, config):
accessor = GitHubAccessor(config, "acme", "proj", "release-2")
assert await ensure_ref(accessor) == "release-2"
mock_get.assert_not_awaited()
@pytest.mark.asyncio
@patch("mirage.core.github.repo.github_get")
async def test_ensure_ref_resolves_once_for_concurrent_readers(
mock_get, config):
mock_get.return_value = {"default_branch": "trunk"}
accessor = GitHubAccessor(config, "acme", "proj")
refs = await asyncio.gather(*(ensure_ref(accessor) for _ in range(4)))
assert refs == ["trunk"] * 4
assert mock_get.await_count == 1
+34 -2
View File
@@ -21,8 +21,8 @@ from mirage.accessor.github import GitHubAccessor
from mirage.cache.index import NULL_INDEX
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.core.github.config import GitHubConfig
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
fetch_tree, index_rows)
from mirage.core.github.tree import (ensure_live_index, ensure_tree,
fetch_dir_tree, fetch_tree, index_rows)
from mirage.core.github.tree_entry import TreeEntry
@@ -251,3 +251,35 @@ def test_index_rows_root_mount_keeps_bare_paths():
def test_index_rows_gives_an_empty_repo_a_root_row():
_entries, children = index_rows({}, "/gh")
assert children == {"/gh": []}
@pytest.mark.asyncio
@patch("mirage.core.github.repo.github_get")
@patch("mirage.core.github.tree.github_get")
async def test_an_unpinned_mount_reads_the_repos_default_branch(
mock_tree_get, mock_repo_get, config):
"""An unresolved ref must be settled before the tree is fetched.
``accessor.ref`` is None until something resolves it, so reading it
straight sends `ref=None` to the one request the whole mount is built
on. This pins the resolution, not the config default -- the mount that
supplies the default lives a layer up, in
tests/resource/github/test_lazy_hydration.py.
"""
mock_repo_get.return_value = {"default_branch": "master"}
mock_tree_get.return_value = {"truncated": False, "tree": []}
accessor = GitHubAccessor(config, "acme", "proj")
await ensure_tree(accessor)
assert mock_tree_get.await_args.kwargs["ref"] == "master"
@pytest.mark.asyncio
@patch("mirage.core.github.repo.github_get")
@patch("mirage.core.github.tree.github_get")
async def test_a_pinned_mount_reads_its_ref_and_never_asks_for_the_branch(
mock_tree_get, mock_repo_get, config):
mock_tree_get.return_value = {"truncated": False, "tree": []}
accessor = GitHubAccessor(config, "acme", "proj", "release-2")
await ensure_tree(accessor)
assert mock_tree_get.await_args.kwargs["ref"] == "release-2"
mock_repo_get.assert_not_awaited()
@@ -52,7 +52,8 @@ def test_long_ascii_title_is_trimmed_to_name_max():
def test_long_cjk_title_is_trimmed_by_bytes_not_characters():
# 3 bytes per character: a character-counted budget would overflow
# NAME_MAX, which is the bug gdocs' sanitize_title still has.
# NAME_MAX. gdocs/gsheets/gslides had exactly that bug until
# sanitize_label grew a byte budget.
name = make_event_filename(EVENT_ID, "0900-1030", "" * 200)
raw = name.encode()
assert len(raw) <= NAME_MAX_BYTES
@@ -14,6 +14,7 @@
from mirage.resource.gdocs.doc_entry import (DocEntry, make_filename,
sanitize_title)
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
TITLE_MAX = 100
@@ -81,3 +82,34 @@ def test_make_filename_duplicate_titles_different_dates():
assert f1 != f2
assert f1 == "2026-04-01_My_Doc__abc123.gdoc.json"
assert f2 == "2026-03-15_My_Doc__def456.gdoc.json"
# A real Google file id is 44 characters, so this is the fixed overhead a
# title actually has to fit inside.
DOC_ID = "1" * 44
def test_make_filename_fits_name_max_for_a_cjk_title():
# 100 characters of CJK is 300 bytes, which the character budget passed
# untouched: with the date, the id and the suffix the name came to 367
# bytes and ext4/APFS reject it with ENAMETOOLONG.
name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z")
assert byte_len(name) <= NAME_MAX_BYTES
assert name.startswith("2026-08-20_")
assert name.endswith(f"__{DOC_ID}.gdoc.json")
# The cut lands on a character boundary, never mid-sequence.
assert "\ufffd" not in name
def test_make_filename_leaves_an_ascii_title_on_the_char_budget():
name = make_filename("a" * 400, DOC_ID, "")
assert byte_len(name) <= NAME_MAX_BYTES
assert name == f"{'a' * 97}...__{DOC_ID}.gdoc.json"
def test_make_filename_keeps_the_id_when_it_leaves_no_room():
# The title is what gives, never the id: a trimmed id would stop
# addressing the document. Same rule as gcal's event filenames.
long_id = "v" * (NAME_MAX_BYTES - 4)
name = make_filename("Some Title", long_id, "")
assert f"__{long_id}.gdoc.json" in name
+4 -1
View File
@@ -34,7 +34,10 @@ def test_github_config_owner_repo_ref_default():
cfg = GitHubConfig(token="ghp_abc123")
assert cfg.owner is None
assert cfg.repo is None
assert cfg.ref == "main"
# Not "main": that guess 404s the tree fetch on every repository whose
# default branch is something else. None means "resolve the default
# branch", which `ensure_ref` does on the first read.
assert cfg.ref is None
def test_github_config_accepts_owner_repo_ref():
@@ -85,3 +85,37 @@ async def test_a_tree_passed_to_the_constructor_counts_as_hydrated(tree_calls):
resource = GitHubResource(CONFIG, "o", "r", "main", tree=dict(TREE))
await ensure_tree(resource.accessor)
assert tree_calls == []
@pytest.fixture
def default_branch(monkeypatch):
async def _fetch(config, owner, repo):
return "master"
monkeypatch.setattr("mirage.core.github.repo.fetch_default_branch", _fetch)
@pytest.mark.asyncio
async def test_a_mount_naming_no_ref_reads_the_repos_default_branch(
tree_calls, default_branch):
# The config used to default `ref` to the literal string "main", so a
# repository whose default branch is anything else 404d on the one
# request the whole mount is built on and the mount read as empty.
resource = GitHubResource(
GitHubConfig(token="ghp_test", owner="o", repo="r"))
assert resource.accessor.ref is None
await ensure_tree(resource.accessor)
assert tree_calls == [("o", "r", "master")]
@pytest.mark.asyncio
async def test_an_unpinned_mount_is_on_the_default_branch_before_any_fetch(
tree_calls):
# Naming no ref *means* following the default branch, so the two agree
# whatever it turns out to be -- no request, and not the "not known
# yet" None a pinned mount answers.
resource = GitHubResource(
GitHubConfig(token="ghp_test", owner="o", repo="r"))
assert resource.is_default_branch is True
assert GitHubResource(CONFIG, "o", "r", "dev").is_default_branch is None
@@ -0,0 +1,77 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.resource.gsheets.sheet_entry import (SheetEntry, make_filename,
sanitize_title)
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
# A real Google file id is 44 characters, so this is the fixed overhead a
# title actually has to fit inside.
DOC_ID = "1" * 44
def test_sheet_entry_creation():
entry = SheetEntry(
id="abc123",
name="My Spreadsheet",
modified_time="2026-04-01T12:00:00.000Z",
created_time="2026-03-01T12:00:00.000Z",
owner="user@gmail.com",
owned_by_me=True,
can_edit=True,
filename="My_Spreadsheet__abc123.gsheet.json",
)
assert entry.id == "abc123"
assert entry.owned_by_me is True
assert entry.can_edit is True
def test_sanitize_title_basic():
assert sanitize_title("Hello World") == "Hello_World"
assert sanitize_title("My/Doc: A\\Test") == "My_Doc_A_Test"
assert sanitize_title("") == "Untitled"
def test_make_filename_with_and_without_a_date():
assert make_filename("My Spreadsheet", "abc123",
"2026-03-15T10:00:00Z") == \
"2026-03-15_My_Spreadsheet__abc123.gsheet.json"
assert make_filename("My Spreadsheet",
"abc123") == "My_Spreadsheet__abc123.gsheet.json"
def test_make_filename_fits_name_max_for_a_cjk_title():
# 100 characters of CJK is 300 bytes, which the character budget passed
# untouched: with the date, the id and the suffix the name came to 367
# bytes and ext4/APFS reject it with ENAMETOOLONG.
name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z")
assert byte_len(name) <= NAME_MAX_BYTES
assert name.startswith("2026-08-20_")
assert name.endswith(f"__{DOC_ID}.gsheet.json")
# The cut lands on a character boundary, never mid-sequence.
assert "\ufffd" not in name
def test_make_filename_leaves_an_ascii_title_on_the_char_budget():
name = make_filename("a" * 400, DOC_ID, "")
assert byte_len(name) <= NAME_MAX_BYTES
assert name == f"{'a' * 97}...__{DOC_ID}.gsheet.json"
def test_make_filename_keeps_the_id_when_it_leaves_no_room():
# The title is what gives, never the id: a trimmed id would stop
# addressing the spreadsheet. Same rule as gcal's event filenames.
long_id = "v" * (NAME_MAX_BYTES - 4)
name = make_filename("Some Title", long_id, "")
assert f"__{long_id}.gsheet.json" in name
@@ -0,0 +1,77 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.resource.gslides.slide_entry import (SlideEntry, make_filename,
sanitize_title)
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
# A real Google file id is 44 characters, so this is the fixed overhead a
# title actually has to fit inside.
DOC_ID = "1" * 44
def test_slide_entry_creation():
entry = SlideEntry(
id="abc123",
name="My Presentation",
modified_time="2026-04-01T12:00:00.000Z",
created_time="2026-03-01T12:00:00.000Z",
owner="user@gmail.com",
owned_by_me=True,
can_edit=True,
filename="My_Presentation__abc123.gslide.json",
)
assert entry.id == "abc123"
assert entry.owned_by_me is True
assert entry.can_edit is True
def test_sanitize_title_basic():
assert sanitize_title("Hello World") == "Hello_World"
assert sanitize_title("My/Doc: A\\Test") == "My_Doc_A_Test"
assert sanitize_title("") == "Untitled"
def test_make_filename_with_and_without_a_date():
assert make_filename("My Presentation", "abc123",
"2026-03-15T10:00:00Z") == \
"2026-03-15_My_Presentation__abc123.gslide.json"
assert make_filename("My Presentation",
"abc123") == "My_Presentation__abc123.gslide.json"
def test_make_filename_fits_name_max_for_a_cjk_title():
# 100 characters of CJK is 300 bytes, which the character budget passed
# untouched: with the date, the id and the suffix the name came to 367
# bytes and ext4/APFS reject it with ENAMETOOLONG.
name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z")
assert byte_len(name) <= NAME_MAX_BYTES
assert name.startswith("2026-08-20_")
assert name.endswith(f"__{DOC_ID}.gslide.json")
# The cut lands on a character boundary, never mid-sequence.
assert "\ufffd" not in name
def test_make_filename_leaves_an_ascii_title_on_the_char_budget():
name = make_filename("a" * 400, DOC_ID, "")
assert byte_len(name) <= NAME_MAX_BYTES
assert name == f"{'a' * 97}...__{DOC_ID}.gslide.json"
def test_make_filename_keeps_the_id_when_it_leaves_no_room():
# The title is what gives, never the id: a trimmed id would stop
# addressing the presentation. Same rule as gcal's event filenames.
long_id = "v" * (NAME_MAX_BYTES - 4)
name = make_filename("Some Title", long_id, "")
assert f"__{long_id}.gslide.json" in name
+1 -1
View File
@@ -61,7 +61,7 @@ UNMIRRORED_DIRS = {
# would count 816 today. What the ratchet buys is narrower than it looks:
# a module whose name appears nowhere in the suite cannot be added
# silently.
MIRROR_BASELINE = 186
MIRROR_BASELINE = 184
def _test_dirs() -> list[pathlib.Path]:
+32 -2
View File
@@ -12,7 +12,7 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.utils.sanitize import sanitize_label
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
def test_sanitize_label_replaces_unsafe_and_spaces():
@@ -60,8 +60,38 @@ def test_sanitize_label_budget_counts_code_points():
def test_sanitize_label_ellipsizes_on_code_point_boundary():
# A byte budget wide enough to stay out of the way, so this pins the
# character budget alone.
label = "\U00010400" * 120
result = sanitize_label(label, fallback="X", max_len=100)
result = sanitize_label(label, fallback="X", max_len=100, max_bytes=10_000)
assert len(result) == 100
assert result.endswith("...")
assert "\ufffd" not in result
def test_sanitize_label_honors_the_byte_ceiling_within_the_char_budget():
# 100 astral code points is 400 bytes, so a name the character budget
# accepts is one ext4 and APFS reject with ENAMETOOLONG. The default
# budget is NAME_MAX, and the cut still lands on a code-point boundary.
label = "\U00010400" * 120
result = sanitize_label(label, fallback="X", max_len=100)
assert len(result) < 100
assert byte_len(result) <= NAME_MAX_BYTES
assert result.endswith("...")
assert "\ufffd" not in result
def test_sanitize_label_byte_budget_is_the_callers_remaining_room():
# What the gdocs/gmail filenames pass: NAME_MAX minus the id, the
# separators and the suffix.
result = sanitize_label("" * 200, fallback="X", max_len=100, max_bytes=60)
assert byte_len(result) <= 60
assert result.endswith("...")
assert "\ufffd" not in result
def test_sanitize_label_drops_the_ellipsis_when_it_cannot_fit():
# Three dots and nothing is not a name; a budget this small yields
# whatever of the label actually fits.
assert sanitize_label("abcdef", fallback="X", max_len=100,
max_bytes=2) == "ab"
+99
View File
@@ -0,0 +1,99 @@
from __future__ import annotations
import re
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parent.parent
TS_SRC = [
REPO / "typescript/packages/core/src",
REPO / "typescript/packages/node/src",
REPO / "typescript/packages/browser/src",
]
# `normalizeFields(input, {rename: {...}})` and the shared `const RENAME`
# maps the S3-alias families feed it.
RENAME_PROP_RE = re.compile(r"rename:\s*\{([^{}]*)\}", re.S)
RENAME_CONST_RE = re.compile(r"^const \w*RENAME\w*[^=\n]*=\s*\{([^{}]*)\}",
re.M | re.S)
PAIR_RE = re.compile(r"([A-Za-z0-9_]+)\s*:\s*'([A-Za-z0-9_]+)'")
SNAKE_RE = re.compile(r"_([a-z0-9])")
def snake_to_camel(snake: str) -> str:
"""Reimplement `snakeToCamel` from utils/normalize.ts.
Args:
snake: The python-side field name.
Returns:
The camelCase spelling `normalizeFields` produces by default.
"""
return SNAKE_RE.sub(lambda m: m.group(1).upper(), snake)
def redundant_pairs(text: str) -> list[tuple[str, str]]:
"""Find rename entries that only restate the default mapping.
Args:
text: One TypeScript source file.
Returns:
The `(source, target)` pairs whose target is exactly what
`snakeToCamel` would have produced anyway.
"""
found: list[tuple[str, str]] = []
for match in RENAME_PROP_RE.finditer(text):
found.extend(PAIR_RE.findall(match.group(1)))
for match in RENAME_CONST_RE.finditer(text):
found.extend(PAIR_RE.findall(match.group(1)))
return [(k, v) for k, v in found if snake_to_camel(k) == v]
def source_files() -> list[Path]:
"""Every TypeScript source file that could hold a rename map.
Returns:
The files to scan, build output and node_modules excluded.
"""
out: list[Path] = []
for root in TS_SRC:
if not root.is_dir():
continue
out.extend(p for p in root.rglob("*.ts")
if "node_modules" not in p.parts and "dist" not in p.parts)
return sorted(out)
def main() -> int:
"""Fail when a rename map restates what `snakeToCamel` already does.
`normalizeFields` falls back to `snakeToCamel(key)` for every key no
rename names, so `api_key: 'apiKey'` is not configuration, it is a
second place for the same fact to be wrong -- and it hides the entries
that *are* load-bearing (`endpoint_url: 'endpoint'`, `timeout:
'timeoutMs'`, `aws_profile: 'profile'`) in a wall of noise.
Returns:
0 when every surviving rename entry is a real override.
"""
offenders: list[tuple[Path, list[tuple[str, str]]]] = []
for path in source_files():
if path.name == "normalize.test.ts":
continue
pairs = redundant_pairs(path.read_text())
if pairs:
offenders.append((path, pairs))
if not offenders:
return 0
total = sum(len(p) for _, p in offenders)
print(f"{total} rename entries only restate snakeToCamel:\n")
for path, pairs in offenders:
print(f" {path.relative_to(REPO)}")
for key, value in pairs:
print(f" {key}: '{value}'")
print("\nDelete them: normalizeFields already maps these by default.")
return 1
if __name__ == "__main__":
sys.exit(main())
+6
View File
@@ -31,6 +31,7 @@ export { OPFSAccessor } from './accessor/opfs.ts'
export { OPFS_COMMANDS } from './commands/builtin/opfs/index.ts'
export { S3Resource, S3_BROWSER_PROMPT, type S3ResourceState } from './resource/s3/s3.ts'
export {
normalizeS3Config,
redactConfig as redactS3Config,
type S3BrowserOperation,
type S3BrowserPresignedUrlProvider,
@@ -165,12 +166,14 @@ export {
} from './resource/qingstor/config.ts'
export { SlackResource, type SlackResourceState } from './resource/slack/slack.ts'
export {
normalizeSlackConfig,
redactSlackConfig,
type SlackConfig,
type SlackConfigRedacted,
} from './resource/slack/config.ts'
export { DiscordResource, type DiscordResourceState } from './resource/discord/discord.ts'
export {
normalizeDiscordConfig,
redactDiscordConfig,
type DiscordConfig,
type DiscordConfigRedacted,
@@ -181,6 +184,7 @@ export { MongoDBResource, type MongoDBResourceOptions } from './resource/mongodb
export { HttpMongoDriver, type HttpMongoDriverOptions } from './resource/mongodb/http_driver.ts'
export { TrelloResource, type TrelloResourceState } from './resource/trello/trello.ts'
export {
normalizeTrelloConfig,
redactTrelloConfig,
type TrelloConfig,
type TrelloConfigRedacted,
@@ -193,12 +197,14 @@ export {
export type { LinearConfig, LinearConfigRedacted } from '@struktoai/mirage-core/core/linear/config'
export { NotionResource, type NotionResourceState } from './resource/notion/notion.ts'
export {
normalizeNotionConfig,
redactNotionConfig,
type NotionConfig,
type NotionConfigRedacted,
} from './resource/notion/config.ts'
export { LangfuseResource, type LangfuseResourceState } from './resource/langfuse/langfuse.ts'
export {
normalizeLangfuseConfig,
redactLangfuseConfig,
type LangfuseConfig,
type LangfuseConfigRedacted,
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<AliyunConfig, AliyunConfigRedacted>({
export const resolvedAliyunEndpoint = alias.resolvedEndpoint
export const aliyunToS3Config = alias.toS3Config
export const redactAliyunConfig = alias.redact
export const normalizeAliyunConfig = alias.normalize
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<BackblazeConfig, BackblazeConfigRedacted>({
export const resolvedBackblazeEndpoint = alias.resolvedEndpoint
export const backblazeToS3Config = alias.toS3Config
export const redactBackblazeConfig = alias.redact
export const normalizeBackblazeConfig = alias.normalize
@@ -62,12 +62,6 @@ export function redactBoxConfig(config: BoxConfig): BoxConfigRedacted {
export function normalizeBoxConfig(input: Record<string, unknown>): BoxConfig {
return normalizeFields(input, {
rename: {
root_folder_id: 'rootFolderId',
content_search: 'contentSearch',
client_id: 'clientId',
client_secret: 'clientSecret',
refresh_token: 'refreshToken',
access_token: 'accessToken',
developer_token: 'accessToken',
},
}) as unknown as BoxConfig
@@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias<CephConfig, CephConfigRedacted>({
export const cephToS3Config = alias.toS3Config
export const redactCephConfig = alias.redact
export const normalizeCephConfig = alias.normalize
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<DigitalOceanConfig, DigitalOceanConfigRedacted>
export const resolvedDigitalOceanEndpoint = alias.resolvedEndpoint
export const digitalOceanToS3Config = alias.toS3Config
export const redactDigitalOceanConfig = alias.redact
export const normalizeDigitalOceanConfig = alias.normalize
@@ -14,6 +14,7 @@
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
type HeaderProvider = () => Promise<Record<string, string>> | Record<string, string>
@@ -31,3 +32,14 @@ export type DiscordConfigRedacted = RedactedConfig<DiscordConfig, 'getHeaders'>
export function redactDiscordConfig(config: DiscordConfig): DiscordConfigRedacted {
return redactConfigWithSchema(DiscordConfigSchema, config) as unknown as DiscordConfigRedacted
}
/**
* Translate a python-style config blob to this one's camelCase.
*
* No rename map: every field's camelCase spelling is what `snakeToCamel`
* already produces, and restating those only creates a second place to be
* wrong. Mirrors node's `normalizeDiscordConfig`.
*/
export function normalizeDiscordConfig(input: Record<string, unknown>): DiscordConfig {
return normalizeFields(input) as unknown as DiscordConfig
}
@@ -47,13 +47,5 @@ export function redactDropboxConfig(config: DropboxConfig): DropboxConfigRedacte
}
export function normalizeDropboxConfig(input: Record<string, unknown>): DropboxConfig {
return normalizeFields(input, {
rename: {
client_id: 'clientId',
client_secret: 'clientSecret',
refresh_token: 'refreshToken',
root_path: 'rootPath',
content_search: 'contentSearch',
},
}) as unknown as DropboxConfig
return normalizeFields(input) as unknown as DropboxConfig
}
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<GCSConfig, GCSConfigRedacted, string>({
export const gcsToS3Config = alias.toS3Config
export const redactGcsConfig = alias.redact
export const normalizeGCSConfig = alias.normalize
@@ -14,6 +14,7 @@
import { redactConfigWithSchema, secretStr, z } from '@struktoai/mirage-core/resource/secrets'
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
const LangfuseConfigSchema = z.object({
publicKey: z.string(),
@@ -31,3 +32,14 @@ export type LangfuseConfigRedacted = RedactedConfig<LangfuseConfig, 'secretKey'>
export function redactLangfuseConfig(config: LangfuseConfig): LangfuseConfigRedacted {
return redactConfigWithSchema(LangfuseConfigSchema, config) as unknown as LangfuseConfigRedacted
}
/**
* Translate a python-style config blob to this one's camelCase.
*
* No rename map: every field's camelCase spelling is what `snakeToCamel`
* already produces, and restating those only creates a second place to be
* wrong. Mirrors node's `normalizeLangfuseConfig`.
*/
export function normalizeLangfuseConfig(input: Record<string, unknown>): LangfuseConfig {
return normalizeFields(input) as unknown as LangfuseConfig
}
@@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias<MinIOConfig, MinIOConfigRedacted>({
export const minioToS3Config = alias.toS3Config
export const redactMinIOConfig = alias.redact
export const normalizeMinIOConfig = alias.normalize
@@ -15,6 +15,7 @@
import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js'
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
const NotionConfigSchema = z.object({
authProvider: secretSchema(
@@ -30,3 +31,14 @@ export type NotionConfigRedacted = RedactedConfig<NotionConfig, 'authProvider'>
export function redactNotionConfig(config: NotionConfig): NotionConfigRedacted {
return redactConfigWithSchema(NotionConfigSchema, config) as unknown as NotionConfigRedacted
}
/**
* Translate a python-style config blob to this one's camelCase.
*
* No rename map: every field's camelCase spelling is what `snakeToCamel`
* already produces, and restating those only creates a second place to be
* wrong. Mirrors node's `normalizeNotionConfig`.
*/
export function normalizeNotionConfig(input: Record<string, unknown>): NotionConfig {
return normalizeFields(input) as unknown as NotionConfig
}
@@ -36,3 +36,4 @@ const alias = makeBrowserS3Alias<OCIConfig, OCIConfigRedacted>({
export const resolvedOciEndpoint = alias.resolvedEndpoint
export const ociToS3Config = alias.toS3Config
export const redactOciConfig = alias.redact
export const normalizeOCIConfig = alias.normalize
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<QingStorConfig, QingStorConfigRedacted>({
export const resolvedQingStorEndpoint = alias.resolvedEndpoint
export const qingStorToS3Config = alias.toS3Config
export const redactQingStorConfig = alias.redact
export const normalizeQingStorConfig = alias.normalize
@@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias<R2Config, R2ConfigRedacted>({
export const resolvedR2Endpoint = alias.resolvedEndpoint
export const r2ToS3Config = alias.toS3Config
export const redactR2Config = alias.redact
export const normalizeR2Config = alias.normalize
@@ -36,6 +36,50 @@ describe('browser resource registry', () => {
}
})
// Every entry used to hand-roll `normalizeFields` with a rename map that
// mostly restated what `snakeToCamel` already does, then cast the result
// through a config interface written a second time in the registry. The
// casts hid a mismatch: nothing checked that the shape the resource wants
// is the shape the entry produces.
it('normalizes snake_case config for every hand-wired backend', async () => {
const provider = (): Promise<string> => Promise.resolve('https://example.com/signed')
const cases: [string, Record<string, unknown>, Record<string, unknown>][] = [
[
'trello',
{ api_key: 'k', api_token: 't', workspace_id: 'w', board_ids: ['b'], base_url: 'u' },
{ workspaceId: 'w', boardIds: ['b'], baseUrl: 'u' },
],
[
'langfuse',
{ public_key: 'p', secret_key: 's', default_trace_limit: 5, default_search_limit: 6 },
{ publicKey: 'p', defaultTraceLimit: 5, defaultSearchLimit: 6 },
],
['slack', { proxy_url: 'http://x' }, { proxyUrl: 'http://x' }],
['discord', { proxy_url: 'http://x' }, { proxyUrl: 'http://x' }],
[
's3',
{ bucket: 'b', presignedUrlProvider: provider, endpoint_url: 'http://e', key_prefix: 'p/' },
{ bucket: 'b', endpoint: 'http://e', keyPrefix: 'p/' },
],
[
'minio',
{ bucket: 'b', presignedUrlProvider: provider, endpoint_url: 'http://e' },
{ bucket: 'b', endpoint: 'http://e' },
],
]
for (const [name, input, expected] of cases) {
const state = (await (await buildResource(name, input)).getState()) as {
config: Record<string, unknown>
}
expect(state.config, name).toMatchObject(expected)
// `endpoint_url` is the one rename that is not mechanical; a leftover
// snake_case key means the entry skipped normalization entirely.
for (const key of Object.keys(state.config)) {
expect(key, `${name}.${key}`).not.toContain('_')
}
}
})
it('lists known resources sorted', () => {
const names = knownResources()
expect(names).toContain('ram')
@@ -18,7 +18,6 @@ import type { DifyConfig } from '@struktoai/mirage-core/resource/dify/config'
import type { QdrantConfig } from '@struktoai/mirage-core/resource/qdrant/config'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
import { compareCodePoints } from '@struktoai/mirage-core/utils/sort'
import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js'
/**
* Construct a resource by registry name in the browser runtime.
@@ -35,78 +34,6 @@ import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.
*/
export type ResourceFactory = (config: Record<string, unknown>) => Promise<Resource>
interface S3BrowserCtorConfig {
bucket: string
presignedUrlProvider: (
path: string,
op: 'GET' | 'PUT' | 'HEAD' | 'DELETE' | 'LIST' | 'COPY',
opts?: {
contentType?: string
ttlSec?: number
listPrefix?: string
listDelimiter?: string
listContinuationToken?: string
copySource?: string
},
) => Promise<string>
}
type GCSBrowserCtorConfig = S3BrowserCtorConfig & { region?: string; endpoint?: string }
type R2BrowserCtorConfig = S3BrowserCtorConfig & {
accountId?: string
region?: string
endpoint?: string
}
type OCIBrowserCtorConfig = S3BrowserCtorConfig & {
namespace?: string
region?: string
endpoint?: string
}
type SupabaseBrowserCtorConfig = S3BrowserCtorConfig & {
projectRef?: string
region?: string
endpoint?: string
}
type S3AliasBrowserCtorConfig = S3BrowserCtorConfig & {
region?: string
endpoint?: string
}
interface SlackBrowserCtorConfig {
proxyUrl: string
getHeaders?: () => Promise<Record<string, string>> | Record<string, string>
}
interface DiscordBrowserCtorConfig {
proxyUrl: string
getHeaders?: () => Promise<Record<string, string>> | Record<string, string>
}
interface NotionBrowserCtorConfig {
authProvider: OAuthClientProvider
serverUrl?: string
}
interface TrelloBrowserCtorConfig {
apiKey: string
apiToken: string
workspaceId?: string
boardIds?: readonly string[]
baseUrl?: string
}
interface LangfuseBrowserCtorConfig {
publicKey: string
secretKey: string
host?: string
defaultTraceLimit?: number
defaultSearchLimit?: number
defaultFromTimestamp?: string
}
interface PostgresBrowserCtorConfig {
dsn: string
schemas?: readonly string[]
defaultRowLimit?: number
maxReadRows?: number
maxReadBytes?: number
defaultSearchLimit?: number
}
const REGISTRY: Record<string, ResourceFactory> = {
ram: async (_config) => {
const { RAMResource } = await import('@struktoai/mirage-core/resource/ram/ram')
@@ -119,135 +46,93 @@ const REGISTRY: Record<string, ResourceFactory> = {
},
s3: async (config) => {
const { S3Resource } = await import('./s3/s3.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new S3Resource(norm as unknown as S3BrowserCtorConfig)
const { normalizeS3Config } = await import('./s3/config.ts')
return new S3Resource(normalizeS3Config(config))
},
gcs: async (config) => {
const { GCSResource } = await import('./gcs/gcs.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new GCSResource(norm as unknown as GCSBrowserCtorConfig)
const { normalizeGCSConfig } = await import('./gcs/config.ts')
return new GCSResource(normalizeGCSConfig(config))
},
r2: async (config) => {
const { R2Resource } = await import('./r2/r2.ts')
const norm = normalizeFields(config, {
rename: { account_id: 'accountId', endpoint_url: 'endpoint' },
})
return new R2Resource(norm as unknown as R2BrowserCtorConfig)
const { normalizeR2Config } = await import('./r2/config.ts')
return new R2Resource(normalizeR2Config(config))
},
oci: async (config) => {
const { OCIResource } = await import('./oci/oci.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new OCIResource(norm as unknown as OCIBrowserCtorConfig)
const { normalizeOCIConfig } = await import('./oci/config.ts')
return new OCIResource(normalizeOCIConfig(config))
},
supabase: async (config) => {
const { SupabaseResource } = await import('./supabase/supabase.ts')
const norm = normalizeFields(config, {
rename: { project_ref: 'projectRef', endpoint_url: 'endpoint' },
})
return new SupabaseResource(norm as unknown as SupabaseBrowserCtorConfig)
const { normalizeSupabaseConfig } = await import('./supabase/config.ts')
return new SupabaseResource(normalizeSupabaseConfig(config))
},
minio: async (config) => {
const { MinIOResource } = await import('./minio/minio.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new MinIOResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeMinIOConfig } = await import('./minio/config.ts')
return new MinIOResource(normalizeMinIOConfig(config))
},
ceph: async (config) => {
const { CephResource } = await import('./ceph/ceph.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new CephResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeCephConfig } = await import('./ceph/config.ts')
return new CephResource(normalizeCephConfig(config))
},
seaweedfs: async (config) => {
const { SeaweedFSResource } = await import('./seaweedfs/seaweedfs.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new SeaweedFSResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeSeaweedFSConfig } = await import('./seaweedfs/config.ts')
return new SeaweedFSResource(normalizeSeaweedFSConfig(config))
},
wasabi: async (config) => {
const { WasabiResource } = await import('./wasabi/wasabi.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new WasabiResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeWasabiConfig } = await import('./wasabi/config.ts')
return new WasabiResource(normalizeWasabiConfig(config))
},
backblaze: async (config) => {
const { BackblazeResource } = await import('./backblaze/backblaze.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new BackblazeResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeBackblazeConfig } = await import('./backblaze/config.ts')
return new BackblazeResource(normalizeBackblazeConfig(config))
},
digitalocean: async (config) => {
const { DigitalOceanResource } = await import('./digitalocean/digitalocean.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new DigitalOceanResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeDigitalOceanConfig } = await import('./digitalocean/config.ts')
return new DigitalOceanResource(normalizeDigitalOceanConfig(config))
},
tencent: async (config) => {
const { TencentResource } = await import('./tencent/tencent.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new TencentResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeTencentConfig } = await import('./tencent/config.ts')
return new TencentResource(normalizeTencentConfig(config))
},
aliyun: async (config) => {
const { AliyunResource } = await import('./aliyun/aliyun.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new AliyunResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeAliyunConfig } = await import('./aliyun/config.ts')
return new AliyunResource(normalizeAliyunConfig(config))
},
scaleway: async (config) => {
const { ScalewayResource } = await import('./scaleway/scaleway.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new ScalewayResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeScalewayConfig } = await import('./scaleway/config.ts')
return new ScalewayResource(normalizeScalewayConfig(config))
},
qingstor: async (config) => {
const { QingStorResource } = await import('./qingstor/qingstor.ts')
const norm = normalizeFields(config, {
rename: { endpoint_url: 'endpoint' },
})
return new QingStorResource(norm as unknown as S3AliasBrowserCtorConfig)
const { normalizeQingStorConfig } = await import('./qingstor/config.ts')
return new QingStorResource(normalizeQingStorConfig(config))
},
slack: async (config) => {
const { SlackResource } = await import('./slack/slack.ts')
const norm = normalizeFields(config, {
rename: { proxy_url: 'proxyUrl', get_headers: 'getHeaders' },
})
return new SlackResource(norm as unknown as SlackBrowserCtorConfig)
const { normalizeSlackConfig } = await import('./slack/config.ts')
return new SlackResource(normalizeSlackConfig(config))
},
discord: async (config) => {
const { DiscordResource } = await import('./discord/discord.ts')
const norm = normalizeFields(config, {
rename: { proxy_url: 'proxyUrl', get_headers: 'getHeaders' },
})
return new DiscordResource(norm as unknown as DiscordBrowserCtorConfig)
const { normalizeDiscordConfig } = await import('./discord/config.ts')
return new DiscordResource(normalizeDiscordConfig(config))
},
trello: async (config) => {
const { TrelloResource } = await import('./trello/trello.ts')
const norm = normalizeFields(config, {
rename: {
api_key: 'apiKey',
api_token: 'apiToken',
workspace_id: 'workspaceId',
board_ids: 'boardIds',
base_url: 'baseUrl',
},
})
return new TrelloResource(norm as unknown as TrelloBrowserCtorConfig)
const { normalizeTrelloConfig } = await import('./trello/config.ts')
return new TrelloResource(normalizeTrelloConfig(config))
},
linear: async (config) => {
const { LinearResource } = await import('./linear/linear.ts')
@@ -256,15 +141,9 @@ const REGISTRY: Record<string, ResourceFactory> = {
},
postgres: async (config) => {
const { PostgresResource } = await import('./postgres/postgres.ts')
const norm = normalizeFields(config, {
rename: {
default_row_limit: 'defaultRowLimit',
max_read_rows: 'maxReadRows',
max_read_bytes: 'maxReadBytes',
default_search_limit: 'defaultSearchLimit',
},
})
return new PostgresResource(norm as unknown as PostgresBrowserCtorConfig)
const { normalizePostgresConfig } =
await import('@struktoai/mirage-core/resource/postgres/config')
return new PostgresResource(normalizePostgresConfig(config))
},
mongodb: async (config) => {
const { MongoDBResource } = await import('./mongodb/mongodb.ts')
@@ -294,23 +173,13 @@ const REGISTRY: Record<string, ResourceFactory> = {
},
notion: async (config) => {
const { NotionResource } = await import('./notion/notion.ts')
const norm = normalizeFields(config, {
rename: { auth_provider: 'authProvider', server_url: 'serverUrl' },
})
return new NotionResource(norm as unknown as NotionBrowserCtorConfig)
const { normalizeNotionConfig } = await import('./notion/config.ts')
return new NotionResource(normalizeNotionConfig(config))
},
langfuse: async (config) => {
const { LangfuseResource } = await import('./langfuse/langfuse.ts')
const norm = normalizeFields(config, {
rename: {
public_key: 'publicKey',
secret_key: 'secretKey',
default_trace_limit: 'defaultTraceLimit',
default_search_limit: 'defaultSearchLimit',
default_from_timestamp: 'defaultFromTimestamp',
},
})
return new LangfuseResource(norm as unknown as LangfuseBrowserCtorConfig)
const { normalizeLangfuseConfig } = await import('./langfuse/config.ts')
return new LangfuseResource(normalizeLangfuseConfig(config))
},
github: async (config) => {
const { GitHubResource } = await import('./github/github.ts')
@@ -12,6 +12,20 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { S3Config } from '@struktoai/mirage-core/resource/s3/config'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
// The one field whose camelCase spelling is not what `snakeToCamel` would
// produce, so it is the whole rename map for the browser's S3 family. Every
// other field (`key_prefix`, `default_content_type`, ...) round-trips through
// the default, and restating those only creates a second place to be wrong.
export const S3_BROWSER_RENAME: Record<string, string> = { endpoint_url: 'endpoint' }
/** Translate a python-style S3 config blob to the browser's camelCase one. */
export function normalizeS3Config(input: Record<string, unknown>): S3Config {
return normalizeFields(input, { rename: S3_BROWSER_RENAME }) as unknown as S3Config
}
export { redactConfig } from '@struktoai/mirage-core/resource/s3/config'
export type {
S3BrowserOperation,
@@ -17,6 +17,8 @@ import type { RegisteredCommand } from '@struktoai/mirage-core/commands/config'
import type { RegisteredOp } from '@struktoai/mirage-core/ops/registry'
import { remapCommandsResource, remapOpsResource } from '@struktoai/mirage-core/resource/s3/remap'
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
import { S3_BROWSER_RENAME } from './s3/config.ts'
import type { S3BrowserPresignedUrlProvider, S3Config, S3ConfigRedacted } from './s3/config.ts'
/**
@@ -92,6 +94,7 @@ export interface BrowserAlias<C, R, E extends string | undefined> {
resolvedEndpoint: (config: C) => string | E
toS3Config: (config: C) => S3Config
redact: (config: C) => R
normalize: (input: Record<string, unknown>) => C
}
/**
@@ -137,6 +140,7 @@ export function makeBrowserS3Alias<
}
},
redact: (config) => redactConfigWithSchema(schema, config) as unknown as R,
normalize: (input) => normalizeFields(input, { rename: S3_BROWSER_RENAME }) as unknown as C,
}
}
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<ScalewayConfig, ScalewayConfigRedacted>({
export const resolvedScalewayEndpoint = alias.resolvedEndpoint
export const scalewayToS3Config = alias.toS3Config
export const redactScalewayConfig = alias.redact
export const normalizeScalewayConfig = alias.normalize
@@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias<SeaweedFSConfig, SeaweedFSConfigRedacted>({
export const seaweedfsToS3Config = alias.toS3Config
export const redactSeaweedFSConfig = alias.redact
export const normalizeSeaweedFSConfig = alias.normalize
@@ -14,6 +14,7 @@
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
type HeaderProvider = () => Promise<Record<string, string>> | Record<string, string>
@@ -31,3 +32,14 @@ export type SlackConfigRedacted = RedactedConfig<SlackConfig, 'getHeaders'>
export function redactSlackConfig(config: SlackConfig): SlackConfigRedacted {
return redactConfigWithSchema(SlackConfigSchema, config) as unknown as SlackConfigRedacted
}
/**
* Translate a python-style config blob to this one's camelCase.
*
* No rename map: every field's camelCase spelling is what `snakeToCamel`
* already produces, and restating those only creates a second place to be
* wrong. Mirrors node's `normalizeSlackConfig`.
*/
export function normalizeSlackConfig(input: Record<string, unknown>): SlackConfig {
return normalizeFields(input) as unknown as SlackConfig
}
@@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias<SupabaseConfig, SupabaseConfigRedacted>({
export const resolvedSupabaseEndpoint = alias.resolvedEndpoint
export const supabaseToS3Config = alias.toS3Config
export const redactSupabaseConfig = alias.redact
export const normalizeSupabaseConfig = alias.normalize
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<TencentConfig, TencentConfigRedacted>({
export const resolvedTencentEndpoint = alias.resolvedEndpoint
export const tencentToS3Config = alias.toS3Config
export const redactTencentConfig = alias.redact
export const normalizeTencentConfig = alias.normalize
@@ -14,6 +14,7 @@
import { redactConfigWithSchema, secretStr, z } from '@struktoai/mirage-core/resource/secrets'
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
const TrelloConfigSchema = z.object({
apiKey: secretStr(),
@@ -30,3 +31,14 @@ export type TrelloConfigRedacted = RedactedConfig<TrelloConfig, 'apiKey' | 'apiT
export function redactTrelloConfig(config: TrelloConfig): TrelloConfigRedacted {
return redactConfigWithSchema(TrelloConfigSchema, config) as unknown as TrelloConfigRedacted
}
/**
* Translate a python-style config blob to this one's camelCase.
*
* No rename map: every field's camelCase spelling is what `snakeToCamel`
* already produces, and restating those only creates a second place to be
* wrong. Mirrors node's `normalizeTrelloConfig`.
*/
export function normalizeTrelloConfig(input: Record<string, unknown>): TrelloConfig {
return normalizeFields(input) as unknown as TrelloConfig
}
@@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias<WasabiConfig, WasabiConfigRedacted, string>({
export const resolvedWasabiEndpoint = alias.resolvedEndpoint
export const wasabiToS3Config = alias.toS3Config
export const redactWasabiConfig = alias.redact
export const normalizeWasabiConfig = alias.normalize
@@ -35,5 +35,5 @@ export function redactDiscordConfig(config: DiscordConfig): DiscordConfigRedacte
}
export function normalizeDiscordConfig(input: Record<string, unknown>): DiscordConfig {
return normalizeFields(input, {}) as unknown as DiscordConfig
return normalizeFields(input) as unknown as DiscordConfig
}
@@ -34,9 +34,7 @@ export function redactGitHubConfig(config: GitHubConfig): GitHubConfigRedacted {
}
export function normalizeGitHubConfig(input: Record<string, unknown>): GitHubConfig {
return normalizeFields(input, {
rename: { base_url: 'baseUrl' },
}) as unknown as GitHubConfig
return normalizeFields(input) as unknown as GitHubConfig
}
export const GhConfigSchema = z.object({
@@ -27,16 +27,27 @@ import {
messageJsonBytes,
} from './messages.ts'
import { enoent } from '../../utils/errors.ts'
import { sanitizeLabel } from '../../utils/sanitize.ts'
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
import { compareCodePoints } from '../../utils/sort.ts'
const TITLE_MAX = 80
const MSG_SUFFIX = '.gmail.json'
export const sanitize = (text: string): string =>
sanitizeLabel(text, { fallback: 'No_Subject', maxLen: TITLE_MAX })
export const sanitize = (text: string, maxBytes?: number): string =>
sanitizeLabel(text, {
fallback: 'No_Subject',
maxLen: TITLE_MAX,
...(maxBytes !== undefined ? { maxBytes } : {}),
})
function msgFilename(subject: string, msgId: string): string {
return `${sanitize(subject)}__${msgId}.gmail.json`
/**
* 80 characters is 240 bytes of CJK, which overflows the 255-byte NAME_MAX
* once the id and `.gmail.json` are added; the filesystem rejects the name
* outright. So the subject takes what the id and the suffix leave.
*/
export function msgFilename(subject: string, msgId: string): string {
const fixed = 2 + byteLength(msgId) + MSG_SUFFIX.length
return `${sanitize(subject, NAME_MAX_BYTES - fixed)}__${msgId}${MSG_SUFFIX}`
}
function dateFromInternal(internalDate: string | undefined): string {
@@ -14,7 +14,7 @@
import type { TokenManager } from '../google/client.ts'
import { decodeBody, extractHeader, getMessageRaw, listMessages } from './messages.ts'
import { sanitize } from './readdir.ts'
import { msgFilename } from './readdir.ts'
import type { GmailScope } from './scope.ts'
const EXCERPT_WINDOW = 120
@@ -115,7 +115,11 @@ export function formatGrepResults(
const label = row.label !== '' ? row.label : (scope.labelName ?? 'INBOX')
const date = row.date
const mid = row.id
const filename = `${sanitize(row.subject || 'No Subject')}__${mid}.gmail.json`
// The same builder readdir names the file with, not a second spelling of
// it: the subject's budget depends on the id and the suffix, so a hit
// composed from a bare `sanitize` pointed at a path that does not exist
// once a long subject was trimmed.
const filename = msgFilename(row.subject || 'No Subject', mid)
const sender = row.sender !== '' ? row.sender : '?'
const haystack = `${row.subject}\n${row.bodyText}`
let excerpt = pattern !== '' ? extractExcerpt(haystack, pattern) : ''
@@ -75,15 +75,5 @@ export function redactGoogleConfig(config: GoogleConfig): GoogleConfigRedacted {
}
export function normalizeGoogleConfig(input: Record<string, unknown>): GoogleConfig {
return normalizeFields(input, {
rename: {
client_id: 'clientId',
client_secret: 'clientSecret',
refresh_token: 'refreshToken',
api_base: 'apiBase',
folder_id: 'folderId',
time_zone: 'timeZone',
min_access_role: 'minAccessRole',
},
}) as unknown as GoogleConfig
return normalizeFields(input) as unknown as GoogleConfig
}
@@ -37,11 +37,5 @@ export function redactLinearConfig(config: LinearConfig): LinearConfigRedacted {
}
export function normalizeLinearConfig(input: Record<string, unknown>): LinearConfig {
return normalizeFields(input, {
rename: {
api_key: 'apiKey',
team_ids: 'teamIds',
base_url: 'baseUrl',
},
}) as unknown as LinearConfig
return normalizeFields(input) as unknown as LinearConfig
}
@@ -35,10 +35,5 @@ export function redactNotionConfig(config: NotionConfig): NotionConfigRedacted {
}
export function normalizeNotionConfig(input: Record<string, unknown>): NotionConfig {
return normalizeFields(input, {
rename: {
api_key: 'apiKey',
base_url: 'baseUrl',
},
}) as unknown as NotionConfig
return normalizeFields(input) as unknown as NotionConfig
}
@@ -36,7 +36,5 @@ export function redactSlackConfig(config: SlackConfig): SlackConfigRedacted {
}
export function normalizeSlackConfig(input: Record<string, unknown>): SlackConfig {
return normalizeFields(input, {
rename: { search_token: 'searchToken', base_url: 'baseUrl' },
}) as unknown as SlackConfig
return normalizeFields(input) as unknown as SlackConfig
}
@@ -61,8 +61,6 @@ export function redactDatabricksVolumeConfig(
export function normalizeDatabricksVolumeConfig(
input: Record<string, unknown>,
): DatabricksVolumeConfig {
const renamed = normalizeFields(input, {
rename: { root_path: 'rootPath' },
})
const renamed = normalizeFields(input)
return DatabricksVolumeConfigSchema.parse(renamed) as DatabricksVolumeConfig
}
@@ -56,7 +56,8 @@ describe('gcal event entry naming', () => {
it('trims a long CJK title by bytes, not characters', () => {
// 3 bytes per character: a character-counted budget would overflow
// NAME_MAX, which is the bug gdocs' sanitizeTitle still has.
// NAME_MAX. gdocs/gsheets/gslides had exactly that bug until
// sanitizeLabel grew a byte budget.
const name = makeEventFilename(EVENT_ID, '0900-1030', '会'.repeat(200))
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name).not.toContain('')
@@ -16,6 +16,7 @@ import { enoent } from '../../utils/errors.ts'
import { makeIdName } from '../../utils/naming.ts'
import {
NAME_MAX_BYTES,
byteLength,
sanitizeName,
stripTrailingUnderscores,
truncateBytes,
@@ -31,8 +32,6 @@ const UNTITLED = 'untitled'
// there is no title to sanitize and "busy" is the honest rendering.
const BUSY = 'busy'
const UTF8 = new TextEncoder()
/** Pick the title segment for an event filename. */
export function eventTitle(summary: string | null, freeBusy = false): string {
if (summary !== null && summary.trim() !== '') return sanitizeName(summary)
@@ -48,7 +47,7 @@ export function eventTitle(summary: string | null, freeBusy = false): string {
* 255-byte NAME_MAX is left rather than a fixed character count.
*/
export function makeEventFilename(eventId: string, hhmm: string, title: string): string {
const fixed = UTF8.encode(eventId).length + 2 + hhmm.length + 1 + EVENT_SUFFIX.length
const fixed = byteLength(eventId) + 2 + hhmm.length + 1 + EVENT_SUFFIX.length
const trimmed = stripTrailingUnderscores(truncateBytes(title, NAME_MAX_BYTES - fixed))
if (trimmed === '') {
// The title is what gives, never the id: trimming the id would make the
@@ -0,0 +1,55 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { makeFilename } from './doc_entry.ts'
import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts'
// A real Google file id is 44 characters, so this is the fixed overhead a
// title actually has to fit inside.
const DOC_ID = '1'.repeat(44)
describe('gdocs document filenames', () => {
it('leads with the date when there is one', () => {
expect(makeFilename('My Document', 'abc123', '2026-03-15T10:00:00Z')).toBe(
'2026-03-15_My_Document__abc123.gdoc.json',
)
expect(makeFilename('My Document', 'abc123')).toBe('My_Document__abc123.gdoc.json')
})
it('fits NAME_MAX for a CJK title', () => {
// 100 characters of CJK is 300 bytes, which the character budget passed
// untouched: with the date, the id and the suffix the name came to 367
// bytes and ext4/APFS reject it with ENAMETOOLONG.
const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z')
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name.startsWith('2026-08-20_')).toBe(true)
expect(name.endsWith(`__${DOC_ID}.gdoc.json`)).toBe(true)
// The cut lands on a character boundary, never mid-sequence.
expect(name).not.toContain('\uFFFD')
})
it('leaves an ascii title on the character budget', () => {
const name = makeFilename('a'.repeat(400), DOC_ID, '')
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gdoc.json`)
})
it('keeps the id when it leaves no room', () => {
// The title is what gives, never the id: a trimmed id would stop
// addressing the document. Same rule as gcal's event filenames.
const longId = 'v'.repeat(NAME_MAX_BYTES - 4)
expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gdoc.json`)
})
})
@@ -12,17 +12,26 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { sanitizeLabel } from '../../utils/sanitize.ts'
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
const TITLE_MAX_CHARS = 100
const SUFFIX = '.gdoc.json'
const DATE_LEN = 10
const sanitizeTitle = (title: string): string =>
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS })
const sanitizeTitle = (title: string, maxBytes: number): string =>
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes })
/**
* Build a filename from title, doc ID, and modified date.
*
* The title takes whatever of the 255-byte NAME_MAX the date, the id and the
* suffix leave, rather than a flat character count: those are the same number
* only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS
* reject outright. The id never gives, so the name keeps addressing the
* document -- same rule as gcal's event filenames.
*/
export function makeFilename(title: string, docId: string, modifiedTime = ''): string {
const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : ''
if (datePrefix !== '') {
return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gdoc.json`
}
return `${sanitizeTitle(title)}__${docId}.gdoc.json`
const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : ''
const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length
return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}`
}
@@ -0,0 +1,55 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { makeFilename } from './sheet_entry.ts'
import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts'
// A real Google file id is 44 characters, so this is the fixed overhead a
// title actually has to fit inside.
const DOC_ID = '1'.repeat(44)
describe('gsheets spreadsheet filenames', () => {
it('leads with the date when there is one', () => {
expect(makeFilename('My Spreadsheet', 'abc123', '2026-03-15T10:00:00Z')).toBe(
'2026-03-15_My_Spreadsheet__abc123.gsheet.json',
)
expect(makeFilename('My Spreadsheet', 'abc123')).toBe('My_Spreadsheet__abc123.gsheet.json')
})
it('fits NAME_MAX for a CJK title', () => {
// 100 characters of CJK is 300 bytes, which the character budget passed
// untouched: with the date, the id and the suffix the name came to 367
// bytes and ext4/APFS reject it with ENAMETOOLONG.
const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z')
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name.startsWith('2026-08-20_')).toBe(true)
expect(name.endsWith(`__${DOC_ID}.gsheet.json`)).toBe(true)
// The cut lands on a character boundary, never mid-sequence.
expect(name).not.toContain('\uFFFD')
})
it('leaves an ascii title on the character budget', () => {
const name = makeFilename('a'.repeat(400), DOC_ID, '')
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gsheet.json`)
})
it('keeps the id when it leaves no room', () => {
// The title is what gives, never the id: a trimmed id would stop
// addressing the spreadsheet. Same rule as gcal's event filenames.
const longId = 'v'.repeat(NAME_MAX_BYTES - 4)
expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gsheet.json`)
})
})
@@ -12,17 +12,26 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { sanitizeLabel } from '../../utils/sanitize.ts'
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
const TITLE_MAX_CHARS = 100
const SUFFIX = '.gsheet.json'
const DATE_LEN = 10
const sanitizeTitle = (title: string): string =>
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS })
const sanitizeTitle = (title: string, maxBytes: number): string =>
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes })
/**
* Build a filename from title, doc ID, and modified date.
*
* The title takes whatever of the 255-byte NAME_MAX the date, the id and the
* suffix leave, rather than a flat character count: those are the same number
* only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS
* reject outright. The id never gives, so the name keeps addressing the
* document -- same rule as gcal's event filenames.
*/
export function makeFilename(title: string, docId: string, modifiedTime = ''): string {
const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : ''
if (datePrefix !== '') {
return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gsheet.json`
}
return `${sanitizeTitle(title)}__${docId}.gsheet.json`
const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : ''
const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length
return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}`
}
@@ -0,0 +1,55 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { makeFilename } from './slide_entry.ts'
import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts'
// A real Google file id is 44 characters, so this is the fixed overhead a
// title actually has to fit inside.
const DOC_ID = '1'.repeat(44)
describe('gslides presentation filenames', () => {
it('leads with the date when there is one', () => {
expect(makeFilename('My Presentation', 'abc123', '2026-03-15T10:00:00Z')).toBe(
'2026-03-15_My_Presentation__abc123.gslide.json',
)
expect(makeFilename('My Presentation', 'abc123')).toBe('My_Presentation__abc123.gslide.json')
})
it('fits NAME_MAX for a CJK title', () => {
// 100 characters of CJK is 300 bytes, which the character budget passed
// untouched: with the date, the id and the suffix the name came to 367
// bytes and ext4/APFS reject it with ENAMETOOLONG.
const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z')
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name.startsWith('2026-08-20_')).toBe(true)
expect(name.endsWith(`__${DOC_ID}.gslide.json`)).toBe(true)
// The cut lands on a character boundary, never mid-sequence.
expect(name).not.toContain('\uFFFD')
})
it('leaves an ascii title on the character budget', () => {
const name = makeFilename('a'.repeat(400), DOC_ID, '')
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gslide.json`)
})
it('keeps the id when it leaves no room', () => {
// The title is what gives, never the id: a trimmed id would stop
// addressing the presentation. Same rule as gcal's event filenames.
const longId = 'v'.repeat(NAME_MAX_BYTES - 4)
expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gslide.json`)
})
})
@@ -12,17 +12,26 @@
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { sanitizeLabel } from '../../utils/sanitize.ts'
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
const TITLE_MAX_CHARS = 100
const SUFFIX = '.gslide.json'
const DATE_LEN = 10
const sanitizeTitle = (title: string): string =>
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS })
const sanitizeTitle = (title: string, maxBytes: number): string =>
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes })
/**
* Build a filename from title, doc ID, and modified date.
*
* The title takes whatever of the 255-byte NAME_MAX the date, the id and the
* suffix leave, rather than a flat character count: those are the same number
* only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS
* reject outright. The id never gives, so the name keeps addressing the
* document -- same rule as gcal's event filenames.
*/
export function makeFilename(title: string, docId: string, modifiedTime = ''): string {
const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : ''
if (datePrefix !== '') {
return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gslide.json`
}
return `${sanitizeTitle(title)}__${docId}.gslide.json`
const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : ''
const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length
return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}`
}
@@ -13,7 +13,13 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { pathSafeName, sanitizeLabel, sanitizeName } from './sanitize.ts'
import {
NAME_MAX_BYTES,
byteLength,
pathSafeName,
sanitizeLabel,
sanitizeName,
} from './sanitize.ts'
describe('sanitizeName', () => {
it('returns "unknown" for empty/whitespace input', () => {
@@ -98,13 +104,42 @@ describe('sanitizeLabel', () => {
})
it('ellipsizes on a code-point boundary', () => {
// A byte budget wide enough to stay out of the way, so this pins the
// character budget alone.
const label = '\u{10400}'.repeat(120)
const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100 })
const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100, maxBytes: 10_000 })
expect(Array.from(out)).toHaveLength(100)
expect(out.endsWith('...')).toBe(true)
expect(out).not.toContain('\uFFFD')
})
it('honors the byte ceiling within the character budget', () => {
// 100 astral code points is 400 bytes, so a name the character budget
// accepts is one ext4 and APFS reject with ENAMETOOLONG. The default
// budget is NAME_MAX, and the cut still lands on a code-point boundary.
const label = '\u{10400}'.repeat(120)
const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100 })
expect(Array.from(out).length).toBeLessThan(100)
expect(byteLength(out)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(out.endsWith('...')).toBe(true)
expect(out).not.toContain('\uFFFD')
})
it('takes the byte budget as the caller remaining room', () => {
// What the gdocs/gmail filenames pass: NAME_MAX minus the id, the
// separators and the suffix.
const out = sanitizeLabel('会'.repeat(200), { fallback: 'X', maxLen: 100, maxBytes: 60 })
expect(byteLength(out)).toBeLessThanOrEqual(60)
expect(out.endsWith('...')).toBe(true)
expect(out).not.toContain('\uFFFD')
})
it('drops the ellipsis when it cannot fit', () => {
// Three dots and nothing is not a name; a budget this small yields
// whatever of the label actually fits.
expect(sanitizeLabel('abcdef', { fallback: 'X', maxLen: 100, maxBytes: 2 })).toBe('ab')
})
it('keeps non-ascii letters, matching python', () => {
// The per-backend copies this replaced used `\w`, which is ascii-only in
// javascript, so a CJK title became a row of underscores while python --
+34 -2
View File
@@ -20,10 +20,16 @@ const MAX_LEN = 100
// characters is the same number only for ASCII: a 100-character CJK title is
// 300 bytes.
export const NAME_MAX_BYTES = 255
const ELLIPSIS = '...'
const UTF8 = new TextEncoder()
const UTF8_DECODER = new TextDecoder('utf-8')
/** Measure a string the way the filesystem does: in UTF-8 bytes. */
export function byteLength(text: string): number {
return UTF8.encode(text).length
}
/**
* Trim a string to fit a byte budget without splitting a character.
*
@@ -51,6 +57,13 @@ export function stripTrailingUnderscores(value: string): string {
return value.slice(0, end)
}
/** python's `str.rstrip("_.")`, so a byte cut cannot leave `Foo.....`. */
function stripTrailingUnderscoresAndDots(value: string): string {
let end = value.length
while (end > 0 && (value[end - 1] === '_' || value[end - 1] === '.')) end -= 1
return value.slice(0, end)
}
export function stripUnderscores(value: string): string {
let start = 0
let end = value.length
@@ -101,8 +114,20 @@ export function pathSafeName(name: string): string {
*
* Unlike `sanitizeName` this ellipsizes rather than hard-cutting, so a
* truncated name reads as truncated.
*
* Two budgets apply, and both have to: `maxLen` is the readable length a
* backend wants, while `maxBytes` is what the filesystem will actually
* accept. They are the same number only for ASCII, so a 100-character CJK
* title passed a 100-character budget untouched and rendered a 300-byte
* filename, which ext4 and APFS reject with ENAMETOOLONG. Pass the bytes the
* *rest* of the filename does not already use -- see `makeFilename` in the
* gdocs/gsheets/gslides entries and `makeEventFilename` in gcal, which is
* where the fixed overhead is known.
*/
export function sanitizeLabel(text: string, options: { fallback: string; maxLen: number }): string {
export function sanitizeLabel(
text: string,
options: { fallback: string; maxLen: number; maxBytes?: number },
): string {
if (text.trim() === '') return options.fallback
let cleaned = text.replace(UNSAFE_CHARS, '_').replace(/ /g, '_').replace(MULTI_UNDERSCORE, '_')
cleaned = stripUnderscores(cleaned)
@@ -112,7 +137,14 @@ export function sanitizeLabel(text: string, options: { fallback: string; maxLen:
// encodes as U+FFFD in the filename.
const points = Array.from(cleaned)
if (points.length > options.maxLen) {
cleaned = `${points.slice(0, options.maxLen - 3).join('')}...`
cleaned = `${points.slice(0, options.maxLen - ELLIPSIS.length).join('')}${ELLIPSIS}`
}
const maxBytes = options.maxBytes ?? NAME_MAX_BYTES
if (byteLength(cleaned) > maxBytes) {
const head = stripTrailingUnderscoresAndDots(
truncateBytes(cleaned, Math.max(maxBytes - ELLIPSIS.length, 0)),
)
cleaned = head !== '' ? `${head}${ELLIPSIS}` : truncateBytes(cleaned, maxBytes)
}
return cleaned
}
@@ -68,18 +68,7 @@ export function buildEmailConfig(input: EmailConfigInput): EmailConfig {
}
export function normalizeEmailConfig(input: Record<string, unknown>): EmailConfig {
const norm = normalizeFields(input, {
rename: {
imap_host: 'imapHost',
imap_port: 'imapPort',
smtp_host: 'smtpHost',
smtp_port: 'smtpPort',
use_ssl: 'useSsl',
max_messages: 'maxMessages',
save_copy: 'saveCopy',
sent_folder: 'sentFolder',
},
})
const norm = normalizeFields(input)
const built: EmailConfigInput = {
imapHost: asString(norm.imapHost),
smtpHost: asString(norm.smtpHost),
@@ -18,6 +18,7 @@ import { PathSpec as PathSpecCtor } from '@struktoai/mirage-core/types'
import type { PathSpec } from '@struktoai/mirage-core/types'
import { enoent } from '@struktoai/mirage-core/utils/errors'
import { mountKey, mountPrefixOf } from '@struktoai/mirage-core/utils/key_prefix'
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '@struktoai/mirage-core/utils/sanitize'
import { compareCodePoints } from '@struktoai/mirage-core/utils/sort'
import type { EmailAccessor } from '../../accessor/email.ts'
import { fetchHeaders, listMessageUids, type FetchedMessage } from './client.ts'
@@ -26,20 +27,26 @@ import { messageJsonBytes } from './render.ts'
import type { ParsedAttachment } from './_parse.ts'
const TITLE_MAX = 80
const UNSAFE = /[^\w\s\-.]/g
const MULTI_UNDERSCORE = /_+/g
const EPOCH_DATE = '1970-01-01'
const MSG_SUFFIX = '.email.json'
export function sanitize(text: string): string {
if (text.trim() === '') return 'No_Subject'
let cleaned = text.replace(UNSAFE, '_').replace(/ /g, '_')
cleaned = cleaned.replace(MULTI_UNDERSCORE, '_').replace(/^_+|_+$/g, '')
if (cleaned.length > TITLE_MAX) cleaned = `${cleaned.slice(0, TITLE_MAX - 3)}...`
return cleaned
}
// Routed through the shared sanitizer rather than a local copy of it. The
// copy's `\w` was JS's ASCII-only one where python's is unicode, so every
// accented or CJK subject came back as a row of underscores here and intact
// there; it also measured the budget in UTF-16 units instead of code points.
const sanitize = (text: string, maxBytes?: number): string =>
sanitizeLabel(text, {
fallback: 'No_Subject',
maxLen: TITLE_MAX,
...(maxBytes !== undefined ? { maxBytes } : {}),
})
function msgFilename(subject: string, uid: string): string {
return `${sanitize(subject)}__${uid}.email.json`
// 80 characters is 240 bytes of CJK, which overflows the 255-byte NAME_MAX
// once the uid and `.email.json` are added, so the subject takes what they
// leave rather than a flat character count.
export function msgFilename(subject: string, uid: string): string {
const fixed = 2 + byteLength(uid) + MSG_SUFFIX.length
return `${sanitize(subject, NAME_MAX_BYTES - fixed)}__${uid}${MSG_SUFFIX}`
}
// RFC 5322's obsolete zone names, the set `parsedate_to_datetime` knows.
@@ -0,0 +1,41 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { NAME_MAX_BYTES, byteLength } from '@struktoai/mirage-core/utils/sanitize'
import { describe, expect, it } from 'vitest'
import { msgFilename } from './readdir.ts'
import { buildVfsPath } from './search.ts'
const CJK_SUBJECT = '会議の記録'.repeat(40)
const MSG = { subject: CJK_SUBJECT, uid: '7', date: 'Mon, 5 Jan 2026 10:00:00 +0000' }
describe('email search paths', () => {
it('names the file readdir created', () => {
// Composed here from a bare `sanitize`, a hit pointed at a path that does
// not exist as soon as the subject was long enough to be trimmed: readdir
// budgets the subject against the uid and the suffix, and this did not,
// so the two names differed.
const path = buildVfsPath('/mail', 'INBOX', MSG as never)
expect(path.endsWith(`/${msgFilename(CJK_SUBJECT, '7')}`)).toBe(true)
})
it('fits NAME_MAX', () => {
const name =
buildVfsPath('/mail', 'INBOX', MSG as never)
.split('/')
.pop() ?? ''
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
expect(name).not.toContain('\uFFFD')
})
})
@@ -14,7 +14,7 @@
import type { EmailAccessor } from '../../accessor/email.ts'
import { fetchMessage, listMessageUids, type FetchedMessage } from './client.ts'
import { dateBucket, sanitize } from './readdir.ts'
import { dateBucket, msgFilename } from './readdir.ts'
import { messageJsonText } from './render.ts'
import type { EmailScope } from './scope.ts'
@@ -62,10 +62,13 @@ async function searchMessages(
return listMessageUids(accessor, folder, criteria, maxResults)
}
function buildVfsPath(prefix: string, folder: string, msg: FetchedMessage): string {
export function buildVfsPath(prefix: string, folder: string, msg: FetchedMessage): string {
const dateStr = dateBucket(msg)
const subject = sanitize(msg.subject !== '' ? msg.subject : 'No Subject')
const filename = `${subject}__${msg.uid}.email.json`
// The same builder readdir names the file with, not a second spelling of
// it: the subject's budget depends on the uid and the suffix, so a hit
// composed here from a bare `sanitize` pointed at a path that does not
// exist as soon as a long subject was trimmed differently.
const filename = msgFilename(msg.subject !== '' ? msg.subject : 'No Subject', msg.uid)
return [prefix, folder, dateStr, filename].filter((p) => p !== '').join('/')
}
@@ -70,13 +70,6 @@ export function redactBoxConfig(config: BoxConfig): BoxConfigRedacted {
export function normalizeBoxConfig(input: Record<string, unknown>): BoxConfig {
return normalizeFields(input, {
rename: {
root_folder_id: 'rootFolderId',
content_search: 'contentSearch',
client_id: 'clientId',
client_secret: 'clientSecret',
refresh_token: 'refreshToken',
enterprise_id: 'enterpriseId',
access_token: 'accessToken',
developer_token: 'accessToken',
},
}) as unknown as BoxConfig
@@ -47,13 +47,5 @@ export function redactDropboxConfig(config: DropboxConfig): DropboxConfigRedacte
}
export function normalizeDropboxConfig(input: Record<string, unknown>): DropboxConfig {
return normalizeFields(input, {
rename: {
client_id: 'clientId',
client_secret: 'clientSecret',
refresh_token: 'refreshToken',
root_path: 'rootPath',
content_search: 'contentSearch',
},
}) as unknown as DropboxConfig
return normalizeFields(input) as unknown as DropboxConfig
}
@@ -81,13 +81,9 @@ export function redactGcsConfig(config: GCSConfig): GCSConfigRedacted {
export function normalizeGcsConfig(input: Record<string, unknown>): GCSConfig {
return normalizeFields(input, {
rename: {
access_key_id: 'accessKeyId',
secret_access_key: 'secretAccessKey',
session_token: 'sessionToken',
aws_profile: 'profile',
endpoint_url: 'endpoint',
path_style: 'forcePathStyle',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {
@@ -39,12 +39,7 @@ export function redactConfig(config: GridFSConfig): GridFSConfigRedacted {
* normalize the key prefix the way the accessor expects.
*/
export function normalizeGridFSConfig(input: Record<string, unknown>): GridFSConfig {
const norm = normalizeFields(input, {
rename: {
key_prefix: 'keyPrefix',
chunk_size_bytes: 'chunkSizeBytes',
},
}) as unknown as GridFSConfig
const norm = normalizeFields(input) as unknown as GridFSConfig
const prefix = normalizeKeyPrefix(norm.keyPrefix)
if (prefix !== undefined) {
norm.keyPrefix = prefix
@@ -59,7 +59,6 @@ export function redactHfBucketsConfig(config: HfBucketsConfig): HfBucketsConfigR
export function normalizeHfBucketsConfig(input: Record<string, unknown>): HfBucketsConfig {
const config = normalizeFields(input, {
rename: {
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {
@@ -102,8 +101,6 @@ export function redactHfRepoConfig(config: HfRepoConfig): HfRepoConfigRedacted {
export function normalizeHfRepoConfig(input: Record<string, unknown>): HfRepoConfig {
const config = normalizeFields(input, {
rename: {
repo_id: 'repoId',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {
@@ -34,12 +34,5 @@ export function redactJaegerConfig(config: JaegerConfig): JaegerConfigRedacted {
}
export function normalizeJaegerConfig(input: Record<string, unknown>): JaegerConfig {
return normalizeFields(input, {
rename: {
default_trace_limit: 'defaultTraceLimit',
default_from_timestamp: 'defaultFromTimestamp',
default_to_timestamp: 'defaultToTimestamp',
request_timeout: 'requestTimeout',
},
}) as unknown as JaegerConfig
return normalizeFields(input) as unknown as JaegerConfig
}
@@ -34,13 +34,5 @@ export function redactLangfuseConfig(config: LangfuseConfig): LangfuseConfigReda
}
export function normalizeLangfuseConfig(input: Record<string, unknown>): LangfuseConfig {
return normalizeFields(input, {
rename: {
public_key: 'publicKey',
secret_key: 'secretKey',
default_trace_limit: 'defaultTraceLimit',
default_search_limit: 'defaultSearchLimit',
default_from_timestamp: 'defaultFromTimestamp',
},
}) as unknown as LangfuseConfig
return normalizeFields(input) as unknown as LangfuseConfig
}
@@ -83,12 +83,8 @@ export function redactOciConfig(config: OCIConfig): OCIConfigRedacted {
export function normalizeOciConfig(input: Record<string, unknown>): OCIConfig {
return normalizeFields(input, {
rename: {
access_key_id: 'accessKeyId',
secret_access_key: 'secretAccessKey',
session_token: 'sessionToken',
aws_profile: 'profile',
endpoint_url: 'endpoint',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {
@@ -89,14 +89,9 @@ export function redactR2Config(config: R2Config): R2ConfigRedacted {
export function normalizeR2Config(input: Record<string, unknown>): R2Config {
return normalizeFields(input, {
rename: {
account_id: 'accountId',
access_key_id: 'accessKeyId',
secret_access_key: 'secretAccessKey',
session_token: 'sessionToken',
aws_profile: 'profile',
endpoint_url: 'endpoint',
path_style: 'forcePathStyle',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {
@@ -68,7 +68,6 @@ export function normalizeS3Config(input: Record<string, unknown>): S3Config {
endpoint_url: 'endpoint',
path_style: 'forcePathStyle',
timeout: 'timeoutMs',
key_prefix: 'keyPrefix',
},
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
@@ -52,16 +52,14 @@ export interface S3AliasConfig {
proxy?: string
}
// Every provider accepts the same snake_case spelling of its fields, and
// Python states timeouts in seconds where TypeScript uses milliseconds.
// Only the entries `snakeToCamel` would get wrong: python spells the profile
// `aws_profile`, the endpoint `endpoint_url`, path style `path_style`, and
// states timeouts in seconds where TypeScript uses milliseconds. Every other
// field maps by default, so it does not belong here.
const RENAME: Record<string, string> = {
access_key_id: 'accessKeyId',
secret_access_key: 'secretAccessKey',
session_token: 'sessionToken',
aws_profile: 'profile',
endpoint_url: 'endpoint',
path_style: 'forcePathStyle',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
}
@@ -38,10 +38,5 @@ export function redactSshConfig(config: SSHConfig): SSHConfigRedacted {
}
export function normalizeSshConfig(input: Record<string, unknown>): SSHConfig {
return normalizeFields(input, {
rename: {
identity_file: 'identityFile',
known_hosts: 'knownHosts',
},
}) as unknown as SSHConfig
return normalizeFields(input) as unknown as SSHConfig
}
@@ -86,13 +86,8 @@ export function redactSupabaseConfig(config: SupabaseConfig): SupabaseConfigReda
export function normalizeSupabaseConfig(input: Record<string, unknown>): SupabaseConfig {
return normalizeFields(input, {
rename: {
project_ref: 'projectRef',
access_key_id: 'accessKeyId',
secret_access_key: 'secretAccessKey',
session_token: 'sessionToken',
aws_profile: 'profile',
endpoint_url: 'endpoint',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {
@@ -33,13 +33,5 @@ export function redactTrelloConfig(config: TrelloConfig): TrelloConfigRedacted {
}
export function normalizeTrelloConfig(input: Record<string, unknown>): TrelloConfig {
return normalizeFields(input, {
rename: {
api_key: 'apiKey',
api_token: 'apiToken',
workspace_id: 'workspaceId',
board_ids: 'boardIds',
base_url: 'baseUrl',
},
}) as unknown as TrelloConfig
return normalizeFields(input) as unknown as TrelloConfig
}
@@ -85,13 +85,9 @@ export function redactWasabiConfig(config: WasabiConfig): WasabiConfigRedacted {
export function normalizeWasabiConfig(input: Record<string, unknown>): WasabiConfig {
return normalizeFields(input, {
rename: {
access_key_id: 'accessKeyId',
secret_access_key: 'secretAccessKey',
session_token: 'sessionToken',
aws_profile: 'profile',
endpoint_url: 'endpoint',
path_style: 'forcePathStyle',
key_prefix: 'keyPrefix',
timeout: 'timeoutMs',
},
transform: {