fix(parity): github ref default, NAME_MAX byte budget, browser registry onto node's config shape
Two py<->ts divergences, one structural alignment, one swept-and-gated class of config noise. 1. GitHub `ref` defaulted to the literal "main" in python where TypeScript resolves `config.ref ?? repoInfo.default_branch`, so every repository whose default branch is something else 404d on the one tree fetch the mount is built on and read as empty. `GitHubConfig.ref` is now optional, and `ensure_ref` resolves it lazily beside the existing `ensure_default_branch` -- python's constructor deliberately does no network. Every reader that needs a concrete ref goes through it. `mirage.accessor.github` is fully annotated and leaves mypy's untyped-defs allowlist. 2. `sanitize_label` capped characters where NAME_MAX counts bytes, which the constant's own comment already spelled out. gdocs/gsheets/gslides rendered ~367-byte filenames for a CJK title and gmail/email ~269; ext4 and APFS reject those. `sanitize_label`/`sanitizeLabel` grew a byte budget and every compose site passes what the date, id and suffix leave. Found alongside: TS email's hand-rolled copy of the sanitizer used JS's ASCII-only `\w` (so non-ASCII subjects became underscores there and stayed intact in python), and three sites re-composed `<subject>__<id>.<ext>` independently of readdir, so a search hit could name a path that does not exist. 3. The browser registry hand-rolled `normalizeFields` per entry and cast through 7 `*BrowserCtorConfig` interfaces -- hand-written twins of the real config types. It now delegates to a `normalizeXConfig` beside each config, as node's registry does; the interfaces and 22 double casts are gone. 4. 95 rename entries that only restated `snakeToCamel` deleted across 24 files, with `scripts/check_normalize_renames.py` and a CI step to keep them from returning. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -93,6 +93,14 @@ jobs:
|
||||
- name: Core barrel surface
|
||||
run: ./python/.venv/bin/python scripts/check_barrel_surface.py
|
||||
|
||||
# `normalizeFields` already maps every unlisted key through
|
||||
# `snakeToCamel`, so a rename entry that restates that mapping is not
|
||||
# configuration -- it is a second place for one fact to be wrong, and it
|
||||
# buries the handful that are real overrides (`endpoint_url`, `timeout`,
|
||||
# `aws_profile`) in a wall of noise. 95 had accumulated.
|
||||
- name: Redundant config renames
|
||||
run: ./python/.venv/bin/python scripts/check_normalize_renames.py
|
||||
|
||||
# The docs build parses frontmatter as YAML and stops on the first page
|
||||
# it cannot read, so an unquoted description holding ": " fails the
|
||||
# deploy with no local signal at all.
|
||||
|
||||
@@ -15,23 +15,27 @@
|
||||
import asyncio
|
||||
|
||||
from mirage.accessor.base import Accessor
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
|
||||
|
||||
class GitHubAccessor(Accessor):
|
||||
|
||||
def __init__(self,
|
||||
config,
|
||||
owner,
|
||||
repo,
|
||||
ref,
|
||||
config: GitHubConfig,
|
||||
owner: str,
|
||||
repo: str,
|
||||
ref: str | None = None,
|
||||
default_branch: str | None = None,
|
||||
tree: dict[str, TreeEntry] | None = None,
|
||||
truncated=False):
|
||||
truncated: bool = False) -> None:
|
||||
self.config = config
|
||||
self.owner = owner
|
||||
self.repo = repo
|
||||
self.ref = ref
|
||||
# None until resolved: an unpinned mount follows the repository's
|
||||
# default branch, which costs a request to learn, so the ref is
|
||||
# settled on the first read that needs one (`ensure_ref`).
|
||||
self.ref: str | None = ref
|
||||
# None until hydrated: the mount is constructed without touching
|
||||
# the network, so the repo's default branch is fetched on the
|
||||
# first read that needs it (`ensure_default_branch`).
|
||||
|
||||
@@ -27,7 +27,7 @@ from mirage.commands.registry import command
|
||||
from mirage.commands.spec import SPECS
|
||||
from mirage.commands.spec.types import FlagView
|
||||
from mirage.core.email.client import fetch_headers
|
||||
from mirage.core.email.readdir import _date_bucket, _sanitize
|
||||
from mirage.core.email.readdir import _date_bucket, _msg_filename
|
||||
from mirage.core.email.readdir import readdir as _readdir
|
||||
from mirage.core.email.search import search_messages
|
||||
from mirage.core.email.stat import stat as _stat
|
||||
@@ -163,9 +163,12 @@ async def _find_server_side(
|
||||
results: list[str] = []
|
||||
for h in headers:
|
||||
date_str = _date_bucket(h)
|
||||
subject = _sanitize(h.get("subject", "No Subject"))
|
||||
uid = h.get("uid", "")
|
||||
filename = f"{subject}__{uid}.email.json"
|
||||
# The same builder readdir names the file with, not a second
|
||||
# spelling of it: the subject's budget depends on the uid and the
|
||||
# suffix, so a hit composed from a bare `_sanitize` pointed at a
|
||||
# path that does not exist once a long subject was trimmed.
|
||||
filename = _msg_filename(h.get("subject", "No Subject"), uid)
|
||||
if fnmatch(filename, name_pattern):
|
||||
vfs_path = "/".join(p
|
||||
for p in [prefix, folder, date_str, filename]
|
||||
|
||||
@@ -20,7 +20,7 @@ from mirage.commands.builtin.grep_helper import (is_literal_pattern,
|
||||
from mirage.core.github.constants import SCOPE_WARN
|
||||
from mirage.core.github.pushdown import (count_scope_files, scope_relative_key,
|
||||
should_use_search)
|
||||
from mirage.core.github.repo import ensure_default_branch
|
||||
from mirage.core.github.repo import ensure_default_branch, ensure_ref
|
||||
from mirage.core.github.search import narrow_paths
|
||||
from mirage.core.github.tree import ensure_tree
|
||||
from mirage.types import PathSpec
|
||||
@@ -87,7 +87,7 @@ async def narrow_scope(
|
||||
use_search = (query is not None and whole_word and literal
|
||||
and file_count > SCOPE_WARN and should_use_search(
|
||||
recursive=recursive,
|
||||
on_default_branch=(accessor.ref == await
|
||||
on_default_branch=(await ensure_ref(accessor) == await
|
||||
ensure_default_branch(accessor)),
|
||||
))
|
||||
if use_search:
|
||||
|
||||
@@ -25,16 +25,22 @@ from mirage.core.email.render import message_json_bytes
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.errors import enoent
|
||||
from mirage.utils.key_prefix import mount_key, mount_prefix_of
|
||||
from mirage.utils.sanitize import sanitize_label
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
|
||||
|
||||
TITLE_MAX = 80
|
||||
EPOCH_DATE = "1970-01-01"
|
||||
MSG_SUFFIX = ".email.json"
|
||||
|
||||
_sanitize = partial(sanitize_label, fallback="No_Subject", max_len=TITLE_MAX)
|
||||
|
||||
|
||||
def _msg_filename(subject: str, uid: str) -> str:
|
||||
return f"{_sanitize(subject)}__{uid}.email.json"
|
||||
# 80 characters is 240 bytes of CJK, which overflows the 255-byte
|
||||
# NAME_MAX once the uid and `.email.json` are added, so the subject
|
||||
# takes what they leave rather than a flat character count.
|
||||
fixed = len("__") + byte_len(uid) + len(MSG_SUFFIX)
|
||||
label = _sanitize(subject, max_bytes=NAME_MAX_BYTES - fixed)
|
||||
return f"{label}__{uid}{MSG_SUFFIX}"
|
||||
|
||||
|
||||
def _parse_date(value: str) -> str | None:
|
||||
|
||||
@@ -16,7 +16,7 @@ from typing import Any
|
||||
|
||||
from mirage.accessor.email import EmailAccessor
|
||||
from mirage.core.email.client import fetch_message, list_message_uids
|
||||
from mirage.core.email.readdir import _date_bucket, _sanitize
|
||||
from mirage.core.email.readdir import _date_bucket, _msg_filename
|
||||
from mirage.core.email.render import message_json_text
|
||||
from mirage.core.email.scope import EmailScope
|
||||
|
||||
@@ -77,9 +77,12 @@ async def search_messages(
|
||||
|
||||
def _build_vfs_path(prefix: str, folder: str, msg: dict[str, Any]) -> str:
|
||||
date_str = _date_bucket(msg)
|
||||
subject = _sanitize(msg.get("subject", "No Subject"))
|
||||
uid = msg.get("uid", "")
|
||||
filename = f"{subject}__{uid}.email.json"
|
||||
# The same builder readdir names the file with, not a second spelling of
|
||||
# it: the subject's budget depends on the uid and the suffix, so a hit
|
||||
# composed here from a bare `_sanitize` pointed at a path that does not
|
||||
# exist as soon as a long subject was trimmed differently.
|
||||
filename = _msg_filename(msg.get("subject", "No Subject"), uid)
|
||||
parts = [prefix, folder, date_str, filename]
|
||||
return "/".join(p for p in parts if p)
|
||||
|
||||
|
||||
@@ -19,7 +19,11 @@ class GitHubConfig(BaseModel):
|
||||
token: SecretStr
|
||||
owner: str | None = None
|
||||
repo: str | None = None
|
||||
ref: str = "main"
|
||||
# None means "whatever the repository's default branch is", resolved on
|
||||
# the first read through ``ensure_ref``. It is not a synonym for "main":
|
||||
# defaulting to that string mounted a nonexistent ref on every repo whose
|
||||
# default is `master`, and the tree fetch 404s rather than falling back.
|
||||
ref: str | None = None
|
||||
base_url: str | None = None
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import logging
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
|
||||
LookupStatus)
|
||||
from mirage.core.github.repo import ensure_ref
|
||||
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
|
||||
refill_index)
|
||||
from mirage.types import PathSpec
|
||||
@@ -120,7 +121,7 @@ async def _resolve_dir_sha(
|
||||
stem = prefix.rstrip("/")
|
||||
rest = norm[len(stem):] if stem and norm.startswith(stem) else norm
|
||||
parts = [p for p in rest.strip("/").split("/") if p]
|
||||
current_sha = accessor.ref
|
||||
current_sha = await ensure_ref(accessor)
|
||||
current_path = stem
|
||||
for part in parts:
|
||||
entries = await fetch_dir_tree(accessor.config, accessor.owner,
|
||||
|
||||
@@ -61,6 +61,34 @@ async def ensure_default_branch(accessor: GitHubAccessor, ) -> str:
|
||||
return accessor.default_branch
|
||||
|
||||
|
||||
async def ensure_ref(accessor: GitHubAccessor) -> str:
|
||||
"""Settle which ref this mount reads, fetching the default branch once.
|
||||
|
||||
A mount that named no ref follows the repository's default branch, and
|
||||
learning that costs a request the constructor cannot make. Every reader
|
||||
that needs a concrete ref -- the tree fetches, the watch walk, readdir's
|
||||
per-directory descent -- goes through here instead of reading
|
||||
``accessor.ref`` directly, so an unpinned mount resolves exactly once and
|
||||
then behaves like a pinned one.
|
||||
|
||||
Defaulting to the string ``"main"`` instead was the bug this replaces: a
|
||||
repository whose default branch is ``master`` (or anything else) 404s on
|
||||
every tree fetch, so the whole mount reads as empty.
|
||||
|
||||
Args:
|
||||
accessor (GitHubAccessor): the mount's accessor.
|
||||
|
||||
Returns:
|
||||
str: the ref to read, as named by the mount or as resolved from the
|
||||
repository's default branch.
|
||||
"""
|
||||
if accessor.ref is not None:
|
||||
return accessor.ref
|
||||
resolved = await ensure_default_branch(accessor)
|
||||
accessor.ref = resolved
|
||||
return resolved
|
||||
|
||||
|
||||
def parse_repo(spec: str) -> RepoRef:
|
||||
"""Split gh's `[HOST/]OWNER/REPO`.
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from mirage.cache.index import (NULL_INDEX, IndexCacheStore, IndexEntry,
|
||||
LookupStatus)
|
||||
from mirage.core.github.client import github_get
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.repo import ensure_ref
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -201,8 +202,9 @@ async def refill_index(
|
||||
"""
|
||||
if index is NULL_INDEX:
|
||||
return False
|
||||
ref = await ensure_ref(accessor)
|
||||
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, accessor.ref)
|
||||
accessor.repo, ref)
|
||||
accessor.truncated = truncated
|
||||
accessor.tree = tree
|
||||
accessor.tree_loaded = True
|
||||
@@ -301,8 +303,9 @@ async def ensure_tree(
|
||||
await ensure_live_index(accessor, index, prefix)
|
||||
if accessor.tree_loaded:
|
||||
return
|
||||
ref = await ensure_ref(accessor)
|
||||
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, accessor.ref)
|
||||
accessor.repo, ref)
|
||||
accessor.truncated = truncated
|
||||
accessor.tree = tree
|
||||
accessor.tree_loaded = True
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.core.github.repo import ensure_ref
|
||||
from mirage.core.github.tree import fetch_tree
|
||||
from mirage.types import PathSpec, WalkEntry
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
@@ -58,12 +59,13 @@ class GitHubWalk:
|
||||
"""
|
||||
accessor = self._accessor
|
||||
prefix = mount_prefix_of(root.virtual, root.resource_path)
|
||||
ref = await ensure_ref(accessor)
|
||||
tree, truncated = await fetch_tree(accessor.config, accessor.owner,
|
||||
accessor.repo, accessor.ref)
|
||||
accessor.repo, ref)
|
||||
if truncated:
|
||||
raise IncompleteWalkError(
|
||||
f"github tree for {accessor.owner}/{accessor.repo}"
|
||||
f"@{accessor.ref} was truncated; cannot diff a partial tree")
|
||||
f"@{ref} was truncated; cannot diff a partial tree")
|
||||
# A complete tree for the ref is exactly what the accessor holds,
|
||||
# and find/du/grep's scope counter read it directly. Discarding it
|
||||
# here left them answering from the tree the mount was built with
|
||||
|
||||
@@ -27,21 +27,43 @@ from mirage.core.gmail.messages import (_extract_attachments, _extract_header,
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.errors import enoent
|
||||
from mirage.utils.key_prefix import mount_key, mount_prefix_of
|
||||
from mirage.utils.sanitize import sanitize_label
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TITLE_MAX = 80
|
||||
MSG_SUFFIX = ".gmail.json"
|
||||
|
||||
_sanitize = partial(sanitize_label, fallback="No_Subject", max_len=TITLE_MAX)
|
||||
|
||||
|
||||
def _subject(subject: str, msg_id: str) -> str:
|
||||
"""Sanitize a subject to fit what the id and suffix leave of NAME_MAX.
|
||||
|
||||
80 characters is 240 bytes of CJK, which overflows the 255-byte
|
||||
NAME_MAX once the id and `.gmail.json` are added; the filesystem
|
||||
rejects the name outright. Both the message file and its attachment
|
||||
directory take the file's (stricter) budget so one subject renders
|
||||
the same in both, rather than the directory getting the eleven bytes
|
||||
the suffix would have used.
|
||||
|
||||
Args:
|
||||
subject (str): raw subject header.
|
||||
msg_id (str): the Gmail message id the name embeds.
|
||||
|
||||
Returns:
|
||||
str: the sanitized subject segment.
|
||||
"""
|
||||
fixed = len("__") + byte_len(msg_id) + len(MSG_SUFFIX)
|
||||
return _sanitize(subject, max_bytes=NAME_MAX_BYTES - fixed)
|
||||
|
||||
|
||||
def _msg_filename(subject: str, msg_id: str) -> str:
|
||||
return f"{_sanitize(subject)}__{msg_id}.gmail.json"
|
||||
return f"{_subject(subject, msg_id)}__{msg_id}{MSG_SUFFIX}"
|
||||
|
||||
|
||||
def _attach_dir_name(subject: str, msg_id: str) -> str:
|
||||
return f"{_sanitize(subject)}__{msg_id}"
|
||||
return f"{_subject(subject, msg_id)}__{msg_id}"
|
||||
|
||||
|
||||
def _attachment_filename(_attachment_id: str, filename: str) -> str:
|
||||
|
||||
@@ -18,7 +18,7 @@ from typing import Any
|
||||
from mirage.core.gmail.messages import (_decode_body, _extract_header,
|
||||
get_message_processed, get_message_raw,
|
||||
list_messages)
|
||||
from mirage.core.gmail.readdir import _sanitize
|
||||
from mirage.core.gmail.readdir import _msg_filename
|
||||
from mirage.core.gmail.scope import GmailScope
|
||||
from mirage.core.google.client import TokenManager
|
||||
|
||||
@@ -116,8 +116,11 @@ def format_grep_results(
|
||||
label = row.get("label") or scope.label_name or "INBOX"
|
||||
date = row.get("date", "")
|
||||
mid = row.get("id", "")
|
||||
subject_clean = _sanitize(row.get("subject") or "No Subject")
|
||||
filename = f"{subject_clean}__{mid}.gmail.json"
|
||||
# The same builder readdir names the file with, not a second
|
||||
# spelling of it: the subject's budget depends on the id and the
|
||||
# suffix, so a hit composed from a bare `_sanitize` pointed at a
|
||||
# path that does not exist once a long subject was trimmed.
|
||||
filename = _msg_filename(row.get("subject") or "No Subject", mid)
|
||||
sender = row.get("sender", "?")
|
||||
haystack = f"{row.get('subject', '')}\n{row.get('body_text', '')}"
|
||||
excerpt = _extract_excerpt(haystack, pattern) if pattern else ""
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.utils.naming import make_id_name
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, sanitize_name, truncate_bytes
|
||||
from mirage.utils.sanitize import (NAME_MAX_BYTES, byte_len, sanitize_name,
|
||||
truncate_bytes)
|
||||
|
||||
EVENT_SUFFIX = ".gcal.json"
|
||||
CALENDAR_FILE = "calendar.json"
|
||||
@@ -58,7 +59,7 @@ def make_event_filename(event_id: str, hhmm: str, title: str) -> str:
|
||||
Returns:
|
||||
str: e.g. ``la9i1t9...__0900-1030_PhD_Defense.gcal.json``.
|
||||
"""
|
||||
fixed = len(event_id.encode()) + len("__") + len(hhmm) + len("_") + len(
|
||||
fixed = byte_len(event_id) + len("__") + len(hhmm) + len("_") + len(
|
||||
EVENT_SUFFIX)
|
||||
trimmed = truncate_bytes(title, NAME_MAX_BYTES - fixed).rstrip("_")
|
||||
if not trimmed:
|
||||
|
||||
@@ -15,9 +15,11 @@
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
||||
from mirage.utils.sanitize import sanitize_label
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
|
||||
|
||||
TITLE_MAX_CHARS = 100
|
||||
SUFFIX = ".gdoc.json"
|
||||
DATE_LEN = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label,
|
||||
def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
|
||||
"""Build a filename from title, doc ID, and modified date.
|
||||
|
||||
The title takes whatever of the 255-byte NAME_MAX the date, the id and
|
||||
the suffix leave, rather than a flat character count: those are the same
|
||||
number only for ASCII, and a 100-character CJK title rendered a name ext4
|
||||
and APFS reject outright. The id never gives, so the name keeps
|
||||
addressing the document -- same rule as gcal's event filenames.
|
||||
|
||||
Args:
|
||||
title (str): raw document title.
|
||||
doc_id (str): Google Docs document ID.
|
||||
@@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
|
||||
Returns:
|
||||
str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json".
|
||||
"""
|
||||
date_prefix = modified_time[:10] if len(modified_time) >= 10 else ""
|
||||
if date_prefix:
|
||||
return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gdoc.json"
|
||||
return f"{sanitize_title(title)}__{doc_id}.gdoc.json"
|
||||
lead = (f"{modified_time[:DATE_LEN]}_"
|
||||
if len(modified_time) >= DATE_LEN else "")
|
||||
fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX)
|
||||
label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed)
|
||||
return f"{lead}{label}__{doc_id}{SUFFIX}"
|
||||
|
||||
@@ -81,7 +81,9 @@ class GitHubResource(BaseResource):
|
||||
repo (str | None): repository name; falls back to
|
||||
``config.repo``.
|
||||
ref (str | None): branch, tag or commit the mount is pinned
|
||||
to; falls back to ``config.ref``.
|
||||
to; falls back to ``config.ref``, and when neither names
|
||||
one the mount follows the repository's default branch,
|
||||
resolved on first read by ``ensure_ref``.
|
||||
default_branch (str | None): the repo's default branch, for
|
||||
``is_default_branch``. Fetched on first use when None.
|
||||
tree (dict[str, TreeEntry] | None): the recursive git tree,
|
||||
@@ -130,8 +132,12 @@ class GitHubResource(BaseResource):
|
||||
def is_default_branch(self) -> bool | None:
|
||||
"""Whether the mount is pinned to the repo's default branch.
|
||||
|
||||
``None`` means not known yet, not "no": the default branch is
|
||||
fetched on first use, and until something calls
|
||||
An unpinned mount answers True without a request: naming no ref
|
||||
*means* following the default branch, so the two agree whatever
|
||||
that branch turns out to be.
|
||||
|
||||
Otherwise ``None`` means not known yet, not "no": the default
|
||||
branch is fetched on first use, and until something calls
|
||||
:func:`mirage.core.github.repo.ensure_default_branch` there is
|
||||
nothing to compare ``ref`` against. Answering ``False`` there
|
||||
would be a wrong answer rather than an absent one, and an
|
||||
@@ -146,6 +152,8 @@ class GitHubResource(BaseResource):
|
||||
Returns:
|
||||
bool | None: the comparison, or None if not yet hydrated.
|
||||
"""
|
||||
if self.accessor.ref is None:
|
||||
return True
|
||||
if self.accessor.default_branch is None:
|
||||
return None
|
||||
return self.accessor.ref == self.accessor.default_branch
|
||||
|
||||
@@ -15,9 +15,11 @@
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
||||
from mirage.utils.sanitize import sanitize_label
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
|
||||
|
||||
TITLE_MAX_CHARS = 100
|
||||
SUFFIX = ".gsheet.json"
|
||||
DATE_LEN = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label,
|
||||
def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
|
||||
"""Build a filename from title, doc ID, and modified date.
|
||||
|
||||
The title takes whatever of the 255-byte NAME_MAX the date, the id and
|
||||
the suffix leave, rather than a flat character count: those are the same
|
||||
number only for ASCII, and a 100-character CJK title rendered a name ext4
|
||||
and APFS reject outright. The id never gives, so the name keeps
|
||||
addressing the document -- same rule as gcal's event filenames.
|
||||
|
||||
Args:
|
||||
title (str): raw document title.
|
||||
doc_id (str): Google Sheets spreadsheet ID.
|
||||
@@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
|
||||
Returns:
|
||||
str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json".
|
||||
"""
|
||||
date_prefix = modified_time[:10] if len(modified_time) >= 10 else ""
|
||||
if date_prefix:
|
||||
return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gsheet.json"
|
||||
return f"{sanitize_title(title)}__{doc_id}.gsheet.json"
|
||||
lead = (f"{modified_time[:DATE_LEN]}_"
|
||||
if len(modified_time) >= DATE_LEN else "")
|
||||
fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX)
|
||||
label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed)
|
||||
return f"{lead}{label}__{doc_id}{SUFFIX}"
|
||||
|
||||
@@ -15,9 +15,11 @@
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
||||
from mirage.utils.sanitize import sanitize_label
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
|
||||
|
||||
TITLE_MAX_CHARS = 100
|
||||
SUFFIX = ".gslide.json"
|
||||
DATE_LEN = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +42,12 @@ sanitize_title = partial(sanitize_label,
|
||||
def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
|
||||
"""Build a filename from title, doc ID, and modified date.
|
||||
|
||||
The title takes whatever of the 255-byte NAME_MAX the date, the id and
|
||||
the suffix leave, rather than a flat character count: those are the same
|
||||
number only for ASCII, and a 100-character CJK title rendered a name ext4
|
||||
and APFS reject outright. The id never gives, so the name keeps
|
||||
addressing the document -- same rule as gcal's event filenames.
|
||||
|
||||
Args:
|
||||
title (str): raw document title.
|
||||
doc_id (str): Google Slides presentation ID.
|
||||
@@ -48,7 +56,8 @@ def make_filename(title: str, doc_id: str, modified_time: str = "") -> str:
|
||||
Returns:
|
||||
str: filename in format "YYYY-MM-DD_Sanitized_Title__docid.json".
|
||||
"""
|
||||
date_prefix = modified_time[:10] if len(modified_time) >= 10 else ""
|
||||
if date_prefix:
|
||||
return f"{date_prefix}_{sanitize_title(title)}__{doc_id}.gslide.json"
|
||||
return f"{sanitize_title(title)}__{doc_id}.gslide.json"
|
||||
lead = (f"{modified_time[:DATE_LEN]}_"
|
||||
if len(modified_time) >= DATE_LEN else "")
|
||||
fixed = byte_len(lead) + len("__") + byte_len(doc_id) + len(SUFFIX)
|
||||
label = sanitize_title(title, max_bytes=NAME_MAX_BYTES - fixed)
|
||||
return f"{lead}{label}__{doc_id}{SUFFIX}"
|
||||
|
||||
@@ -21,6 +21,19 @@ MAX_LEN = 100
|
||||
# characters is the same number only for ASCII: a 100-character CJK title is
|
||||
# 300 bytes.
|
||||
NAME_MAX_BYTES = 255
|
||||
ELLIPSIS = "..."
|
||||
|
||||
|
||||
def byte_len(text: str) -> int:
|
||||
"""Measure a string the way the filesystem does.
|
||||
|
||||
Args:
|
||||
text (str): the string to measure.
|
||||
|
||||
Returns:
|
||||
int: the length of ``text`` in UTF-8 bytes.
|
||||
"""
|
||||
return len(text.encode("utf-8"))
|
||||
|
||||
|
||||
def truncate_bytes(text: str, budget: int) -> str:
|
||||
@@ -88,7 +101,11 @@ def path_safe_name(name: str) -> str:
|
||||
return name.replace("/", "∕")
|
||||
|
||||
|
||||
def sanitize_label(text: str, *, fallback: str, max_len: int) -> str:
|
||||
def sanitize_label(text: str,
|
||||
*,
|
||||
fallback: str,
|
||||
max_len: int,
|
||||
max_bytes: int = NAME_MAX_BYTES) -> str:
|
||||
"""Sanitize an API-supplied label for use inside a filename.
|
||||
|
||||
The shared body behind every backend's title/subject sanitizer:
|
||||
@@ -100,11 +117,24 @@ def sanitize_label(text: str, *, fallback: str, max_len: int) -> str:
|
||||
Unlike ``sanitize_name`` this ellipsizes rather than hard-cutting, so
|
||||
a truncated name reads as truncated.
|
||||
|
||||
Two budgets apply, and both have to: ``max_len`` is the readable
|
||||
length a backend wants, while ``max_bytes`` is what the filesystem
|
||||
will actually accept. They are the same number only for ASCII, so a
|
||||
100-character CJK title passed a 100-character budget untouched and
|
||||
rendered a 300-byte filename, which ext4 and APFS reject with
|
||||
ENAMETOOLONG. Pass the bytes the *rest* of the filename does not
|
||||
already use -- see ``make_filename`` in the gdocs/gsheets/gslides
|
||||
entries and ``make_event_filename`` in gcal, which is where the
|
||||
fixed overhead is known.
|
||||
|
||||
Args:
|
||||
text (str): raw label from the API.
|
||||
fallback (str): what an empty or whitespace-only label becomes.
|
||||
max_len (int): budget in characters; a longer label keeps its
|
||||
first ``max_len - 3`` characters plus an ellipsis.
|
||||
max_bytes (int): budget in UTF-8 bytes for the label alone. A
|
||||
budget too small to hold even the ellipsis yields a bare
|
||||
truncation rather than three dots and nothing.
|
||||
|
||||
Returns:
|
||||
str: the sanitized label.
|
||||
@@ -114,5 +144,12 @@ def sanitize_label(text: str, *, fallback: str, max_len: int) -> str:
|
||||
cleaned = UNSAFE_CHARS.sub("_", text).replace(" ", "_")
|
||||
cleaned = MULTI_UNDERSCORE.sub("_", cleaned).strip("_")
|
||||
if len(cleaned) > max_len:
|
||||
cleaned = cleaned[:max_len - 3] + "..."
|
||||
cleaned = cleaned[:max_len - len(ELLIPSIS)] + ELLIPSIS
|
||||
if byte_len(cleaned) > max_bytes:
|
||||
head = truncate_bytes(cleaned, max(max_bytes - len(ELLIPSIS), 0))
|
||||
trimmed = head.rstrip("_.")
|
||||
if trimmed:
|
||||
cleaned = trimmed + ELLIPSIS
|
||||
else:
|
||||
cleaned = truncate_bytes(cleaned, max_bytes)
|
||||
return cleaned
|
||||
|
||||
@@ -311,7 +311,6 @@ module = [
|
||||
"mirage.accessor._hf",
|
||||
"mirage.accessor.gdocs",
|
||||
"mirage.accessor.gdrive",
|
||||
"mirage.accessor.github",
|
||||
"mirage.accessor.gmail",
|
||||
"mirage.accessor.gsheets",
|
||||
"mirage.accessor.gslides",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.core.email.readdir import _msg_filename
|
||||
from mirage.core.email.search import _build_vfs_path
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
|
||||
|
||||
CJK_SUBJECT = "会議の記録" * 40
|
||||
|
||||
|
||||
def test_a_hit_names_the_file_readdir_created():
|
||||
# Composed here from a bare `_sanitize`, a hit pointed at a path that
|
||||
# does not exist as soon as the subject was long enough to be trimmed:
|
||||
# readdir budgets the subject against the uid and the suffix, and this
|
||||
# did not, so the two names differed.
|
||||
msg = {
|
||||
"subject": CJK_SUBJECT,
|
||||
"uid": "7",
|
||||
"date": "Mon, 5 Jan 2026 10:00:00 +0000"
|
||||
}
|
||||
path = _build_vfs_path("/mail", "INBOX", msg)
|
||||
assert path.endswith("/" + _msg_filename(CJK_SUBJECT, "7"))
|
||||
|
||||
|
||||
def test_a_hits_filename_fits_name_max():
|
||||
msg = {
|
||||
"subject": CJK_SUBJECT,
|
||||
"uid": "7",
|
||||
"date": "Mon, 5 Jan 2026 10:00:00 +0000"
|
||||
}
|
||||
name = _build_vfs_path("/mail", "INBOX", msg).rsplit("/", 1)[-1]
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert "\ufffd" not in name
|
||||
@@ -12,12 +12,15 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.repo import fetch_default_branch, parse_repo
|
||||
from mirage.core.github.repo import (ensure_ref, fetch_default_branch,
|
||||
parse_repo)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -64,3 +67,35 @@ def test_parse_repo_drops_the_optional_host():
|
||||
def test_parse_repo_refuses_a_spec_that_is_not_the_format(spec):
|
||||
with pytest.raises(ValueError, match="OWNER/REPO"):
|
||||
parse_repo(spec)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.repo.github_get")
|
||||
async def test_ensure_ref_resolves_the_default_branch_when_none_was_named(
|
||||
mock_get, config):
|
||||
mock_get.return_value = {"default_branch": "master"}
|
||||
accessor = GitHubAccessor(config, "acme", "proj")
|
||||
assert await ensure_ref(accessor) == "master"
|
||||
# Settled on the accessor, so the next reader neither refetches nor
|
||||
# disagrees with the ref this one read.
|
||||
assert accessor.ref == "master"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.repo.github_get")
|
||||
async def test_ensure_ref_keeps_a_pinned_ref_without_a_request(
|
||||
mock_get, config):
|
||||
accessor = GitHubAccessor(config, "acme", "proj", "release-2")
|
||||
assert await ensure_ref(accessor) == "release-2"
|
||||
mock_get.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.repo.github_get")
|
||||
async def test_ensure_ref_resolves_once_for_concurrent_readers(
|
||||
mock_get, config):
|
||||
mock_get.return_value = {"default_branch": "trunk"}
|
||||
accessor = GitHubAccessor(config, "acme", "proj")
|
||||
refs = await asyncio.gather(*(ensure_ref(accessor) for _ in range(4)))
|
||||
assert refs == ["trunk"] * 4
|
||||
assert mock_get.await_count == 1
|
||||
|
||||
@@ -21,8 +21,8 @@ from mirage.accessor.github import GitHubAccessor
|
||||
from mirage.cache.index import NULL_INDEX
|
||||
from mirage.cache.index.ram import RAMIndexCacheStore
|
||||
from mirage.core.github.config import GitHubConfig
|
||||
from mirage.core.github.tree import (ensure_live_index, fetch_dir_tree,
|
||||
fetch_tree, index_rows)
|
||||
from mirage.core.github.tree import (ensure_live_index, ensure_tree,
|
||||
fetch_dir_tree, fetch_tree, index_rows)
|
||||
from mirage.core.github.tree_entry import TreeEntry
|
||||
|
||||
|
||||
@@ -251,3 +251,35 @@ def test_index_rows_root_mount_keeps_bare_paths():
|
||||
def test_index_rows_gives_an_empty_repo_a_root_row():
|
||||
_entries, children = index_rows({}, "/gh")
|
||||
assert children == {"/gh": []}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.repo.github_get")
|
||||
@patch("mirage.core.github.tree.github_get")
|
||||
async def test_an_unpinned_mount_reads_the_repos_default_branch(
|
||||
mock_tree_get, mock_repo_get, config):
|
||||
"""An unresolved ref must be settled before the tree is fetched.
|
||||
|
||||
``accessor.ref`` is None until something resolves it, so reading it
|
||||
straight sends `ref=None` to the one request the whole mount is built
|
||||
on. This pins the resolution, not the config default -- the mount that
|
||||
supplies the default lives a layer up, in
|
||||
tests/resource/github/test_lazy_hydration.py.
|
||||
"""
|
||||
mock_repo_get.return_value = {"default_branch": "master"}
|
||||
mock_tree_get.return_value = {"truncated": False, "tree": []}
|
||||
accessor = GitHubAccessor(config, "acme", "proj")
|
||||
await ensure_tree(accessor)
|
||||
assert mock_tree_get.await_args.kwargs["ref"] == "master"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mirage.core.github.repo.github_get")
|
||||
@patch("mirage.core.github.tree.github_get")
|
||||
async def test_a_pinned_mount_reads_its_ref_and_never_asks_for_the_branch(
|
||||
mock_tree_get, mock_repo_get, config):
|
||||
mock_tree_get.return_value = {"truncated": False, "tree": []}
|
||||
accessor = GitHubAccessor(config, "acme", "proj", "release-2")
|
||||
await ensure_tree(accessor)
|
||||
assert mock_tree_get.await_args.kwargs["ref"] == "release-2"
|
||||
mock_repo_get.assert_not_awaited()
|
||||
|
||||
@@ -52,7 +52,8 @@ def test_long_ascii_title_is_trimmed_to_name_max():
|
||||
|
||||
def test_long_cjk_title_is_trimmed_by_bytes_not_characters():
|
||||
# 3 bytes per character: a character-counted budget would overflow
|
||||
# NAME_MAX, which is the bug gdocs' sanitize_title still has.
|
||||
# NAME_MAX. gdocs/gsheets/gslides had exactly that bug until
|
||||
# sanitize_label grew a byte budget.
|
||||
name = make_event_filename(EVENT_ID, "0900-1030", "会" * 200)
|
||||
raw = name.encode()
|
||||
assert len(raw) <= NAME_MAX_BYTES
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
from mirage.resource.gdocs.doc_entry import (DocEntry, make_filename,
|
||||
sanitize_title)
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
|
||||
|
||||
TITLE_MAX = 100
|
||||
|
||||
@@ -81,3 +82,34 @@ def test_make_filename_duplicate_titles_different_dates():
|
||||
assert f1 != f2
|
||||
assert f1 == "2026-04-01_My_Doc__abc123.gdoc.json"
|
||||
assert f2 == "2026-03-15_My_Doc__def456.gdoc.json"
|
||||
|
||||
|
||||
# A real Google file id is 44 characters, so this is the fixed overhead a
|
||||
# title actually has to fit inside.
|
||||
DOC_ID = "1" * 44
|
||||
|
||||
|
||||
def test_make_filename_fits_name_max_for_a_cjk_title():
|
||||
# 100 characters of CJK is 300 bytes, which the character budget passed
|
||||
# untouched: with the date, the id and the suffix the name came to 367
|
||||
# bytes and ext4/APFS reject it with ENAMETOOLONG.
|
||||
name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z")
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert name.startswith("2026-08-20_")
|
||||
assert name.endswith(f"__{DOC_ID}.gdoc.json")
|
||||
# The cut lands on a character boundary, never mid-sequence.
|
||||
assert "\ufffd" not in name
|
||||
|
||||
|
||||
def test_make_filename_leaves_an_ascii_title_on_the_char_budget():
|
||||
name = make_filename("a" * 400, DOC_ID, "")
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert name == f"{'a' * 97}...__{DOC_ID}.gdoc.json"
|
||||
|
||||
|
||||
def test_make_filename_keeps_the_id_when_it_leaves_no_room():
|
||||
# The title is what gives, never the id: a trimmed id would stop
|
||||
# addressing the document. Same rule as gcal's event filenames.
|
||||
long_id = "v" * (NAME_MAX_BYTES - 4)
|
||||
name = make_filename("Some Title", long_id, "")
|
||||
assert f"__{long_id}.gdoc.json" in name
|
||||
|
||||
@@ -34,7 +34,10 @@ def test_github_config_owner_repo_ref_default():
|
||||
cfg = GitHubConfig(token="ghp_abc123")
|
||||
assert cfg.owner is None
|
||||
assert cfg.repo is None
|
||||
assert cfg.ref == "main"
|
||||
# Not "main": that guess 404s the tree fetch on every repository whose
|
||||
# default branch is something else. None means "resolve the default
|
||||
# branch", which `ensure_ref` does on the first read.
|
||||
assert cfg.ref is None
|
||||
|
||||
|
||||
def test_github_config_accepts_owner_repo_ref():
|
||||
|
||||
@@ -85,3 +85,37 @@ async def test_a_tree_passed_to_the_constructor_counts_as_hydrated(tree_calls):
|
||||
resource = GitHubResource(CONFIG, "o", "r", "main", tree=dict(TREE))
|
||||
await ensure_tree(resource.accessor)
|
||||
assert tree_calls == []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def default_branch(monkeypatch):
|
||||
|
||||
async def _fetch(config, owner, repo):
|
||||
return "master"
|
||||
|
||||
monkeypatch.setattr("mirage.core.github.repo.fetch_default_branch", _fetch)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_mount_naming_no_ref_reads_the_repos_default_branch(
|
||||
tree_calls, default_branch):
|
||||
# The config used to default `ref` to the literal string "main", so a
|
||||
# repository whose default branch is anything else 404d on the one
|
||||
# request the whole mount is built on and the mount read as empty.
|
||||
resource = GitHubResource(
|
||||
GitHubConfig(token="ghp_test", owner="o", repo="r"))
|
||||
assert resource.accessor.ref is None
|
||||
await ensure_tree(resource.accessor)
|
||||
assert tree_calls == [("o", "r", "master")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unpinned_mount_is_on_the_default_branch_before_any_fetch(
|
||||
tree_calls):
|
||||
# Naming no ref *means* following the default branch, so the two agree
|
||||
# whatever it turns out to be -- no request, and not the "not known
|
||||
# yet" None a pinned mount answers.
|
||||
resource = GitHubResource(
|
||||
GitHubConfig(token="ghp_test", owner="o", repo="r"))
|
||||
assert resource.is_default_branch is True
|
||||
assert GitHubResource(CONFIG, "o", "r", "dev").is_default_branch is None
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.resource.gsheets.sheet_entry import (SheetEntry, make_filename,
|
||||
sanitize_title)
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
|
||||
|
||||
# A real Google file id is 44 characters, so this is the fixed overhead a
|
||||
# title actually has to fit inside.
|
||||
DOC_ID = "1" * 44
|
||||
|
||||
|
||||
def test_sheet_entry_creation():
|
||||
entry = SheetEntry(
|
||||
id="abc123",
|
||||
name="My Spreadsheet",
|
||||
modified_time="2026-04-01T12:00:00.000Z",
|
||||
created_time="2026-03-01T12:00:00.000Z",
|
||||
owner="user@gmail.com",
|
||||
owned_by_me=True,
|
||||
can_edit=True,
|
||||
filename="My_Spreadsheet__abc123.gsheet.json",
|
||||
)
|
||||
assert entry.id == "abc123"
|
||||
assert entry.owned_by_me is True
|
||||
assert entry.can_edit is True
|
||||
|
||||
|
||||
def test_sanitize_title_basic():
|
||||
assert sanitize_title("Hello World") == "Hello_World"
|
||||
assert sanitize_title("My/Doc: A\\Test") == "My_Doc_A_Test"
|
||||
assert sanitize_title("") == "Untitled"
|
||||
|
||||
|
||||
def test_make_filename_with_and_without_a_date():
|
||||
assert make_filename("My Spreadsheet", "abc123",
|
||||
"2026-03-15T10:00:00Z") == \
|
||||
"2026-03-15_My_Spreadsheet__abc123.gsheet.json"
|
||||
assert make_filename("My Spreadsheet",
|
||||
"abc123") == "My_Spreadsheet__abc123.gsheet.json"
|
||||
|
||||
|
||||
def test_make_filename_fits_name_max_for_a_cjk_title():
|
||||
# 100 characters of CJK is 300 bytes, which the character budget passed
|
||||
# untouched: with the date, the id and the suffix the name came to 367
|
||||
# bytes and ext4/APFS reject it with ENAMETOOLONG.
|
||||
name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z")
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert name.startswith("2026-08-20_")
|
||||
assert name.endswith(f"__{DOC_ID}.gsheet.json")
|
||||
# The cut lands on a character boundary, never mid-sequence.
|
||||
assert "\ufffd" not in name
|
||||
|
||||
|
||||
def test_make_filename_leaves_an_ascii_title_on_the_char_budget():
|
||||
name = make_filename("a" * 400, DOC_ID, "")
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert name == f"{'a' * 97}...__{DOC_ID}.gsheet.json"
|
||||
|
||||
|
||||
def test_make_filename_keeps_the_id_when_it_leaves_no_room():
|
||||
# The title is what gives, never the id: a trimmed id would stop
|
||||
# addressing the spreadsheet. Same rule as gcal's event filenames.
|
||||
long_id = "v" * (NAME_MAX_BYTES - 4)
|
||||
name = make_filename("Some Title", long_id, "")
|
||||
assert f"__{long_id}.gsheet.json" in name
|
||||
@@ -0,0 +1,77 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.resource.gslides.slide_entry import (SlideEntry, make_filename,
|
||||
sanitize_title)
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len
|
||||
|
||||
# A real Google file id is 44 characters, so this is the fixed overhead a
|
||||
# title actually has to fit inside.
|
||||
DOC_ID = "1" * 44
|
||||
|
||||
|
||||
def test_slide_entry_creation():
|
||||
entry = SlideEntry(
|
||||
id="abc123",
|
||||
name="My Presentation",
|
||||
modified_time="2026-04-01T12:00:00.000Z",
|
||||
created_time="2026-03-01T12:00:00.000Z",
|
||||
owner="user@gmail.com",
|
||||
owned_by_me=True,
|
||||
can_edit=True,
|
||||
filename="My_Presentation__abc123.gslide.json",
|
||||
)
|
||||
assert entry.id == "abc123"
|
||||
assert entry.owned_by_me is True
|
||||
assert entry.can_edit is True
|
||||
|
||||
|
||||
def test_sanitize_title_basic():
|
||||
assert sanitize_title("Hello World") == "Hello_World"
|
||||
assert sanitize_title("My/Doc: A\\Test") == "My_Doc_A_Test"
|
||||
assert sanitize_title("") == "Untitled"
|
||||
|
||||
|
||||
def test_make_filename_with_and_without_a_date():
|
||||
assert make_filename("My Presentation", "abc123",
|
||||
"2026-03-15T10:00:00Z") == \
|
||||
"2026-03-15_My_Presentation__abc123.gslide.json"
|
||||
assert make_filename("My Presentation",
|
||||
"abc123") == "My_Presentation__abc123.gslide.json"
|
||||
|
||||
|
||||
def test_make_filename_fits_name_max_for_a_cjk_title():
|
||||
# 100 characters of CJK is 300 bytes, which the character budget passed
|
||||
# untouched: with the date, the id and the suffix the name came to 367
|
||||
# bytes and ext4/APFS reject it with ENAMETOOLONG.
|
||||
name = make_filename("会議の記録" * 40, DOC_ID, "2026-08-20T12:00:00Z")
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert name.startswith("2026-08-20_")
|
||||
assert name.endswith(f"__{DOC_ID}.gslide.json")
|
||||
# The cut lands on a character boundary, never mid-sequence.
|
||||
assert "\ufffd" not in name
|
||||
|
||||
|
||||
def test_make_filename_leaves_an_ascii_title_on_the_char_budget():
|
||||
name = make_filename("a" * 400, DOC_ID, "")
|
||||
assert byte_len(name) <= NAME_MAX_BYTES
|
||||
assert name == f"{'a' * 97}...__{DOC_ID}.gslide.json"
|
||||
|
||||
|
||||
def test_make_filename_keeps_the_id_when_it_leaves_no_room():
|
||||
# The title is what gives, never the id: a trimmed id would stop
|
||||
# addressing the presentation. Same rule as gcal's event filenames.
|
||||
long_id = "v" * (NAME_MAX_BYTES - 4)
|
||||
name = make_filename("Some Title", long_id, "")
|
||||
assert f"__{long_id}.gslide.json" in name
|
||||
@@ -61,7 +61,7 @@ UNMIRRORED_DIRS = {
|
||||
# would count 816 today. What the ratchet buys is narrower than it looks:
|
||||
# a module whose name appears nowhere in the suite cannot be added
|
||||
# silently.
|
||||
MIRROR_BASELINE = 186
|
||||
MIRROR_BASELINE = 184
|
||||
|
||||
|
||||
def _test_dirs() -> list[pathlib.Path]:
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.utils.sanitize import sanitize_label
|
||||
from mirage.utils.sanitize import NAME_MAX_BYTES, byte_len, sanitize_label
|
||||
|
||||
|
||||
def test_sanitize_label_replaces_unsafe_and_spaces():
|
||||
@@ -60,8 +60,38 @@ def test_sanitize_label_budget_counts_code_points():
|
||||
|
||||
|
||||
def test_sanitize_label_ellipsizes_on_code_point_boundary():
|
||||
# A byte budget wide enough to stay out of the way, so this pins the
|
||||
# character budget alone.
|
||||
label = "\U00010400" * 120
|
||||
result = sanitize_label(label, fallback="X", max_len=100)
|
||||
result = sanitize_label(label, fallback="X", max_len=100, max_bytes=10_000)
|
||||
assert len(result) == 100
|
||||
assert result.endswith("...")
|
||||
assert "\ufffd" not in result
|
||||
|
||||
|
||||
def test_sanitize_label_honors_the_byte_ceiling_within_the_char_budget():
|
||||
# 100 astral code points is 400 bytes, so a name the character budget
|
||||
# accepts is one ext4 and APFS reject with ENAMETOOLONG. The default
|
||||
# budget is NAME_MAX, and the cut still lands on a code-point boundary.
|
||||
label = "\U00010400" * 120
|
||||
result = sanitize_label(label, fallback="X", max_len=100)
|
||||
assert len(result) < 100
|
||||
assert byte_len(result) <= NAME_MAX_BYTES
|
||||
assert result.endswith("...")
|
||||
assert "\ufffd" not in result
|
||||
|
||||
|
||||
def test_sanitize_label_byte_budget_is_the_callers_remaining_room():
|
||||
# What the gdocs/gmail filenames pass: NAME_MAX minus the id, the
|
||||
# separators and the suffix.
|
||||
result = sanitize_label("会" * 200, fallback="X", max_len=100, max_bytes=60)
|
||||
assert byte_len(result) <= 60
|
||||
assert result.endswith("...")
|
||||
assert "\ufffd" not in result
|
||||
|
||||
|
||||
def test_sanitize_label_drops_the_ellipsis_when_it_cannot_fit():
|
||||
# Three dots and nothing is not a name; a budget this small yields
|
||||
# whatever of the label actually fits.
|
||||
assert sanitize_label("abcdef", fallback="X", max_len=100,
|
||||
max_bytes=2) == "ab"
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
TS_SRC = [
|
||||
REPO / "typescript/packages/core/src",
|
||||
REPO / "typescript/packages/node/src",
|
||||
REPO / "typescript/packages/browser/src",
|
||||
]
|
||||
# `normalizeFields(input, {rename: {...}})` and the shared `const RENAME`
|
||||
# maps the S3-alias families feed it.
|
||||
RENAME_PROP_RE = re.compile(r"rename:\s*\{([^{}]*)\}", re.S)
|
||||
RENAME_CONST_RE = re.compile(r"^const \w*RENAME\w*[^=\n]*=\s*\{([^{}]*)\}",
|
||||
re.M | re.S)
|
||||
PAIR_RE = re.compile(r"([A-Za-z0-9_]+)\s*:\s*'([A-Za-z0-9_]+)'")
|
||||
SNAKE_RE = re.compile(r"_([a-z0-9])")
|
||||
|
||||
|
||||
def snake_to_camel(snake: str) -> str:
|
||||
"""Reimplement `snakeToCamel` from utils/normalize.ts.
|
||||
|
||||
Args:
|
||||
snake: The python-side field name.
|
||||
|
||||
Returns:
|
||||
The camelCase spelling `normalizeFields` produces by default.
|
||||
"""
|
||||
return SNAKE_RE.sub(lambda m: m.group(1).upper(), snake)
|
||||
|
||||
|
||||
def redundant_pairs(text: str) -> list[tuple[str, str]]:
|
||||
"""Find rename entries that only restate the default mapping.
|
||||
|
||||
Args:
|
||||
text: One TypeScript source file.
|
||||
|
||||
Returns:
|
||||
The `(source, target)` pairs whose target is exactly what
|
||||
`snakeToCamel` would have produced anyway.
|
||||
"""
|
||||
found: list[tuple[str, str]] = []
|
||||
for match in RENAME_PROP_RE.finditer(text):
|
||||
found.extend(PAIR_RE.findall(match.group(1)))
|
||||
for match in RENAME_CONST_RE.finditer(text):
|
||||
found.extend(PAIR_RE.findall(match.group(1)))
|
||||
return [(k, v) for k, v in found if snake_to_camel(k) == v]
|
||||
|
||||
|
||||
def source_files() -> list[Path]:
|
||||
"""Every TypeScript source file that could hold a rename map.
|
||||
|
||||
Returns:
|
||||
The files to scan, build output and node_modules excluded.
|
||||
"""
|
||||
out: list[Path] = []
|
||||
for root in TS_SRC:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
out.extend(p for p in root.rglob("*.ts")
|
||||
if "node_modules" not in p.parts and "dist" not in p.parts)
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Fail when a rename map restates what `snakeToCamel` already does.
|
||||
|
||||
`normalizeFields` falls back to `snakeToCamel(key)` for every key no
|
||||
rename names, so `api_key: 'apiKey'` is not configuration, it is a
|
||||
second place for the same fact to be wrong -- and it hides the entries
|
||||
that *are* load-bearing (`endpoint_url: 'endpoint'`, `timeout:
|
||||
'timeoutMs'`, `aws_profile: 'profile'`) in a wall of noise.
|
||||
|
||||
Returns:
|
||||
0 when every surviving rename entry is a real override.
|
||||
"""
|
||||
offenders: list[tuple[Path, list[tuple[str, str]]]] = []
|
||||
for path in source_files():
|
||||
if path.name == "normalize.test.ts":
|
||||
continue
|
||||
pairs = redundant_pairs(path.read_text())
|
||||
if pairs:
|
||||
offenders.append((path, pairs))
|
||||
if not offenders:
|
||||
return 0
|
||||
total = sum(len(p) for _, p in offenders)
|
||||
print(f"{total} rename entries only restate snakeToCamel:\n")
|
||||
for path, pairs in offenders:
|
||||
print(f" {path.relative_to(REPO)}")
|
||||
for key, value in pairs:
|
||||
print(f" {key}: '{value}'")
|
||||
print("\nDelete them: normalizeFields already maps these by default.")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -31,6 +31,7 @@ export { OPFSAccessor } from './accessor/opfs.ts'
|
||||
export { OPFS_COMMANDS } from './commands/builtin/opfs/index.ts'
|
||||
export { S3Resource, S3_BROWSER_PROMPT, type S3ResourceState } from './resource/s3/s3.ts'
|
||||
export {
|
||||
normalizeS3Config,
|
||||
redactConfig as redactS3Config,
|
||||
type S3BrowserOperation,
|
||||
type S3BrowserPresignedUrlProvider,
|
||||
@@ -165,12 +166,14 @@ export {
|
||||
} from './resource/qingstor/config.ts'
|
||||
export { SlackResource, type SlackResourceState } from './resource/slack/slack.ts'
|
||||
export {
|
||||
normalizeSlackConfig,
|
||||
redactSlackConfig,
|
||||
type SlackConfig,
|
||||
type SlackConfigRedacted,
|
||||
} from './resource/slack/config.ts'
|
||||
export { DiscordResource, type DiscordResourceState } from './resource/discord/discord.ts'
|
||||
export {
|
||||
normalizeDiscordConfig,
|
||||
redactDiscordConfig,
|
||||
type DiscordConfig,
|
||||
type DiscordConfigRedacted,
|
||||
@@ -181,6 +184,7 @@ export { MongoDBResource, type MongoDBResourceOptions } from './resource/mongodb
|
||||
export { HttpMongoDriver, type HttpMongoDriverOptions } from './resource/mongodb/http_driver.ts'
|
||||
export { TrelloResource, type TrelloResourceState } from './resource/trello/trello.ts'
|
||||
export {
|
||||
normalizeTrelloConfig,
|
||||
redactTrelloConfig,
|
||||
type TrelloConfig,
|
||||
type TrelloConfigRedacted,
|
||||
@@ -193,12 +197,14 @@ export {
|
||||
export type { LinearConfig, LinearConfigRedacted } from '@struktoai/mirage-core/core/linear/config'
|
||||
export { NotionResource, type NotionResourceState } from './resource/notion/notion.ts'
|
||||
export {
|
||||
normalizeNotionConfig,
|
||||
redactNotionConfig,
|
||||
type NotionConfig,
|
||||
type NotionConfigRedacted,
|
||||
} from './resource/notion/config.ts'
|
||||
export { LangfuseResource, type LangfuseResourceState } from './resource/langfuse/langfuse.ts'
|
||||
export {
|
||||
normalizeLangfuseConfig,
|
||||
redactLangfuseConfig,
|
||||
type LangfuseConfig,
|
||||
type LangfuseConfigRedacted,
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<AliyunConfig, AliyunConfigRedacted>({
|
||||
export const resolvedAliyunEndpoint = alias.resolvedEndpoint
|
||||
export const aliyunToS3Config = alias.toS3Config
|
||||
export const redactAliyunConfig = alias.redact
|
||||
export const normalizeAliyunConfig = alias.normalize
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<BackblazeConfig, BackblazeConfigRedacted>({
|
||||
export const resolvedBackblazeEndpoint = alias.resolvedEndpoint
|
||||
export const backblazeToS3Config = alias.toS3Config
|
||||
export const redactBackblazeConfig = alias.redact
|
||||
export const normalizeBackblazeConfig = alias.normalize
|
||||
|
||||
@@ -62,12 +62,6 @@ export function redactBoxConfig(config: BoxConfig): BoxConfigRedacted {
|
||||
export function normalizeBoxConfig(input: Record<string, unknown>): BoxConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
root_folder_id: 'rootFolderId',
|
||||
content_search: 'contentSearch',
|
||||
client_id: 'clientId',
|
||||
client_secret: 'clientSecret',
|
||||
refresh_token: 'refreshToken',
|
||||
access_token: 'accessToken',
|
||||
developer_token: 'accessToken',
|
||||
},
|
||||
}) as unknown as BoxConfig
|
||||
|
||||
@@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias<CephConfig, CephConfigRedacted>({
|
||||
|
||||
export const cephToS3Config = alias.toS3Config
|
||||
export const redactCephConfig = alias.redact
|
||||
export const normalizeCephConfig = alias.normalize
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<DigitalOceanConfig, DigitalOceanConfigRedacted>
|
||||
export const resolvedDigitalOceanEndpoint = alias.resolvedEndpoint
|
||||
export const digitalOceanToS3Config = alias.toS3Config
|
||||
export const redactDigitalOceanConfig = alias.redact
|
||||
export const normalizeDigitalOceanConfig = alias.normalize
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
|
||||
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
|
||||
type HeaderProvider = () => Promise<Record<string, string>> | Record<string, string>
|
||||
|
||||
@@ -31,3 +32,14 @@ export type DiscordConfigRedacted = RedactedConfig<DiscordConfig, 'getHeaders'>
|
||||
export function redactDiscordConfig(config: DiscordConfig): DiscordConfigRedacted {
|
||||
return redactConfigWithSchema(DiscordConfigSchema, config) as unknown as DiscordConfigRedacted
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a python-style config blob to this one's camelCase.
|
||||
*
|
||||
* No rename map: every field's camelCase spelling is what `snakeToCamel`
|
||||
* already produces, and restating those only creates a second place to be
|
||||
* wrong. Mirrors node's `normalizeDiscordConfig`.
|
||||
*/
|
||||
export function normalizeDiscordConfig(input: Record<string, unknown>): DiscordConfig {
|
||||
return normalizeFields(input) as unknown as DiscordConfig
|
||||
}
|
||||
|
||||
@@ -47,13 +47,5 @@ export function redactDropboxConfig(config: DropboxConfig): DropboxConfigRedacte
|
||||
}
|
||||
|
||||
export function normalizeDropboxConfig(input: Record<string, unknown>): DropboxConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
client_id: 'clientId',
|
||||
client_secret: 'clientSecret',
|
||||
refresh_token: 'refreshToken',
|
||||
root_path: 'rootPath',
|
||||
content_search: 'contentSearch',
|
||||
},
|
||||
}) as unknown as DropboxConfig
|
||||
return normalizeFields(input) as unknown as DropboxConfig
|
||||
}
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<GCSConfig, GCSConfigRedacted, string>({
|
||||
|
||||
export const gcsToS3Config = alias.toS3Config
|
||||
export const redactGcsConfig = alias.redact
|
||||
export const normalizeGCSConfig = alias.normalize
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
import { redactConfigWithSchema, secretStr, z } from '@struktoai/mirage-core/resource/secrets'
|
||||
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
|
||||
const LangfuseConfigSchema = z.object({
|
||||
publicKey: z.string(),
|
||||
@@ -31,3 +32,14 @@ export type LangfuseConfigRedacted = RedactedConfig<LangfuseConfig, 'secretKey'>
|
||||
export function redactLangfuseConfig(config: LangfuseConfig): LangfuseConfigRedacted {
|
||||
return redactConfigWithSchema(LangfuseConfigSchema, config) as unknown as LangfuseConfigRedacted
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a python-style config blob to this one's camelCase.
|
||||
*
|
||||
* No rename map: every field's camelCase spelling is what `snakeToCamel`
|
||||
* already produces, and restating those only creates a second place to be
|
||||
* wrong. Mirrors node's `normalizeLangfuseConfig`.
|
||||
*/
|
||||
export function normalizeLangfuseConfig(input: Record<string, unknown>): LangfuseConfig {
|
||||
return normalizeFields(input) as unknown as LangfuseConfig
|
||||
}
|
||||
|
||||
@@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias<MinIOConfig, MinIOConfigRedacted>({
|
||||
|
||||
export const minioToS3Config = alias.toS3Config
|
||||
export const redactMinIOConfig = alias.redact
|
||||
export const normalizeMinIOConfig = alias.normalize
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js'
|
||||
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
|
||||
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
|
||||
const NotionConfigSchema = z.object({
|
||||
authProvider: secretSchema(
|
||||
@@ -30,3 +31,14 @@ export type NotionConfigRedacted = RedactedConfig<NotionConfig, 'authProvider'>
|
||||
export function redactNotionConfig(config: NotionConfig): NotionConfigRedacted {
|
||||
return redactConfigWithSchema(NotionConfigSchema, config) as unknown as NotionConfigRedacted
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a python-style config blob to this one's camelCase.
|
||||
*
|
||||
* No rename map: every field's camelCase spelling is what `snakeToCamel`
|
||||
* already produces, and restating those only creates a second place to be
|
||||
* wrong. Mirrors node's `normalizeNotionConfig`.
|
||||
*/
|
||||
export function normalizeNotionConfig(input: Record<string, unknown>): NotionConfig {
|
||||
return normalizeFields(input) as unknown as NotionConfig
|
||||
}
|
||||
|
||||
@@ -36,3 +36,4 @@ const alias = makeBrowserS3Alias<OCIConfig, OCIConfigRedacted>({
|
||||
export const resolvedOciEndpoint = alias.resolvedEndpoint
|
||||
export const ociToS3Config = alias.toS3Config
|
||||
export const redactOciConfig = alias.redact
|
||||
export const normalizeOCIConfig = alias.normalize
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<QingStorConfig, QingStorConfigRedacted>({
|
||||
export const resolvedQingStorEndpoint = alias.resolvedEndpoint
|
||||
export const qingStorToS3Config = alias.toS3Config
|
||||
export const redactQingStorConfig = alias.redact
|
||||
export const normalizeQingStorConfig = alias.normalize
|
||||
|
||||
@@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias<R2Config, R2ConfigRedacted>({
|
||||
export const resolvedR2Endpoint = alias.resolvedEndpoint
|
||||
export const r2ToS3Config = alias.toS3Config
|
||||
export const redactR2Config = alias.redact
|
||||
export const normalizeR2Config = alias.normalize
|
||||
|
||||
@@ -36,6 +36,50 @@ describe('browser resource registry', () => {
|
||||
}
|
||||
})
|
||||
|
||||
// Every entry used to hand-roll `normalizeFields` with a rename map that
|
||||
// mostly restated what `snakeToCamel` already does, then cast the result
|
||||
// through a config interface written a second time in the registry. The
|
||||
// casts hid a mismatch: nothing checked that the shape the resource wants
|
||||
// is the shape the entry produces.
|
||||
it('normalizes snake_case config for every hand-wired backend', async () => {
|
||||
const provider = (): Promise<string> => Promise.resolve('https://example.com/signed')
|
||||
const cases: [string, Record<string, unknown>, Record<string, unknown>][] = [
|
||||
[
|
||||
'trello',
|
||||
{ api_key: 'k', api_token: 't', workspace_id: 'w', board_ids: ['b'], base_url: 'u' },
|
||||
{ workspaceId: 'w', boardIds: ['b'], baseUrl: 'u' },
|
||||
],
|
||||
[
|
||||
'langfuse',
|
||||
{ public_key: 'p', secret_key: 's', default_trace_limit: 5, default_search_limit: 6 },
|
||||
{ publicKey: 'p', defaultTraceLimit: 5, defaultSearchLimit: 6 },
|
||||
],
|
||||
['slack', { proxy_url: 'http://x' }, { proxyUrl: 'http://x' }],
|
||||
['discord', { proxy_url: 'http://x' }, { proxyUrl: 'http://x' }],
|
||||
[
|
||||
's3',
|
||||
{ bucket: 'b', presignedUrlProvider: provider, endpoint_url: 'http://e', key_prefix: 'p/' },
|
||||
{ bucket: 'b', endpoint: 'http://e', keyPrefix: 'p/' },
|
||||
],
|
||||
[
|
||||
'minio',
|
||||
{ bucket: 'b', presignedUrlProvider: provider, endpoint_url: 'http://e' },
|
||||
{ bucket: 'b', endpoint: 'http://e' },
|
||||
],
|
||||
]
|
||||
for (const [name, input, expected] of cases) {
|
||||
const state = (await (await buildResource(name, input)).getState()) as {
|
||||
config: Record<string, unknown>
|
||||
}
|
||||
expect(state.config, name).toMatchObject(expected)
|
||||
// `endpoint_url` is the one rename that is not mechanical; a leftover
|
||||
// snake_case key means the entry skipped normalization entirely.
|
||||
for (const key of Object.keys(state.config)) {
|
||||
expect(key, `${name}.${key}`).not.toContain('_')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('lists known resources sorted', () => {
|
||||
const names = knownResources()
|
||||
expect(names).toContain('ram')
|
||||
|
||||
@@ -18,7 +18,6 @@ import type { DifyConfig } from '@struktoai/mirage-core/resource/dify/config'
|
||||
import type { QdrantConfig } from '@struktoai/mirage-core/resource/qdrant/config'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
import { compareCodePoints } from '@struktoai/mirage-core/utils/sort'
|
||||
import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js'
|
||||
|
||||
/**
|
||||
* Construct a resource by registry name in the browser runtime.
|
||||
@@ -35,78 +34,6 @@ import type { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.
|
||||
*/
|
||||
export type ResourceFactory = (config: Record<string, unknown>) => Promise<Resource>
|
||||
|
||||
interface S3BrowserCtorConfig {
|
||||
bucket: string
|
||||
presignedUrlProvider: (
|
||||
path: string,
|
||||
op: 'GET' | 'PUT' | 'HEAD' | 'DELETE' | 'LIST' | 'COPY',
|
||||
opts?: {
|
||||
contentType?: string
|
||||
ttlSec?: number
|
||||
listPrefix?: string
|
||||
listDelimiter?: string
|
||||
listContinuationToken?: string
|
||||
copySource?: string
|
||||
},
|
||||
) => Promise<string>
|
||||
}
|
||||
|
||||
type GCSBrowserCtorConfig = S3BrowserCtorConfig & { region?: string; endpoint?: string }
|
||||
type R2BrowserCtorConfig = S3BrowserCtorConfig & {
|
||||
accountId?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}
|
||||
type OCIBrowserCtorConfig = S3BrowserCtorConfig & {
|
||||
namespace?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}
|
||||
type SupabaseBrowserCtorConfig = S3BrowserCtorConfig & {
|
||||
projectRef?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}
|
||||
type S3AliasBrowserCtorConfig = S3BrowserCtorConfig & {
|
||||
region?: string
|
||||
endpoint?: string
|
||||
}
|
||||
interface SlackBrowserCtorConfig {
|
||||
proxyUrl: string
|
||||
getHeaders?: () => Promise<Record<string, string>> | Record<string, string>
|
||||
}
|
||||
interface DiscordBrowserCtorConfig {
|
||||
proxyUrl: string
|
||||
getHeaders?: () => Promise<Record<string, string>> | Record<string, string>
|
||||
}
|
||||
interface NotionBrowserCtorConfig {
|
||||
authProvider: OAuthClientProvider
|
||||
serverUrl?: string
|
||||
}
|
||||
interface TrelloBrowserCtorConfig {
|
||||
apiKey: string
|
||||
apiToken: string
|
||||
workspaceId?: string
|
||||
boardIds?: readonly string[]
|
||||
baseUrl?: string
|
||||
}
|
||||
interface LangfuseBrowserCtorConfig {
|
||||
publicKey: string
|
||||
secretKey: string
|
||||
host?: string
|
||||
defaultTraceLimit?: number
|
||||
defaultSearchLimit?: number
|
||||
defaultFromTimestamp?: string
|
||||
}
|
||||
interface PostgresBrowserCtorConfig {
|
||||
dsn: string
|
||||
schemas?: readonly string[]
|
||||
defaultRowLimit?: number
|
||||
maxReadRows?: number
|
||||
maxReadBytes?: number
|
||||
defaultSearchLimit?: number
|
||||
}
|
||||
|
||||
const REGISTRY: Record<string, ResourceFactory> = {
|
||||
ram: async (_config) => {
|
||||
const { RAMResource } = await import('@struktoai/mirage-core/resource/ram/ram')
|
||||
@@ -119,135 +46,93 @@ const REGISTRY: Record<string, ResourceFactory> = {
|
||||
},
|
||||
s3: async (config) => {
|
||||
const { S3Resource } = await import('./s3/s3.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new S3Resource(norm as unknown as S3BrowserCtorConfig)
|
||||
const { normalizeS3Config } = await import('./s3/config.ts')
|
||||
return new S3Resource(normalizeS3Config(config))
|
||||
},
|
||||
gcs: async (config) => {
|
||||
const { GCSResource } = await import('./gcs/gcs.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new GCSResource(norm as unknown as GCSBrowserCtorConfig)
|
||||
const { normalizeGCSConfig } = await import('./gcs/config.ts')
|
||||
return new GCSResource(normalizeGCSConfig(config))
|
||||
},
|
||||
r2: async (config) => {
|
||||
const { R2Resource } = await import('./r2/r2.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { account_id: 'accountId', endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new R2Resource(norm as unknown as R2BrowserCtorConfig)
|
||||
const { normalizeR2Config } = await import('./r2/config.ts')
|
||||
return new R2Resource(normalizeR2Config(config))
|
||||
},
|
||||
oci: async (config) => {
|
||||
const { OCIResource } = await import('./oci/oci.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new OCIResource(norm as unknown as OCIBrowserCtorConfig)
|
||||
const { normalizeOCIConfig } = await import('./oci/config.ts')
|
||||
return new OCIResource(normalizeOCIConfig(config))
|
||||
},
|
||||
supabase: async (config) => {
|
||||
const { SupabaseResource } = await import('./supabase/supabase.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { project_ref: 'projectRef', endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new SupabaseResource(norm as unknown as SupabaseBrowserCtorConfig)
|
||||
const { normalizeSupabaseConfig } = await import('./supabase/config.ts')
|
||||
return new SupabaseResource(normalizeSupabaseConfig(config))
|
||||
},
|
||||
minio: async (config) => {
|
||||
const { MinIOResource } = await import('./minio/minio.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new MinIOResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeMinIOConfig } = await import('./minio/config.ts')
|
||||
return new MinIOResource(normalizeMinIOConfig(config))
|
||||
},
|
||||
ceph: async (config) => {
|
||||
const { CephResource } = await import('./ceph/ceph.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new CephResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeCephConfig } = await import('./ceph/config.ts')
|
||||
return new CephResource(normalizeCephConfig(config))
|
||||
},
|
||||
seaweedfs: async (config) => {
|
||||
const { SeaweedFSResource } = await import('./seaweedfs/seaweedfs.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new SeaweedFSResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeSeaweedFSConfig } = await import('./seaweedfs/config.ts')
|
||||
return new SeaweedFSResource(normalizeSeaweedFSConfig(config))
|
||||
},
|
||||
wasabi: async (config) => {
|
||||
const { WasabiResource } = await import('./wasabi/wasabi.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new WasabiResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeWasabiConfig } = await import('./wasabi/config.ts')
|
||||
return new WasabiResource(normalizeWasabiConfig(config))
|
||||
},
|
||||
backblaze: async (config) => {
|
||||
const { BackblazeResource } = await import('./backblaze/backblaze.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new BackblazeResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeBackblazeConfig } = await import('./backblaze/config.ts')
|
||||
return new BackblazeResource(normalizeBackblazeConfig(config))
|
||||
},
|
||||
digitalocean: async (config) => {
|
||||
const { DigitalOceanResource } = await import('./digitalocean/digitalocean.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new DigitalOceanResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeDigitalOceanConfig } = await import('./digitalocean/config.ts')
|
||||
return new DigitalOceanResource(normalizeDigitalOceanConfig(config))
|
||||
},
|
||||
tencent: async (config) => {
|
||||
const { TencentResource } = await import('./tencent/tencent.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new TencentResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeTencentConfig } = await import('./tencent/config.ts')
|
||||
return new TencentResource(normalizeTencentConfig(config))
|
||||
},
|
||||
aliyun: async (config) => {
|
||||
const { AliyunResource } = await import('./aliyun/aliyun.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new AliyunResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeAliyunConfig } = await import('./aliyun/config.ts')
|
||||
return new AliyunResource(normalizeAliyunConfig(config))
|
||||
},
|
||||
scaleway: async (config) => {
|
||||
const { ScalewayResource } = await import('./scaleway/scaleway.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new ScalewayResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeScalewayConfig } = await import('./scaleway/config.ts')
|
||||
return new ScalewayResource(normalizeScalewayConfig(config))
|
||||
},
|
||||
qingstor: async (config) => {
|
||||
const { QingStorResource } = await import('./qingstor/qingstor.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { endpoint_url: 'endpoint' },
|
||||
})
|
||||
return new QingStorResource(norm as unknown as S3AliasBrowserCtorConfig)
|
||||
const { normalizeQingStorConfig } = await import('./qingstor/config.ts')
|
||||
return new QingStorResource(normalizeQingStorConfig(config))
|
||||
},
|
||||
slack: async (config) => {
|
||||
const { SlackResource } = await import('./slack/slack.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { proxy_url: 'proxyUrl', get_headers: 'getHeaders' },
|
||||
})
|
||||
return new SlackResource(norm as unknown as SlackBrowserCtorConfig)
|
||||
const { normalizeSlackConfig } = await import('./slack/config.ts')
|
||||
return new SlackResource(normalizeSlackConfig(config))
|
||||
},
|
||||
discord: async (config) => {
|
||||
const { DiscordResource } = await import('./discord/discord.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { proxy_url: 'proxyUrl', get_headers: 'getHeaders' },
|
||||
})
|
||||
return new DiscordResource(norm as unknown as DiscordBrowserCtorConfig)
|
||||
const { normalizeDiscordConfig } = await import('./discord/config.ts')
|
||||
return new DiscordResource(normalizeDiscordConfig(config))
|
||||
},
|
||||
trello: async (config) => {
|
||||
const { TrelloResource } = await import('./trello/trello.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: {
|
||||
api_key: 'apiKey',
|
||||
api_token: 'apiToken',
|
||||
workspace_id: 'workspaceId',
|
||||
board_ids: 'boardIds',
|
||||
base_url: 'baseUrl',
|
||||
},
|
||||
})
|
||||
return new TrelloResource(norm as unknown as TrelloBrowserCtorConfig)
|
||||
const { normalizeTrelloConfig } = await import('./trello/config.ts')
|
||||
return new TrelloResource(normalizeTrelloConfig(config))
|
||||
},
|
||||
linear: async (config) => {
|
||||
const { LinearResource } = await import('./linear/linear.ts')
|
||||
@@ -256,15 +141,9 @@ const REGISTRY: Record<string, ResourceFactory> = {
|
||||
},
|
||||
postgres: async (config) => {
|
||||
const { PostgresResource } = await import('./postgres/postgres.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: {
|
||||
default_row_limit: 'defaultRowLimit',
|
||||
max_read_rows: 'maxReadRows',
|
||||
max_read_bytes: 'maxReadBytes',
|
||||
default_search_limit: 'defaultSearchLimit',
|
||||
},
|
||||
})
|
||||
return new PostgresResource(norm as unknown as PostgresBrowserCtorConfig)
|
||||
const { normalizePostgresConfig } =
|
||||
await import('@struktoai/mirage-core/resource/postgres/config')
|
||||
return new PostgresResource(normalizePostgresConfig(config))
|
||||
},
|
||||
mongodb: async (config) => {
|
||||
const { MongoDBResource } = await import('./mongodb/mongodb.ts')
|
||||
@@ -294,23 +173,13 @@ const REGISTRY: Record<string, ResourceFactory> = {
|
||||
},
|
||||
notion: async (config) => {
|
||||
const { NotionResource } = await import('./notion/notion.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: { auth_provider: 'authProvider', server_url: 'serverUrl' },
|
||||
})
|
||||
return new NotionResource(norm as unknown as NotionBrowserCtorConfig)
|
||||
const { normalizeNotionConfig } = await import('./notion/config.ts')
|
||||
return new NotionResource(normalizeNotionConfig(config))
|
||||
},
|
||||
langfuse: async (config) => {
|
||||
const { LangfuseResource } = await import('./langfuse/langfuse.ts')
|
||||
const norm = normalizeFields(config, {
|
||||
rename: {
|
||||
public_key: 'publicKey',
|
||||
secret_key: 'secretKey',
|
||||
default_trace_limit: 'defaultTraceLimit',
|
||||
default_search_limit: 'defaultSearchLimit',
|
||||
default_from_timestamp: 'defaultFromTimestamp',
|
||||
},
|
||||
})
|
||||
return new LangfuseResource(norm as unknown as LangfuseBrowserCtorConfig)
|
||||
const { normalizeLangfuseConfig } = await import('./langfuse/config.ts')
|
||||
return new LangfuseResource(normalizeLangfuseConfig(config))
|
||||
},
|
||||
github: async (config) => {
|
||||
const { GitHubResource } = await import('./github/github.ts')
|
||||
|
||||
@@ -12,6 +12,20 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import type { S3Config } from '@struktoai/mirage-core/resource/s3/config'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
|
||||
// The one field whose camelCase spelling is not what `snakeToCamel` would
|
||||
// produce, so it is the whole rename map for the browser's S3 family. Every
|
||||
// other field (`key_prefix`, `default_content_type`, ...) round-trips through
|
||||
// the default, and restating those only creates a second place to be wrong.
|
||||
export const S3_BROWSER_RENAME: Record<string, string> = { endpoint_url: 'endpoint' }
|
||||
|
||||
/** Translate a python-style S3 config blob to the browser's camelCase one. */
|
||||
export function normalizeS3Config(input: Record<string, unknown>): S3Config {
|
||||
return normalizeFields(input, { rename: S3_BROWSER_RENAME }) as unknown as S3Config
|
||||
}
|
||||
|
||||
export { redactConfig } from '@struktoai/mirage-core/resource/s3/config'
|
||||
export type {
|
||||
S3BrowserOperation,
|
||||
|
||||
@@ -17,6 +17,8 @@ import type { RegisteredCommand } from '@struktoai/mirage-core/commands/config'
|
||||
import type { RegisteredOp } from '@struktoai/mirage-core/ops/registry'
|
||||
import { remapCommandsResource, remapOpsResource } from '@struktoai/mirage-core/resource/s3/remap'
|
||||
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
import { S3_BROWSER_RENAME } from './s3/config.ts'
|
||||
import type { S3BrowserPresignedUrlProvider, S3Config, S3ConfigRedacted } from './s3/config.ts'
|
||||
|
||||
/**
|
||||
@@ -92,6 +94,7 @@ export interface BrowserAlias<C, R, E extends string | undefined> {
|
||||
resolvedEndpoint: (config: C) => string | E
|
||||
toS3Config: (config: C) => S3Config
|
||||
redact: (config: C) => R
|
||||
normalize: (input: Record<string, unknown>) => C
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -137,6 +140,7 @@ export function makeBrowserS3Alias<
|
||||
}
|
||||
},
|
||||
redact: (config) => redactConfigWithSchema(schema, config) as unknown as R,
|
||||
normalize: (input) => normalizeFields(input, { rename: S3_BROWSER_RENAME }) as unknown as C,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<ScalewayConfig, ScalewayConfigRedacted>({
|
||||
export const resolvedScalewayEndpoint = alias.resolvedEndpoint
|
||||
export const scalewayToS3Config = alias.toS3Config
|
||||
export const redactScalewayConfig = alias.redact
|
||||
export const normalizeScalewayConfig = alias.normalize
|
||||
|
||||
@@ -27,3 +27,4 @@ const alias = makeBrowserS3Alias<SeaweedFSConfig, SeaweedFSConfigRedacted>({
|
||||
|
||||
export const seaweedfsToS3Config = alias.toS3Config
|
||||
export const redactSeaweedFSConfig = alias.redact
|
||||
export const normalizeSeaweedFSConfig = alias.normalize
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
import { redactConfigWithSchema, secretSchema, z } from '@struktoai/mirage-core/resource/secrets'
|
||||
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
|
||||
type HeaderProvider = () => Promise<Record<string, string>> | Record<string, string>
|
||||
|
||||
@@ -31,3 +32,14 @@ export type SlackConfigRedacted = RedactedConfig<SlackConfig, 'getHeaders'>
|
||||
export function redactSlackConfig(config: SlackConfig): SlackConfigRedacted {
|
||||
return redactConfigWithSchema(SlackConfigSchema, config) as unknown as SlackConfigRedacted
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a python-style config blob to this one's camelCase.
|
||||
*
|
||||
* No rename map: every field's camelCase spelling is what `snakeToCamel`
|
||||
* already produces, and restating those only creates a second place to be
|
||||
* wrong. Mirrors node's `normalizeSlackConfig`.
|
||||
*/
|
||||
export function normalizeSlackConfig(input: Record<string, unknown>): SlackConfig {
|
||||
return normalizeFields(input) as unknown as SlackConfig
|
||||
}
|
||||
|
||||
@@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias<SupabaseConfig, SupabaseConfigRedacted>({
|
||||
export const resolvedSupabaseEndpoint = alias.resolvedEndpoint
|
||||
export const supabaseToS3Config = alias.toS3Config
|
||||
export const redactSupabaseConfig = alias.redact
|
||||
export const normalizeSupabaseConfig = alias.normalize
|
||||
|
||||
@@ -29,3 +29,4 @@ const alias = makeBrowserS3Alias<TencentConfig, TencentConfigRedacted>({
|
||||
export const resolvedTencentEndpoint = alias.resolvedEndpoint
|
||||
export const tencentToS3Config = alias.toS3Config
|
||||
export const redactTencentConfig = alias.redact
|
||||
export const normalizeTencentConfig = alias.normalize
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
import { redactConfigWithSchema, secretStr, z } from '@struktoai/mirage-core/resource/secrets'
|
||||
import type { ConfigOf, RedactedConfig } from '@struktoai/mirage-core/resource/secrets'
|
||||
import { normalizeFields } from '@struktoai/mirage-core/utils/normalize'
|
||||
|
||||
const TrelloConfigSchema = z.object({
|
||||
apiKey: secretStr(),
|
||||
@@ -30,3 +31,14 @@ export type TrelloConfigRedacted = RedactedConfig<TrelloConfig, 'apiKey' | 'apiT
|
||||
export function redactTrelloConfig(config: TrelloConfig): TrelloConfigRedacted {
|
||||
return redactConfigWithSchema(TrelloConfigSchema, config) as unknown as TrelloConfigRedacted
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a python-style config blob to this one's camelCase.
|
||||
*
|
||||
* No rename map: every field's camelCase spelling is what `snakeToCamel`
|
||||
* already produces, and restating those only creates a second place to be
|
||||
* wrong. Mirrors node's `normalizeTrelloConfig`.
|
||||
*/
|
||||
export function normalizeTrelloConfig(input: Record<string, unknown>): TrelloConfig {
|
||||
return normalizeFields(input) as unknown as TrelloConfig
|
||||
}
|
||||
|
||||
@@ -34,3 +34,4 @@ const alias = makeBrowserS3Alias<WasabiConfig, WasabiConfigRedacted, string>({
|
||||
export const resolvedWasabiEndpoint = alias.resolvedEndpoint
|
||||
export const wasabiToS3Config = alias.toS3Config
|
||||
export const redactWasabiConfig = alias.redact
|
||||
export const normalizeWasabiConfig = alias.normalize
|
||||
|
||||
@@ -35,5 +35,5 @@ export function redactDiscordConfig(config: DiscordConfig): DiscordConfigRedacte
|
||||
}
|
||||
|
||||
export function normalizeDiscordConfig(input: Record<string, unknown>): DiscordConfig {
|
||||
return normalizeFields(input, {}) as unknown as DiscordConfig
|
||||
return normalizeFields(input) as unknown as DiscordConfig
|
||||
}
|
||||
|
||||
@@ -34,9 +34,7 @@ export function redactGitHubConfig(config: GitHubConfig): GitHubConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeGitHubConfig(input: Record<string, unknown>): GitHubConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: { base_url: 'baseUrl' },
|
||||
}) as unknown as GitHubConfig
|
||||
return normalizeFields(input) as unknown as GitHubConfig
|
||||
}
|
||||
|
||||
export const GhConfigSchema = z.object({
|
||||
|
||||
@@ -27,16 +27,27 @@ import {
|
||||
messageJsonBytes,
|
||||
} from './messages.ts'
|
||||
import { enoent } from '../../utils/errors.ts'
|
||||
import { sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
import { compareCodePoints } from '../../utils/sort.ts'
|
||||
|
||||
const TITLE_MAX = 80
|
||||
const MSG_SUFFIX = '.gmail.json'
|
||||
|
||||
export const sanitize = (text: string): string =>
|
||||
sanitizeLabel(text, { fallback: 'No_Subject', maxLen: TITLE_MAX })
|
||||
export const sanitize = (text: string, maxBytes?: number): string =>
|
||||
sanitizeLabel(text, {
|
||||
fallback: 'No_Subject',
|
||||
maxLen: TITLE_MAX,
|
||||
...(maxBytes !== undefined ? { maxBytes } : {}),
|
||||
})
|
||||
|
||||
function msgFilename(subject: string, msgId: string): string {
|
||||
return `${sanitize(subject)}__${msgId}.gmail.json`
|
||||
/**
|
||||
* 80 characters is 240 bytes of CJK, which overflows the 255-byte NAME_MAX
|
||||
* once the id and `.gmail.json` are added; the filesystem rejects the name
|
||||
* outright. So the subject takes what the id and the suffix leave.
|
||||
*/
|
||||
export function msgFilename(subject: string, msgId: string): string {
|
||||
const fixed = 2 + byteLength(msgId) + MSG_SUFFIX.length
|
||||
return `${sanitize(subject, NAME_MAX_BYTES - fixed)}__${msgId}${MSG_SUFFIX}`
|
||||
}
|
||||
|
||||
function dateFromInternal(internalDate: string | undefined): string {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
import type { TokenManager } from '../google/client.ts'
|
||||
import { decodeBody, extractHeader, getMessageRaw, listMessages } from './messages.ts'
|
||||
import { sanitize } from './readdir.ts'
|
||||
import { msgFilename } from './readdir.ts'
|
||||
import type { GmailScope } from './scope.ts'
|
||||
|
||||
const EXCERPT_WINDOW = 120
|
||||
@@ -115,7 +115,11 @@ export function formatGrepResults(
|
||||
const label = row.label !== '' ? row.label : (scope.labelName ?? 'INBOX')
|
||||
const date = row.date
|
||||
const mid = row.id
|
||||
const filename = `${sanitize(row.subject || 'No Subject')}__${mid}.gmail.json`
|
||||
// The same builder readdir names the file with, not a second spelling of
|
||||
// it: the subject's budget depends on the id and the suffix, so a hit
|
||||
// composed from a bare `sanitize` pointed at a path that does not exist
|
||||
// once a long subject was trimmed.
|
||||
const filename = msgFilename(row.subject || 'No Subject', mid)
|
||||
const sender = row.sender !== '' ? row.sender : '?'
|
||||
const haystack = `${row.subject}\n${row.bodyText}`
|
||||
let excerpt = pattern !== '' ? extractExcerpt(haystack, pattern) : ''
|
||||
|
||||
@@ -75,15 +75,5 @@ export function redactGoogleConfig(config: GoogleConfig): GoogleConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeGoogleConfig(input: Record<string, unknown>): GoogleConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
client_id: 'clientId',
|
||||
client_secret: 'clientSecret',
|
||||
refresh_token: 'refreshToken',
|
||||
api_base: 'apiBase',
|
||||
folder_id: 'folderId',
|
||||
time_zone: 'timeZone',
|
||||
min_access_role: 'minAccessRole',
|
||||
},
|
||||
}) as unknown as GoogleConfig
|
||||
return normalizeFields(input) as unknown as GoogleConfig
|
||||
}
|
||||
|
||||
@@ -37,11 +37,5 @@ export function redactLinearConfig(config: LinearConfig): LinearConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeLinearConfig(input: Record<string, unknown>): LinearConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
api_key: 'apiKey',
|
||||
team_ids: 'teamIds',
|
||||
base_url: 'baseUrl',
|
||||
},
|
||||
}) as unknown as LinearConfig
|
||||
return normalizeFields(input) as unknown as LinearConfig
|
||||
}
|
||||
|
||||
@@ -35,10 +35,5 @@ export function redactNotionConfig(config: NotionConfig): NotionConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeNotionConfig(input: Record<string, unknown>): NotionConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
api_key: 'apiKey',
|
||||
base_url: 'baseUrl',
|
||||
},
|
||||
}) as unknown as NotionConfig
|
||||
return normalizeFields(input) as unknown as NotionConfig
|
||||
}
|
||||
|
||||
@@ -36,7 +36,5 @@ export function redactSlackConfig(config: SlackConfig): SlackConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeSlackConfig(input: Record<string, unknown>): SlackConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: { search_token: 'searchToken', base_url: 'baseUrl' },
|
||||
}) as unknown as SlackConfig
|
||||
return normalizeFields(input) as unknown as SlackConfig
|
||||
}
|
||||
|
||||
@@ -61,8 +61,6 @@ export function redactDatabricksVolumeConfig(
|
||||
export function normalizeDatabricksVolumeConfig(
|
||||
input: Record<string, unknown>,
|
||||
): DatabricksVolumeConfig {
|
||||
const renamed = normalizeFields(input, {
|
||||
rename: { root_path: 'rootPath' },
|
||||
})
|
||||
const renamed = normalizeFields(input)
|
||||
return DatabricksVolumeConfigSchema.parse(renamed) as DatabricksVolumeConfig
|
||||
}
|
||||
|
||||
@@ -56,7 +56,8 @@ describe('gcal event entry naming', () => {
|
||||
|
||||
it('trims a long CJK title by bytes, not characters', () => {
|
||||
// 3 bytes per character: a character-counted budget would overflow
|
||||
// NAME_MAX, which is the bug gdocs' sanitizeTitle still has.
|
||||
// NAME_MAX. gdocs/gsheets/gslides had exactly that bug until
|
||||
// sanitizeLabel grew a byte budget.
|
||||
const name = makeEventFilename(EVENT_ID, '0900-1030', '会'.repeat(200))
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name).not.toContain('�')
|
||||
|
||||
@@ -16,6 +16,7 @@ import { enoent } from '../../utils/errors.ts'
|
||||
import { makeIdName } from '../../utils/naming.ts'
|
||||
import {
|
||||
NAME_MAX_BYTES,
|
||||
byteLength,
|
||||
sanitizeName,
|
||||
stripTrailingUnderscores,
|
||||
truncateBytes,
|
||||
@@ -31,8 +32,6 @@ const UNTITLED = 'untitled'
|
||||
// there is no title to sanitize and "busy" is the honest rendering.
|
||||
const BUSY = 'busy'
|
||||
|
||||
const UTF8 = new TextEncoder()
|
||||
|
||||
/** Pick the title segment for an event filename. */
|
||||
export function eventTitle(summary: string | null, freeBusy = false): string {
|
||||
if (summary !== null && summary.trim() !== '') return sanitizeName(summary)
|
||||
@@ -48,7 +47,7 @@ export function eventTitle(summary: string | null, freeBusy = false): string {
|
||||
* 255-byte NAME_MAX is left rather than a fixed character count.
|
||||
*/
|
||||
export function makeEventFilename(eventId: string, hhmm: string, title: string): string {
|
||||
const fixed = UTF8.encode(eventId).length + 2 + hhmm.length + 1 + EVENT_SUFFIX.length
|
||||
const fixed = byteLength(eventId) + 2 + hhmm.length + 1 + EVENT_SUFFIX.length
|
||||
const trimmed = stripTrailingUnderscores(truncateBytes(title, NAME_MAX_BYTES - fixed))
|
||||
if (trimmed === '') {
|
||||
// The title is what gives, never the id: trimming the id would make the
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { makeFilename } from './doc_entry.ts'
|
||||
import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts'
|
||||
|
||||
// A real Google file id is 44 characters, so this is the fixed overhead a
|
||||
// title actually has to fit inside.
|
||||
const DOC_ID = '1'.repeat(44)
|
||||
|
||||
describe('gdocs document filenames', () => {
|
||||
it('leads with the date when there is one', () => {
|
||||
expect(makeFilename('My Document', 'abc123', '2026-03-15T10:00:00Z')).toBe(
|
||||
'2026-03-15_My_Document__abc123.gdoc.json',
|
||||
)
|
||||
expect(makeFilename('My Document', 'abc123')).toBe('My_Document__abc123.gdoc.json')
|
||||
})
|
||||
|
||||
it('fits NAME_MAX for a CJK title', () => {
|
||||
// 100 characters of CJK is 300 bytes, which the character budget passed
|
||||
// untouched: with the date, the id and the suffix the name came to 367
|
||||
// bytes and ext4/APFS reject it with ENAMETOOLONG.
|
||||
const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z')
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name.startsWith('2026-08-20_')).toBe(true)
|
||||
expect(name.endsWith(`__${DOC_ID}.gdoc.json`)).toBe(true)
|
||||
// The cut lands on a character boundary, never mid-sequence.
|
||||
expect(name).not.toContain('\uFFFD')
|
||||
})
|
||||
|
||||
it('leaves an ascii title on the character budget', () => {
|
||||
const name = makeFilename('a'.repeat(400), DOC_ID, '')
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gdoc.json`)
|
||||
})
|
||||
|
||||
it('keeps the id when it leaves no room', () => {
|
||||
// The title is what gives, never the id: a trimmed id would stop
|
||||
// addressing the document. Same rule as gcal's event filenames.
|
||||
const longId = 'v'.repeat(NAME_MAX_BYTES - 4)
|
||||
expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gdoc.json`)
|
||||
})
|
||||
})
|
||||
@@ -12,17 +12,26 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
|
||||
const TITLE_MAX_CHARS = 100
|
||||
const SUFFIX = '.gdoc.json'
|
||||
const DATE_LEN = 10
|
||||
|
||||
const sanitizeTitle = (title: string): string =>
|
||||
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS })
|
||||
const sanitizeTitle = (title: string, maxBytes: number): string =>
|
||||
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes })
|
||||
|
||||
/**
|
||||
* Build a filename from title, doc ID, and modified date.
|
||||
*
|
||||
* The title takes whatever of the 255-byte NAME_MAX the date, the id and the
|
||||
* suffix leave, rather than a flat character count: those are the same number
|
||||
* only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS
|
||||
* reject outright. The id never gives, so the name keeps addressing the
|
||||
* document -- same rule as gcal's event filenames.
|
||||
*/
|
||||
export function makeFilename(title: string, docId: string, modifiedTime = ''): string {
|
||||
const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : ''
|
||||
if (datePrefix !== '') {
|
||||
return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gdoc.json`
|
||||
}
|
||||
return `${sanitizeTitle(title)}__${docId}.gdoc.json`
|
||||
const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : ''
|
||||
const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length
|
||||
return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { makeFilename } from './sheet_entry.ts'
|
||||
import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts'
|
||||
|
||||
// A real Google file id is 44 characters, so this is the fixed overhead a
|
||||
// title actually has to fit inside.
|
||||
const DOC_ID = '1'.repeat(44)
|
||||
|
||||
describe('gsheets spreadsheet filenames', () => {
|
||||
it('leads with the date when there is one', () => {
|
||||
expect(makeFilename('My Spreadsheet', 'abc123', '2026-03-15T10:00:00Z')).toBe(
|
||||
'2026-03-15_My_Spreadsheet__abc123.gsheet.json',
|
||||
)
|
||||
expect(makeFilename('My Spreadsheet', 'abc123')).toBe('My_Spreadsheet__abc123.gsheet.json')
|
||||
})
|
||||
|
||||
it('fits NAME_MAX for a CJK title', () => {
|
||||
// 100 characters of CJK is 300 bytes, which the character budget passed
|
||||
// untouched: with the date, the id and the suffix the name came to 367
|
||||
// bytes and ext4/APFS reject it with ENAMETOOLONG.
|
||||
const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z')
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name.startsWith('2026-08-20_')).toBe(true)
|
||||
expect(name.endsWith(`__${DOC_ID}.gsheet.json`)).toBe(true)
|
||||
// The cut lands on a character boundary, never mid-sequence.
|
||||
expect(name).not.toContain('\uFFFD')
|
||||
})
|
||||
|
||||
it('leaves an ascii title on the character budget', () => {
|
||||
const name = makeFilename('a'.repeat(400), DOC_ID, '')
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gsheet.json`)
|
||||
})
|
||||
|
||||
it('keeps the id when it leaves no room', () => {
|
||||
// The title is what gives, never the id: a trimmed id would stop
|
||||
// addressing the spreadsheet. Same rule as gcal's event filenames.
|
||||
const longId = 'v'.repeat(NAME_MAX_BYTES - 4)
|
||||
expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gsheet.json`)
|
||||
})
|
||||
})
|
||||
@@ -12,17 +12,26 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
|
||||
const TITLE_MAX_CHARS = 100
|
||||
const SUFFIX = '.gsheet.json'
|
||||
const DATE_LEN = 10
|
||||
|
||||
const sanitizeTitle = (title: string): string =>
|
||||
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS })
|
||||
const sanitizeTitle = (title: string, maxBytes: number): string =>
|
||||
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes })
|
||||
|
||||
/**
|
||||
* Build a filename from title, doc ID, and modified date.
|
||||
*
|
||||
* The title takes whatever of the 255-byte NAME_MAX the date, the id and the
|
||||
* suffix leave, rather than a flat character count: those are the same number
|
||||
* only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS
|
||||
* reject outright. The id never gives, so the name keeps addressing the
|
||||
* document -- same rule as gcal's event filenames.
|
||||
*/
|
||||
export function makeFilename(title: string, docId: string, modifiedTime = ''): string {
|
||||
const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : ''
|
||||
if (datePrefix !== '') {
|
||||
return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gsheet.json`
|
||||
}
|
||||
return `${sanitizeTitle(title)}__${docId}.gsheet.json`
|
||||
const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : ''
|
||||
const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length
|
||||
return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { makeFilename } from './slide_entry.ts'
|
||||
import { NAME_MAX_BYTES, byteLength } from '../../utils/sanitize.ts'
|
||||
|
||||
// A real Google file id is 44 characters, so this is the fixed overhead a
|
||||
// title actually has to fit inside.
|
||||
const DOC_ID = '1'.repeat(44)
|
||||
|
||||
describe('gslides presentation filenames', () => {
|
||||
it('leads with the date when there is one', () => {
|
||||
expect(makeFilename('My Presentation', 'abc123', '2026-03-15T10:00:00Z')).toBe(
|
||||
'2026-03-15_My_Presentation__abc123.gslide.json',
|
||||
)
|
||||
expect(makeFilename('My Presentation', 'abc123')).toBe('My_Presentation__abc123.gslide.json')
|
||||
})
|
||||
|
||||
it('fits NAME_MAX for a CJK title', () => {
|
||||
// 100 characters of CJK is 300 bytes, which the character budget passed
|
||||
// untouched: with the date, the id and the suffix the name came to 367
|
||||
// bytes and ext4/APFS reject it with ENAMETOOLONG.
|
||||
const name = makeFilename('会議の記録'.repeat(40), DOC_ID, '2026-08-20T12:00:00Z')
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name.startsWith('2026-08-20_')).toBe(true)
|
||||
expect(name.endsWith(`__${DOC_ID}.gslide.json`)).toBe(true)
|
||||
// The cut lands on a character boundary, never mid-sequence.
|
||||
expect(name).not.toContain('\uFFFD')
|
||||
})
|
||||
|
||||
it('leaves an ascii title on the character budget', () => {
|
||||
const name = makeFilename('a'.repeat(400), DOC_ID, '')
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name).toBe(`${'a'.repeat(97)}...__${DOC_ID}.gslide.json`)
|
||||
})
|
||||
|
||||
it('keeps the id when it leaves no room', () => {
|
||||
// The title is what gives, never the id: a trimmed id would stop
|
||||
// addressing the presentation. Same rule as gcal's event filenames.
|
||||
const longId = 'v'.repeat(NAME_MAX_BYTES - 4)
|
||||
expect(makeFilename('Some Title', longId, '')).toContain(`__${longId}.gslide.json`)
|
||||
})
|
||||
})
|
||||
@@ -12,17 +12,26 @@
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '../../utils/sanitize.ts'
|
||||
|
||||
const TITLE_MAX_CHARS = 100
|
||||
const SUFFIX = '.gslide.json'
|
||||
const DATE_LEN = 10
|
||||
|
||||
const sanitizeTitle = (title: string): string =>
|
||||
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS })
|
||||
const sanitizeTitle = (title: string, maxBytes: number): string =>
|
||||
sanitizeLabel(title, { fallback: 'Untitled', maxLen: TITLE_MAX_CHARS, maxBytes })
|
||||
|
||||
/**
|
||||
* Build a filename from title, doc ID, and modified date.
|
||||
*
|
||||
* The title takes whatever of the 255-byte NAME_MAX the date, the id and the
|
||||
* suffix leave, rather than a flat character count: those are the same number
|
||||
* only for ASCII, and a 100-character CJK title rendered a name ext4 and APFS
|
||||
* reject outright. The id never gives, so the name keeps addressing the
|
||||
* document -- same rule as gcal's event filenames.
|
||||
*/
|
||||
export function makeFilename(title: string, docId: string, modifiedTime = ''): string {
|
||||
const datePrefix = modifiedTime.length >= 10 ? modifiedTime.slice(0, 10) : ''
|
||||
if (datePrefix !== '') {
|
||||
return `${datePrefix}_${sanitizeTitle(title)}__${docId}.gslide.json`
|
||||
}
|
||||
return `${sanitizeTitle(title)}__${docId}.gslide.json`
|
||||
const lead = modifiedTime.length >= DATE_LEN ? `${modifiedTime.slice(0, DATE_LEN)}_` : ''
|
||||
const fixed = byteLength(lead) + 2 + byteLength(docId) + SUFFIX.length
|
||||
return `${lead}${sanitizeTitle(title, NAME_MAX_BYTES - fixed)}__${docId}${SUFFIX}`
|
||||
}
|
||||
|
||||
@@ -13,7 +13,13 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { pathSafeName, sanitizeLabel, sanitizeName } from './sanitize.ts'
|
||||
import {
|
||||
NAME_MAX_BYTES,
|
||||
byteLength,
|
||||
pathSafeName,
|
||||
sanitizeLabel,
|
||||
sanitizeName,
|
||||
} from './sanitize.ts'
|
||||
|
||||
describe('sanitizeName', () => {
|
||||
it('returns "unknown" for empty/whitespace input', () => {
|
||||
@@ -98,13 +104,42 @@ describe('sanitizeLabel', () => {
|
||||
})
|
||||
|
||||
it('ellipsizes on a code-point boundary', () => {
|
||||
// A byte budget wide enough to stay out of the way, so this pins the
|
||||
// character budget alone.
|
||||
const label = '\u{10400}'.repeat(120)
|
||||
const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100 })
|
||||
const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100, maxBytes: 10_000 })
|
||||
expect(Array.from(out)).toHaveLength(100)
|
||||
expect(out.endsWith('...')).toBe(true)
|
||||
expect(out).not.toContain('\uFFFD')
|
||||
})
|
||||
|
||||
it('honors the byte ceiling within the character budget', () => {
|
||||
// 100 astral code points is 400 bytes, so a name the character budget
|
||||
// accepts is one ext4 and APFS reject with ENAMETOOLONG. The default
|
||||
// budget is NAME_MAX, and the cut still lands on a code-point boundary.
|
||||
const label = '\u{10400}'.repeat(120)
|
||||
const out = sanitizeLabel(label, { fallback: 'X', maxLen: 100 })
|
||||
expect(Array.from(out).length).toBeLessThan(100)
|
||||
expect(byteLength(out)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(out.endsWith('...')).toBe(true)
|
||||
expect(out).not.toContain('\uFFFD')
|
||||
})
|
||||
|
||||
it('takes the byte budget as the caller remaining room', () => {
|
||||
// What the gdocs/gmail filenames pass: NAME_MAX minus the id, the
|
||||
// separators and the suffix.
|
||||
const out = sanitizeLabel('会'.repeat(200), { fallback: 'X', maxLen: 100, maxBytes: 60 })
|
||||
expect(byteLength(out)).toBeLessThanOrEqual(60)
|
||||
expect(out.endsWith('...')).toBe(true)
|
||||
expect(out).not.toContain('\uFFFD')
|
||||
})
|
||||
|
||||
it('drops the ellipsis when it cannot fit', () => {
|
||||
// Three dots and nothing is not a name; a budget this small yields
|
||||
// whatever of the label actually fits.
|
||||
expect(sanitizeLabel('abcdef', { fallback: 'X', maxLen: 100, maxBytes: 2 })).toBe('ab')
|
||||
})
|
||||
|
||||
it('keeps non-ascii letters, matching python', () => {
|
||||
// The per-backend copies this replaced used `\w`, which is ascii-only in
|
||||
// javascript, so a CJK title became a row of underscores while python --
|
||||
|
||||
@@ -20,10 +20,16 @@ const MAX_LEN = 100
|
||||
// characters is the same number only for ASCII: a 100-character CJK title is
|
||||
// 300 bytes.
|
||||
export const NAME_MAX_BYTES = 255
|
||||
const ELLIPSIS = '...'
|
||||
|
||||
const UTF8 = new TextEncoder()
|
||||
const UTF8_DECODER = new TextDecoder('utf-8')
|
||||
|
||||
/** Measure a string the way the filesystem does: in UTF-8 bytes. */
|
||||
export function byteLength(text: string): number {
|
||||
return UTF8.encode(text).length
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim a string to fit a byte budget without splitting a character.
|
||||
*
|
||||
@@ -51,6 +57,13 @@ export function stripTrailingUnderscores(value: string): string {
|
||||
return value.slice(0, end)
|
||||
}
|
||||
|
||||
/** python's `str.rstrip("_.")`, so a byte cut cannot leave `Foo.....`. */
|
||||
function stripTrailingUnderscoresAndDots(value: string): string {
|
||||
let end = value.length
|
||||
while (end > 0 && (value[end - 1] === '_' || value[end - 1] === '.')) end -= 1
|
||||
return value.slice(0, end)
|
||||
}
|
||||
|
||||
export function stripUnderscores(value: string): string {
|
||||
let start = 0
|
||||
let end = value.length
|
||||
@@ -101,8 +114,20 @@ export function pathSafeName(name: string): string {
|
||||
*
|
||||
* Unlike `sanitizeName` this ellipsizes rather than hard-cutting, so a
|
||||
* truncated name reads as truncated.
|
||||
*
|
||||
* Two budgets apply, and both have to: `maxLen` is the readable length a
|
||||
* backend wants, while `maxBytes` is what the filesystem will actually
|
||||
* accept. They are the same number only for ASCII, so a 100-character CJK
|
||||
* title passed a 100-character budget untouched and rendered a 300-byte
|
||||
* filename, which ext4 and APFS reject with ENAMETOOLONG. Pass the bytes the
|
||||
* *rest* of the filename does not already use -- see `makeFilename` in the
|
||||
* gdocs/gsheets/gslides entries and `makeEventFilename` in gcal, which is
|
||||
* where the fixed overhead is known.
|
||||
*/
|
||||
export function sanitizeLabel(text: string, options: { fallback: string; maxLen: number }): string {
|
||||
export function sanitizeLabel(
|
||||
text: string,
|
||||
options: { fallback: string; maxLen: number; maxBytes?: number },
|
||||
): string {
|
||||
if (text.trim() === '') return options.fallback
|
||||
let cleaned = text.replace(UNSAFE_CHARS, '_').replace(/ /g, '_').replace(MULTI_UNDERSCORE, '_')
|
||||
cleaned = stripUnderscores(cleaned)
|
||||
@@ -112,7 +137,14 @@ export function sanitizeLabel(text: string, options: { fallback: string; maxLen:
|
||||
// encodes as U+FFFD in the filename.
|
||||
const points = Array.from(cleaned)
|
||||
if (points.length > options.maxLen) {
|
||||
cleaned = `${points.slice(0, options.maxLen - 3).join('')}...`
|
||||
cleaned = `${points.slice(0, options.maxLen - ELLIPSIS.length).join('')}${ELLIPSIS}`
|
||||
}
|
||||
const maxBytes = options.maxBytes ?? NAME_MAX_BYTES
|
||||
if (byteLength(cleaned) > maxBytes) {
|
||||
const head = stripTrailingUnderscoresAndDots(
|
||||
truncateBytes(cleaned, Math.max(maxBytes - ELLIPSIS.length, 0)),
|
||||
)
|
||||
cleaned = head !== '' ? `${head}${ELLIPSIS}` : truncateBytes(cleaned, maxBytes)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
@@ -68,18 +68,7 @@ export function buildEmailConfig(input: EmailConfigInput): EmailConfig {
|
||||
}
|
||||
|
||||
export function normalizeEmailConfig(input: Record<string, unknown>): EmailConfig {
|
||||
const norm = normalizeFields(input, {
|
||||
rename: {
|
||||
imap_host: 'imapHost',
|
||||
imap_port: 'imapPort',
|
||||
smtp_host: 'smtpHost',
|
||||
smtp_port: 'smtpPort',
|
||||
use_ssl: 'useSsl',
|
||||
max_messages: 'maxMessages',
|
||||
save_copy: 'saveCopy',
|
||||
sent_folder: 'sentFolder',
|
||||
},
|
||||
})
|
||||
const norm = normalizeFields(input)
|
||||
const built: EmailConfigInput = {
|
||||
imapHost: asString(norm.imapHost),
|
||||
smtpHost: asString(norm.smtpHost),
|
||||
|
||||
@@ -18,6 +18,7 @@ import { PathSpec as PathSpecCtor } from '@struktoai/mirage-core/types'
|
||||
import type { PathSpec } from '@struktoai/mirage-core/types'
|
||||
import { enoent } from '@struktoai/mirage-core/utils/errors'
|
||||
import { mountKey, mountPrefixOf } from '@struktoai/mirage-core/utils/key_prefix'
|
||||
import { NAME_MAX_BYTES, byteLength, sanitizeLabel } from '@struktoai/mirage-core/utils/sanitize'
|
||||
import { compareCodePoints } from '@struktoai/mirage-core/utils/sort'
|
||||
import type { EmailAccessor } from '../../accessor/email.ts'
|
||||
import { fetchHeaders, listMessageUids, type FetchedMessage } from './client.ts'
|
||||
@@ -26,20 +27,26 @@ import { messageJsonBytes } from './render.ts'
|
||||
import type { ParsedAttachment } from './_parse.ts'
|
||||
|
||||
const TITLE_MAX = 80
|
||||
const UNSAFE = /[^\w\s\-.]/g
|
||||
const MULTI_UNDERSCORE = /_+/g
|
||||
const EPOCH_DATE = '1970-01-01'
|
||||
const MSG_SUFFIX = '.email.json'
|
||||
|
||||
export function sanitize(text: string): string {
|
||||
if (text.trim() === '') return 'No_Subject'
|
||||
let cleaned = text.replace(UNSAFE, '_').replace(/ /g, '_')
|
||||
cleaned = cleaned.replace(MULTI_UNDERSCORE, '_').replace(/^_+|_+$/g, '')
|
||||
if (cleaned.length > TITLE_MAX) cleaned = `${cleaned.slice(0, TITLE_MAX - 3)}...`
|
||||
return cleaned
|
||||
}
|
||||
// Routed through the shared sanitizer rather than a local copy of it. The
|
||||
// copy's `\w` was JS's ASCII-only one where python's is unicode, so every
|
||||
// accented or CJK subject came back as a row of underscores here and intact
|
||||
// there; it also measured the budget in UTF-16 units instead of code points.
|
||||
const sanitize = (text: string, maxBytes?: number): string =>
|
||||
sanitizeLabel(text, {
|
||||
fallback: 'No_Subject',
|
||||
maxLen: TITLE_MAX,
|
||||
...(maxBytes !== undefined ? { maxBytes } : {}),
|
||||
})
|
||||
|
||||
function msgFilename(subject: string, uid: string): string {
|
||||
return `${sanitize(subject)}__${uid}.email.json`
|
||||
// 80 characters is 240 bytes of CJK, which overflows the 255-byte NAME_MAX
|
||||
// once the uid and `.email.json` are added, so the subject takes what they
|
||||
// leave rather than a flat character count.
|
||||
export function msgFilename(subject: string, uid: string): string {
|
||||
const fixed = 2 + byteLength(uid) + MSG_SUFFIX.length
|
||||
return `${sanitize(subject, NAME_MAX_BYTES - fixed)}__${uid}${MSG_SUFFIX}`
|
||||
}
|
||||
|
||||
// RFC 5322's obsolete zone names, the set `parsedate_to_datetime` knows.
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import { NAME_MAX_BYTES, byteLength } from '@struktoai/mirage-core/utils/sanitize'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { msgFilename } from './readdir.ts'
|
||||
import { buildVfsPath } from './search.ts'
|
||||
|
||||
const CJK_SUBJECT = '会議の記録'.repeat(40)
|
||||
const MSG = { subject: CJK_SUBJECT, uid: '7', date: 'Mon, 5 Jan 2026 10:00:00 +0000' }
|
||||
|
||||
describe('email search paths', () => {
|
||||
it('names the file readdir created', () => {
|
||||
// Composed here from a bare `sanitize`, a hit pointed at a path that does
|
||||
// not exist as soon as the subject was long enough to be trimmed: readdir
|
||||
// budgets the subject against the uid and the suffix, and this did not,
|
||||
// so the two names differed.
|
||||
const path = buildVfsPath('/mail', 'INBOX', MSG as never)
|
||||
expect(path.endsWith(`/${msgFilename(CJK_SUBJECT, '7')}`)).toBe(true)
|
||||
})
|
||||
|
||||
it('fits NAME_MAX', () => {
|
||||
const name =
|
||||
buildVfsPath('/mail', 'INBOX', MSG as never)
|
||||
.split('/')
|
||||
.pop() ?? ''
|
||||
expect(byteLength(name)).toBeLessThanOrEqual(NAME_MAX_BYTES)
|
||||
expect(name).not.toContain('\uFFFD')
|
||||
})
|
||||
})
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
import type { EmailAccessor } from '../../accessor/email.ts'
|
||||
import { fetchMessage, listMessageUids, type FetchedMessage } from './client.ts'
|
||||
import { dateBucket, sanitize } from './readdir.ts'
|
||||
import { dateBucket, msgFilename } from './readdir.ts'
|
||||
import { messageJsonText } from './render.ts'
|
||||
import type { EmailScope } from './scope.ts'
|
||||
|
||||
@@ -62,10 +62,13 @@ async function searchMessages(
|
||||
return listMessageUids(accessor, folder, criteria, maxResults)
|
||||
}
|
||||
|
||||
function buildVfsPath(prefix: string, folder: string, msg: FetchedMessage): string {
|
||||
export function buildVfsPath(prefix: string, folder: string, msg: FetchedMessage): string {
|
||||
const dateStr = dateBucket(msg)
|
||||
const subject = sanitize(msg.subject !== '' ? msg.subject : 'No Subject')
|
||||
const filename = `${subject}__${msg.uid}.email.json`
|
||||
// The same builder readdir names the file with, not a second spelling of
|
||||
// it: the subject's budget depends on the uid and the suffix, so a hit
|
||||
// composed here from a bare `sanitize` pointed at a path that does not
|
||||
// exist as soon as a long subject was trimmed differently.
|
||||
const filename = msgFilename(msg.subject !== '' ? msg.subject : 'No Subject', msg.uid)
|
||||
return [prefix, folder, dateStr, filename].filter((p) => p !== '').join('/')
|
||||
}
|
||||
|
||||
|
||||
@@ -70,13 +70,6 @@ export function redactBoxConfig(config: BoxConfig): BoxConfigRedacted {
|
||||
export function normalizeBoxConfig(input: Record<string, unknown>): BoxConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
root_folder_id: 'rootFolderId',
|
||||
content_search: 'contentSearch',
|
||||
client_id: 'clientId',
|
||||
client_secret: 'clientSecret',
|
||||
refresh_token: 'refreshToken',
|
||||
enterprise_id: 'enterpriseId',
|
||||
access_token: 'accessToken',
|
||||
developer_token: 'accessToken',
|
||||
},
|
||||
}) as unknown as BoxConfig
|
||||
|
||||
@@ -47,13 +47,5 @@ export function redactDropboxConfig(config: DropboxConfig): DropboxConfigRedacte
|
||||
}
|
||||
|
||||
export function normalizeDropboxConfig(input: Record<string, unknown>): DropboxConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
client_id: 'clientId',
|
||||
client_secret: 'clientSecret',
|
||||
refresh_token: 'refreshToken',
|
||||
root_path: 'rootPath',
|
||||
content_search: 'contentSearch',
|
||||
},
|
||||
}) as unknown as DropboxConfig
|
||||
return normalizeFields(input) as unknown as DropboxConfig
|
||||
}
|
||||
|
||||
@@ -81,13 +81,9 @@ export function redactGcsConfig(config: GCSConfig): GCSConfigRedacted {
|
||||
export function normalizeGcsConfig(input: Record<string, unknown>): GCSConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
access_key_id: 'accessKeyId',
|
||||
secret_access_key: 'secretAccessKey',
|
||||
session_token: 'sessionToken',
|
||||
aws_profile: 'profile',
|
||||
endpoint_url: 'endpoint',
|
||||
path_style: 'forcePathStyle',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
|
||||
@@ -39,12 +39,7 @@ export function redactConfig(config: GridFSConfig): GridFSConfigRedacted {
|
||||
* normalize the key prefix the way the accessor expects.
|
||||
*/
|
||||
export function normalizeGridFSConfig(input: Record<string, unknown>): GridFSConfig {
|
||||
const norm = normalizeFields(input, {
|
||||
rename: {
|
||||
key_prefix: 'keyPrefix',
|
||||
chunk_size_bytes: 'chunkSizeBytes',
|
||||
},
|
||||
}) as unknown as GridFSConfig
|
||||
const norm = normalizeFields(input) as unknown as GridFSConfig
|
||||
const prefix = normalizeKeyPrefix(norm.keyPrefix)
|
||||
if (prefix !== undefined) {
|
||||
norm.keyPrefix = prefix
|
||||
|
||||
@@ -59,7 +59,6 @@ export function redactHfBucketsConfig(config: HfBucketsConfig): HfBucketsConfigR
|
||||
export function normalizeHfBucketsConfig(input: Record<string, unknown>): HfBucketsConfig {
|
||||
const config = normalizeFields(input, {
|
||||
rename: {
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
@@ -102,8 +101,6 @@ export function redactHfRepoConfig(config: HfRepoConfig): HfRepoConfigRedacted {
|
||||
export function normalizeHfRepoConfig(input: Record<string, unknown>): HfRepoConfig {
|
||||
const config = normalizeFields(input, {
|
||||
rename: {
|
||||
repo_id: 'repoId',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
|
||||
@@ -34,12 +34,5 @@ export function redactJaegerConfig(config: JaegerConfig): JaegerConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeJaegerConfig(input: Record<string, unknown>): JaegerConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
default_trace_limit: 'defaultTraceLimit',
|
||||
default_from_timestamp: 'defaultFromTimestamp',
|
||||
default_to_timestamp: 'defaultToTimestamp',
|
||||
request_timeout: 'requestTimeout',
|
||||
},
|
||||
}) as unknown as JaegerConfig
|
||||
return normalizeFields(input) as unknown as JaegerConfig
|
||||
}
|
||||
|
||||
@@ -34,13 +34,5 @@ export function redactLangfuseConfig(config: LangfuseConfig): LangfuseConfigReda
|
||||
}
|
||||
|
||||
export function normalizeLangfuseConfig(input: Record<string, unknown>): LangfuseConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
public_key: 'publicKey',
|
||||
secret_key: 'secretKey',
|
||||
default_trace_limit: 'defaultTraceLimit',
|
||||
default_search_limit: 'defaultSearchLimit',
|
||||
default_from_timestamp: 'defaultFromTimestamp',
|
||||
},
|
||||
}) as unknown as LangfuseConfig
|
||||
return normalizeFields(input) as unknown as LangfuseConfig
|
||||
}
|
||||
|
||||
@@ -83,12 +83,8 @@ export function redactOciConfig(config: OCIConfig): OCIConfigRedacted {
|
||||
export function normalizeOciConfig(input: Record<string, unknown>): OCIConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
access_key_id: 'accessKeyId',
|
||||
secret_access_key: 'secretAccessKey',
|
||||
session_token: 'sessionToken',
|
||||
aws_profile: 'profile',
|
||||
endpoint_url: 'endpoint',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
|
||||
@@ -89,14 +89,9 @@ export function redactR2Config(config: R2Config): R2ConfigRedacted {
|
||||
export function normalizeR2Config(input: Record<string, unknown>): R2Config {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
account_id: 'accountId',
|
||||
access_key_id: 'accessKeyId',
|
||||
secret_access_key: 'secretAccessKey',
|
||||
session_token: 'sessionToken',
|
||||
aws_profile: 'profile',
|
||||
endpoint_url: 'endpoint',
|
||||
path_style: 'forcePathStyle',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
|
||||
@@ -68,7 +68,6 @@ export function normalizeS3Config(input: Record<string, unknown>): S3Config {
|
||||
endpoint_url: 'endpoint',
|
||||
path_style: 'forcePathStyle',
|
||||
timeout: 'timeoutMs',
|
||||
key_prefix: 'keyPrefix',
|
||||
},
|
||||
transform: {
|
||||
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
|
||||
|
||||
@@ -52,16 +52,14 @@ export interface S3AliasConfig {
|
||||
proxy?: string
|
||||
}
|
||||
|
||||
// Every provider accepts the same snake_case spelling of its fields, and
|
||||
// Python states timeouts in seconds where TypeScript uses milliseconds.
|
||||
// Only the entries `snakeToCamel` would get wrong: python spells the profile
|
||||
// `aws_profile`, the endpoint `endpoint_url`, path style `path_style`, and
|
||||
// states timeouts in seconds where TypeScript uses milliseconds. Every other
|
||||
// field maps by default, so it does not belong here.
|
||||
const RENAME: Record<string, string> = {
|
||||
access_key_id: 'accessKeyId',
|
||||
secret_access_key: 'secretAccessKey',
|
||||
session_token: 'sessionToken',
|
||||
aws_profile: 'profile',
|
||||
endpoint_url: 'endpoint',
|
||||
path_style: 'forcePathStyle',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
}
|
||||
|
||||
|
||||
@@ -38,10 +38,5 @@ export function redactSshConfig(config: SSHConfig): SSHConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeSshConfig(input: Record<string, unknown>): SSHConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
identity_file: 'identityFile',
|
||||
known_hosts: 'knownHosts',
|
||||
},
|
||||
}) as unknown as SSHConfig
|
||||
return normalizeFields(input) as unknown as SSHConfig
|
||||
}
|
||||
|
||||
@@ -86,13 +86,8 @@ export function redactSupabaseConfig(config: SupabaseConfig): SupabaseConfigReda
|
||||
export function normalizeSupabaseConfig(input: Record<string, unknown>): SupabaseConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
project_ref: 'projectRef',
|
||||
access_key_id: 'accessKeyId',
|
||||
secret_access_key: 'secretAccessKey',
|
||||
session_token: 'sessionToken',
|
||||
aws_profile: 'profile',
|
||||
endpoint_url: 'endpoint',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
|
||||
@@ -33,13 +33,5 @@ export function redactTrelloConfig(config: TrelloConfig): TrelloConfigRedacted {
|
||||
}
|
||||
|
||||
export function normalizeTrelloConfig(input: Record<string, unknown>): TrelloConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
api_key: 'apiKey',
|
||||
api_token: 'apiToken',
|
||||
workspace_id: 'workspaceId',
|
||||
board_ids: 'boardIds',
|
||||
base_url: 'baseUrl',
|
||||
},
|
||||
}) as unknown as TrelloConfig
|
||||
return normalizeFields(input) as unknown as TrelloConfig
|
||||
}
|
||||
|
||||
@@ -85,13 +85,9 @@ export function redactWasabiConfig(config: WasabiConfig): WasabiConfigRedacted {
|
||||
export function normalizeWasabiConfig(input: Record<string, unknown>): WasabiConfig {
|
||||
return normalizeFields(input, {
|
||||
rename: {
|
||||
access_key_id: 'accessKeyId',
|
||||
secret_access_key: 'secretAccessKey',
|
||||
session_token: 'sessionToken',
|
||||
aws_profile: 'profile',
|
||||
endpoint_url: 'endpoint',
|
||||
path_style: 'forcePathStyle',
|
||||
key_prefix: 'keyPrefix',
|
||||
timeout: 'timeoutMs',
|
||||
},
|
||||
transform: {
|
||||
|
||||
Reference in New Issue
Block a user