github-featured/strukto-ai--mirage
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat(ops): optional byte-range reads across backends, and drop github_ci

Browse Source 
Adds an optional read_range slot to the op table. Backends that can fetch a
window do; the generic read op falls back to read-and-slice for the rest, so
no backend has to implement one. Wired in both languages for box, databricks,
dify, discord, disk, dropbox, gdrive, gridfs, hf, nextcloud, onedrive, opfs,
ram, redis, s3, sharepoint, slack and ssh.

Shared helpers live in utils/ranges (range_header for a raw push-down,
slice_window for rendered content, is_unsatisfiable_range to normalise a 416).
Backends that render their bytes take the window right after building them,
which is why dify and ram are on the native list too: a windowed read is
answered the same way everywhere, whatever is behind the mount.

Fixes a crash on hf and nextcloud: OpenDAL's reader seeks rather than sending
a header, so a window past EOF raised from the seek instead of surfacing a
416. Both now read as empty. Caught by the integ battery, not by unit tests.

Removes the github_ci resource, its commands, ops, docs and examples.

Integ: a shared ranges/read.json across 21 targets, plus per-backend cases for
slack, discord and dify, reached through ws.dispatch since no shell command
asks for a window.
This commit is contained in:
Zecheng Zhang
2026-08-15 02:15:52 -07:00
parent 6554c2d695
commit 88698fa08b
320 changed files with 1635 additions and 7176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/python/github_ci/github_ci.py
-246
View File
@@ -1,246 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import os
from dotenv import load_dotenv
from mirage import MountMode, Workspace
from mirage.resource.github_ci import GitHubCIConfig, GitHubCIResource
from mirage.types import PathSpec
load_dotenv(".env.development")
config = GitHubCIConfig(
token=os.environ["GITHUB_TOKEN"],
owner="strukto-ai",
repo="mirage",
max_runs=300,
)
resource = GitHubCIResource(config=config)
async def main():
ws = Workspace({"/ci": resource}, mode=MountMode.READ)
print("=== not-found errors show the full virtual path ===")
for cmd in ("cat /ci/__nf_missing__.txt", "head /ci/__nf_missing__.txt",
"stat /ci/__nf_missing__.txt"):
result = await ws.execute(cmd)
print(f"$ {cmd}")
print(f" exit={result.exit_code} "
f"{(await result.stderr_str()).strip()}")
# ── discover structure ────────────────────────────
print("=== ls /ci/ (root) ===")
r = await ws.execute("ls /ci/")
print(await r.stdout_str())
# ── list workflows ────────────────────────────────
print("=== ls /ci/workflows/ ===")
r = await ws.execute("ls /ci/workflows/")
print(await r.stdout_str())
workflows = (await r.stdout_str()).strip().splitlines()
if not workflows or not workflows[0]:
print("no workflows found")
return
wf_name = workflows[0].strip()
# ── read workflow metadata ────────────────────────
print(f"=== cat /ci/workflows/{wf_name} ===")
r = await ws.execute(f'cat "/ci/workflows/{wf_name}"')
print((await r.stdout_str())[:500])
# ── list runs ─────────────────────────────────────
print("\n=== ls /ci/runs/ ===")
r = await ws.execute("ls /ci/runs/")
print(await r.stdout_str())
print("\n=== ls -l /ci/runs/ (mtime from updated_at) ===")
long_runs = await ws.execute("ls -l /ci/runs/ | head -n 5")
print(await long_runs.stdout_str())
runs = (await r.stdout_str()).strip().splitlines()
if not runs or not runs[0]:
print("no runs found")
return
run_name = runs[0].strip()
run_path = f"/ci/runs/{run_name}"
# ── list run contents ─────────────────────────────
print(f"=== ls {run_path}/ ===")
r = await ws.execute(f'ls "{run_path}/"')
print(await r.stdout_str())
# ── read run metadata ─────────────────────────────
print(f"=== cat {run_path}/run.json | head -n 20 ===")
r = await ws.execute(f'cat "{run_path}/run.json" | head -n 20')
print(await r.stdout_str())
# ── stat on the run ───────────────────────────────
print(f"=== stat {run_path} ===")
r = await ws.execute(f'stat "{run_path}"')
print(f" {(await r.stdout_str()).strip()}")
# chmod/chown/touch never hit the Actions API: attrs land in the
# workspace namespace (durable, snapshot-captured) and merge into
# dispatch-level stat.
print(f"=== metadata overlay on {run_path} ===")
meta_res = await ws.execute(f'chmod 640 "{run_path}"'
f' && chown 500:dev "{run_path}"'
f' && touch -t 202601021530 "{run_path}"')
print(f" chmod/chown/touch exit={meta_res.exit_code}")
meta_st, _ = await ws.dispatch("stat",
PathSpec.from_str_path(f"{run_path}"))
print(f" dispatch stat: mode={oct(meta_st.mode)[2:]} uid={meta_st.uid} "
f"gid={meta_st.gid} mtime={meta_st.modified}")
# ── list jobs ─────────────────────────────────────
jobs_path = f"{run_path}/jobs"
print(f"\n=== ls {jobs_path}/ ===")
r = await ws.execute(f'ls "{jobs_path}/"')
print(await r.stdout_str())
jobs_out = (await r.stdout_str()).strip().splitlines()
json_jobs = [j.strip() for j in jobs_out if j.strip().endswith(".json")]
log_jobs = [j.strip() for j in jobs_out if j.strip().endswith(".log")]
# ── read a job .json ──────────────────────────────
if json_jobs:
job_name = json_jobs[0]
job_path = f"{jobs_path}/{job_name}"
print(f"=== cat {job_name} | head -n 20 ===")
r = await ws.execute(f'cat "{job_path}" | head -n 20')
print(await r.stdout_str())
print(f"=== stat {job_name} ===")
r = await ws.execute(f'stat "{job_path}"')
print(f" {(await r.stdout_str()).strip()}")
# ── read a job .log ───────────────────────────────
if log_jobs:
log_name = log_jobs[0]
log_path = f"{jobs_path}/{log_name}"
print(f"=== head -n 20 {log_name} ===")
r = await ws.execute(f'head -n 20 "{log_path}"')
print((await r.stdout_str())[:1000])
print(f"\n=== tail -n 10 {log_name} ===")
r = await ws.execute(f'tail -n 10 "{log_path}"')
print((await r.stdout_str())[:500])
print(f"\n=== wc -l {log_name} ===")
r = await ws.execute(f'wc -l "{log_path}"')
print(f" {(await r.stdout_str()).strip()}")
# ── annotations ───────────────────────────────────
print(f"\n=== cat {run_path}/annotations.jsonl ===")
r = await ws.execute(f'cat "{run_path}/annotations.jsonl"')
out = (await r.stdout_str()).strip()
if out:
for line in out.splitlines()[:5]:
print(f" {line[:120]}")
else:
print(" (no annotations)")
# ── artifacts ─────────────────────────────────────
print(f"\n=== ls {run_path}/artifacts/ ===")
r = await ws.execute(f'ls "{run_path}/artifacts/"')
out = (await r.stdout_str()).strip()
if out:
print(out)
else:
print(" (no artifacts)")
# ── tree ──────────────────────────────────────────
print("\n=== tree -L 2 /ci/ ===")
r = await ws.execute("tree -L 2 /ci/")
print(await r.stdout_str())
# ── find (scoped to a single run) ─────────────────
print(f"=== find {run_path}/ -name '*.log' | head -n 10 ===")
r = await ws.execute(f'find "{run_path}/" -name "*.log" | head -n 10')
print(await r.stdout_str())
print(f"=== find {run_path}/ -name '*.json' | head -n 10 ===")
r = await ws.execute(f'find "{run_path}/" -name "*.json" | head -n 10')
print(await r.stdout_str())
# -path matches the display path; -size counts dirs and sizeless
# rendered files as 0 (so +0c drops them, -1k keeps them).
print(f"=== find {run_path}/ -path '*jobs*' | head -n 5 ===")
r = await ws.execute(f'find "{run_path}/" -path "*jobs*" | head -n 5')
print(await r.stdout_str())
print(f"=== find {run_path}/ -maxdepth 1 -size +0c ===")
r = await ws.execute(f'find "{run_path}/" -maxdepth 1 -size +0c')
print(f" exit={r.exit_code} (sizeless entries count as 0,"
" expect no output)")
# ── cd into a run ─────────────────────────────────
print("=== pwd ===")
r = await ws.execute("pwd")
print(f" {(await r.stdout_str()).strip()}")
print(f'\n=== cd "{run_path}" ===')
r = await ws.execute(f'cd "{run_path}"')
print(f" exit={r.exit_code}")
print("\n=== pwd (after cd) ===")
r = await ws.execute("pwd")
print(f" {(await r.stdout_str()).strip()}")
print("\n=== ls (relative, in run dir) ===")
r = await ws.execute("ls")
print(await r.stdout_str())
print("=== cat run.json | head -n 5 (relative) ===")
r = await ws.execute("cat run.json | head -n 5")
print(await r.stdout_str())
# ── cd into jobs ──────────────────────────────────
print('=== cd jobs ===')
r = await ws.execute("cd jobs")
print(f" exit={r.exit_code}")
print("\n=== pwd (after cd jobs) ===")
r = await ws.execute("pwd")
print(f" {(await r.stdout_str()).strip()}")
print("\n=== ls (relative, in jobs dir) ===")
r = await ws.execute("ls")
print(await r.stdout_str())
if log_jobs:
log_name = log_jobs[0]
print(f"=== head -n 5 {log_name} (relative) ===")
r = await ws.execute(f"head -n 5 {log_name}")
print((await r.stdout_str())[:300])
# ── stat on workflows ─────────────────────────────
print("\n=== stat /ci/workflows/ ===")
r = await ws.execute('stat "/ci/workflows/"')
print(f" {(await r.stdout_str()).strip()}")
print("\n=== stat /ci/runs/ ===")
r = await ws.execute('stat "/ci/runs/"')
print(f" {(await r.stdout_str()).strip()}")
if __name__ == "__main__":
asyncio.run(main())
examples/python/github_ci/github_ci_fuse.py
-129
View File
@@ -1,129 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import json
import os
from dotenv import load_dotenv
from mirage import Mount, MountBackend, MountMode, Workspace
from mirage.resource.github_ci import GitHubCIConfig, GitHubCIResource
load_dotenv(".env.development")
config = GitHubCIConfig(
token=os.environ["GITHUB_TOKEN"],
owner="strukto-ai",
repo="mirage",
max_runs=300,
)
resource = GitHubCIResource(config=config)
with Workspace(
{"/ci/": Mount(resource, mode=MountMode.READ,
backend=MountBackend.FUSE)}) as ws:
mp = ws.fuse_mountpoint
print(f"=== FUSE MODE: mounted at {mp} ===\n")
# ── list root ────────────────────────────────
print("--- os.listdir() root ---")
entries = os.listdir(mp)
for e in entries:
print(f" {e}")
# ── list workflows ───────────────────────────
print("\n--- os.listdir() workflows ---")
workflows = os.listdir(f"{mp}/workflows")
for wf in workflows[:10]:
print(f" {wf}")
if workflows:
wf_path = f"{mp}/workflows/{workflows[0]}"
print(f"\n--- open() + read {workflows[0]} ---")
with open(wf_path) as f:
data = json.loads(f.read())
print(f" name: {data.get('name')}")
print(f" state: {data.get('state')}")
# ── list runs ────────────────────────────────
print("\n--- os.listdir() runs ---")
runs = os.listdir(f"{mp}/runs")
for r in runs[:5]:
print(f" {r}")
if len(runs) > 5:
print(f" ... ({len(runs)} total)")
if runs:
run = runs[0]
run_dir = f"{mp}/runs/{run}"
# ── list run contents ────────────────────
print(f"\n--- os.listdir() {run} ---")
contents = os.listdir(run_dir)
for c in contents:
print(f" {c}")
# ── read run.json ────────────────────────
run_json = f"{run_dir}/run.json"
if os.path.exists(run_json):
print("\n--- open() + read run.json ---")
with open(run_json) as f:
data = json.loads(f.read())
print(f" status: {data.get('status')}")
print(f" conclusion: {data.get('conclusion')}")
print(f" event: {data.get('event')}")
# ── list and read jobs ───────────────────
jobs_dir = f"{run_dir}/jobs"
if os.path.isdir(jobs_dir):
print("\n--- os.listdir() jobs ---")
jobs = os.listdir(jobs_dir)
for j in jobs:
print(f" {j}")
log_files = [j for j in jobs if j.endswith(".log")]
if log_files:
log_path = f"{jobs_dir}/{log_files[0]}"
print(
f"\n--- open() + read {log_files[0]} (first 10 lines) ---")
with open(log_path) as f:
for i, line in enumerate(f):
if i >= 10:
print(" ...")
break
print(f" {line.rstrip()[:120]}")
# ── list artifacts ───────────────────────
artifacts_dir = f"{run_dir}/artifacts"
if os.path.isdir(artifacts_dir):
print("\n--- os.listdir() artifacts ---")
artifacts = os.listdir(artifacts_dir)
for a in artifacts:
print(f" {a}")
if not artifacts:
print(" (none)")
# ── interactive ──────────────────────────────
print(f"\n>>> FUSE mounted at: {mp}")
print(">>> Open another terminal and run:")
print(f">>> ls {mp}/")
print(f">>> ls {mp}/runs/")
print(f">>> cat {mp}/runs/<run>/run.json")
print(">>> Press Enter to unmount and exit...")
input()
records = ws.ops.records
total = sum(r.bytes for r in records)
print(f"\nStats: {len(records)} ops, {total} bytes")
examples/python/github_ci/github_ci_vfs.py
-127
View File
@@ -1,127 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import json
import os
import sys
from dotenv import load_dotenv
from mirage import MountMode, Workspace
from mirage.resource.github_ci import GitHubCIConfig, GitHubCIResource
load_dotenv(".env.development")
config = GitHubCIConfig(
token=os.environ["GITHUB_TOKEN"],
owner="strukto-ai",
repo="mirage",
max_runs=300,
)
resource = GitHubCIResource(config=config)
async def main():
with Workspace({"/ci/": resource}, mode=MountMode.READ) as ws:
vos = sys.modules["os"]
print("=== VFS MODE: open() reads from GitHub CI transparently ===\n")
print("--- os.listdir() root ---")
entries = vos.listdir("/ci")
for e in entries:
print(f" {e}")
print("\n--- os.listdir() workflows ---")
workflows = vos.listdir("/ci/workflows")
for wf in workflows[:10]:
print(f" {wf}")
if workflows:
wf_path = f"/ci/workflows/{workflows[0]}"
print(f"\n--- open() + read {wf_path} ---")
with open(wf_path) as f:
data = json.loads(f.read())
print(f" name: {data.get('name')}")
print(f" path: {data.get('path')}")
print(f" state: {data.get('state')}")
print("\n--- os.listdir() runs ---")
runs = vos.listdir("/ci/runs")
for r in runs[:5]:
print(f" {r}")
if len(runs) > 5:
print(f" ... ({len(runs)} total)")
if runs:
run_path = f"/ci/runs/{runs[0]}"
print(f"\n--- os.listdir() {run_path} ---")
contents = vos.listdir(run_path)
for c in contents:
print(f" {c}")
if "run.json" in contents:
print("\n--- open() + read run.json ---")
with open(f"{run_path}/run.json") as f:
data = json.loads(f.read())
print(f" status: {data.get('status')}")
print(f" conclusion: {data.get('conclusion')}")
print(f" event: {data.get('event')}")
print(f" branch: {data.get('head_branch')}")
if "jobs" in contents:
jobs_path = f"{run_path}/jobs"
print("\n--- os.listdir() jobs ---")
jobs = vos.listdir(jobs_path)
for j in jobs[:10]:
print(f" {j}")
json_jobs = [j for j in jobs if j.endswith(".json")]
log_jobs = [j for j in jobs if j.endswith(".log")]
if json_jobs:
print("\n--- open() + read job .json ---")
with open(f"{jobs_path}/{json_jobs[0]}") as f:
data = json.loads(f.read())
print(f" name: {data.get('name')}")
print(f" status: {data.get('status')}")
print(f" conclusion: {data.get('conclusion')}")
steps = data.get("steps", [])
print(f" steps: {len(steps)}")
for s in steps[:3]:
print(f" {s.get('number')}. {s.get('name')}"
f" -> {s.get('conclusion')}")
if log_jobs:
print("\n--- open() + read job .log (first 10 lines) ---")
with open(f"{jobs_path}/{log_jobs[0]}") as f:
for i, line in enumerate(f):
if i >= 10:
print(" ...")
break
print(f" {line.rstrip()[:120]}")
print("\n--- bash history ---")
with open("/.bash_history") as f:
for i, line in enumerate(f):
if i >= 6:
break
print(f" {line.rstrip()[:120]}")
records = ws.ops.records
total = sum(r.bytes for r in records)
print(f"\nStats: {len(records)} ops, {total} bytes transferred")
asyncio.run(main())
examples/python/provision/example.py
-8
View File
@@ -65,13 +65,6 @@ def build(backend: str):
username=os.environ["EMAIL_USERNAME"],
password=os.environ["EMAIL_PASSWORD"],
max_messages=5))
if backend == "github_ci":
from mirage.resource.github_ci import GitHubCIConfig, GitHubCIResource
return GitHubCIResource(
config=GitHubCIConfig(token=os.environ["GITHUB_TOKEN"],
owner="strukto-ai",
repo="mirage",
max_runs=20))
if backend == "gdocs":
from mirage.resource.gdocs import GDocsConfig, GDocsResource
return GDocsResource(config=GDocsConfig(
@@ -94,7 +87,6 @@ BACKENDS = [
"trello",
"langfuse",
"email",
"github_ci",
"gdocs",
"gmail",
]
examples/typescript/github_ci/github_ci.ts
-182
View File
@@ -1,182 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import dotenv from 'dotenv'
import { GitHubCIResource, MountMode, Workspace, type FileStat } from '@struktoai/mirage-node'
const __HERE = fileURLToPath(new URL('.', import.meta.url))
dotenv.config({ path: resolve(__HERE, '../../../.env.development') })
const TOKEN = process.env.GITHUB_TOKEN
if (TOKEN === undefined || TOKEN === '') {
throw new Error('GITHUB_TOKEN env var is required')
}
async function run(ws: Workspace, cmd: string): Promise<{ out: string; err: string; code: number }> {
try {
const r = await ws.execute(cmd)
return { out: r.stdoutText, err: r.stderrText, code: r.exitCode }
} catch (err) {
return { out: '', err: err instanceof Error ? err.message : String(err), code: 1 }
}
}
function printOut(label: string, out: string, err: string, max = 500): void {
console.log(`=== ${label} ===`)
if (out !== '') console.log(out.length > max ? out.slice(0, max) + '...' : out)
if (err !== '') process.stderr.write(` STDERR: ${err.trim().slice(0, 200)}\n`)
}
async function main(): Promise<void> {
const resource = new GitHubCIResource({
token: TOKEN!,
owner: 'strukto-ai',
repo: 'mirage',
maxRuns: 300,
})
const ws = new Workspace({ '/ci': resource }, { mode: MountMode.READ })
try {
const root = await run(ws, 'ls /ci/')
printOut('ls /ci/ (root)', root.out, root.err)
const workflowsResult = await run(ws, 'ls /ci/workflows/')
printOut('ls /ci/workflows/', workflowsResult.out, workflowsResult.err)
const workflows = workflowsResult.out.trim().split('\n').filter((s) => s !== '')
if (workflows.length === 0) {
console.log('no workflows found')
return
}
const wfName = workflows[0]!.trim().split('/').pop() ?? ''
const wfShow = await run(ws, `cat "/ci/workflows/${wfName}"`)
printOut(`cat /ci/workflows/${wfName}`, wfShow.out, wfShow.err)
const runsResult = await run(ws, 'ls /ci/runs/')
printOut('ls /ci/runs/', runsResult.out, runsResult.err)
const runs = runsResult.out.trim().split('\n').filter((s) => s !== '')
if (runs.length === 0) {
console.log('no runs found')
return
}
const runName = runs[0]!.trim().split('/').pop() ?? ''
const runPath = `/ci/runs/${runName}`
const runLs = await run(ws, `ls "${runPath}/"`)
printOut(`ls ${runPath}/`, runLs.out, runLs.err)
const runJson = await run(ws, `cat "${runPath}/run.json" | head -n 20`)
printOut(`cat ${runPath}/run.json | head -n 20`, runJson.out, runJson.err)
const runStat = await run(ws, `stat "${runPath}"`)
console.log(`=== stat ${runPath} ===`)
console.log(` ${runStat.out.trim()}`)
// chmod/chown/touch never hit the Actions API: attrs land in the
// workspace namespace (durable, snapshot-captured) and merge into
// dispatch-level stat.
console.log(`=== metadata overlay on ${runPath} ===`)
const metaRes = await ws.execute(
`chmod 640 "${runPath}" && chown 500:dev "${runPath}" && touch -t 202601021530 "${runPath}"`,
)
console.log(` chmod/chown/touch exit=${String(metaRes.exitCode)}`)
try {
const metaSt = (await ws.dispatch('stat', `${runPath}`)) as FileStat
const metaMode = metaSt.mode !== null ? metaSt.mode.toString(8) : '-'
console.log(
` dispatch stat: mode=${metaMode} uid=${String(metaSt.uid)} gid=${String(metaSt.gid)} mtime=${String(metaSt.modified)}`,
)
} catch {
console.log(' dispatch stat: run path unavailable (check GITHUB_TOKEN)')
}
const jobsPath = `${runPath}/jobs`
const jobsLs = await run(ws, `ls "${jobsPath}/"`)
printOut(`ls ${jobsPath}/`, jobsLs.out, jobsLs.err)
const jobsLines = jobsLs.out.trim().split('\n').filter((s) => s !== '')
const jsonJobs = jobsLines.filter((j) => j.endsWith('.json'))
const logJobs = jobsLines.filter((j) => j.endsWith('.log'))
const firstJsonJob = jsonJobs[0]
if (firstJsonJob !== undefined) {
const jobName = firstJsonJob
const jobPath = `${jobsPath}/${jobName}`
const jobJson = await run(ws, `cat "${jobPath}" | head -n 20`)
printOut(`cat ${jobName} | head -n 20`, jobJson.out, jobJson.err)
const jobStat = await run(ws, `stat "${jobPath}"`)
console.log(`=== stat ${jobName} ===`)
console.log(` ${jobStat.out.trim()}`)
}
const firstLogJob = logJobs[0]
if (firstLogJob !== undefined) {
const logName = firstLogJob
const logPath = `${jobsPath}/${logName}`
const logHead = await run(ws, `head -n 20 "${logPath}"`)
printOut(`head -n 20 ${logName}`, logHead.out, logHead.err, 1000)
const logTail = await run(ws, `tail -n 10 "${logPath}"`)
printOut(`tail -n 10 ${logName}`, logTail.out, logTail.err)
const logWc = await run(ws, `wc -l "${logPath}"`)
console.log(`=== wc -l ${logName} ===`)
console.log(` ${logWc.out.trim()}`)
}
const annPath = `${runPath}/annotations.jsonl`
const annResult = await run(ws, `cat "${annPath}"`)
console.log(`=== cat ${annPath} ===`)
const annOut = annResult.out.trim()
if (annOut !== '') {
for (const line of annOut.split('\n').slice(0, 5)) console.log(` ${line.slice(0, 120)}`)
} else {
console.log(' (no annotations)')
}
const artifactsLs = await run(ws, `ls "${runPath}/artifacts/"`)
console.log(`=== ls ${runPath}/artifacts/ ===`)
if (artifactsLs.out.trim() !== '') console.log(artifactsLs.out)
else console.log(' (no artifacts)')
const treeOut = await run(ws, 'tree -L 2 /ci/')
printOut('tree -L 2 /ci/', treeOut.out, treeOut.err, 1500)
const findLog = await run(ws, "find /ci/runs/ -name '*.log' | head -n 10")
printOut("find /ci/runs/ -name '*.log' | head -n 10", findLog.out, findLog.err)
const findJson = await run(ws, "find /ci/runs/ -name '*.json' | head -n 10")
printOut("find /ci/runs/ -name '*.json' | head -n 10", findJson.out, findJson.err)
const wfStat = await run(ws, 'stat "/ci/workflows/"')
console.log('=== stat /ci/workflows/ ===')
console.log(` ${wfStat.out.trim()}`)
const runsStat = await run(ws, 'stat "/ci/runs/"')
console.log('=== stat /ci/runs/ ===')
console.log(` ${runsStat.out.trim()}`)
} finally {
await ws.close()
}
}
main().catch((err: unknown) => {
console.error(err)
process.exit(1)
})
examples/typescript/github_ci/github_ci_browser/main.ts
-79
View File
@@ -1,79 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import dotenv from 'dotenv'
import { GitHubCIResource, MountMode, Workspace } from '@struktoai/mirage-browser'
const __HERE = fileURLToPath(new URL('.', import.meta.url))
dotenv.config({ path: resolve(__HERE, '../../../../.env.development') })
function buildConfig(): { token: string; owner: string; repo: string } {
const token = process.env.GITHUB_TOKEN
if (token === undefined || token === '') throw new Error('GITHUB_TOKEN env var is required')
const owner = process.env.GITHUB_OWNER ?? 'strukto-ai'
const repo = process.env.GITHUB_REPO ?? 'mirage'
return { token, owner, repo }
}
async function run(ws: Workspace, cmd: string): Promise<string> {
console.log(`$ ${cmd}`)
const r = await ws.execute(cmd)
if (r.exitCode !== 0 && r.stderrText !== '') {
console.log(` STDERR: ${r.stderrText.slice(0, 200)}`)
}
const out = r.stdoutText.replace(/\s+$/, '')
if (out !== '') {
for (const line of out.split('\n').slice(0, 12)) console.log(` ${line.slice(0, 200)}`)
}
return out
}
async function main(): Promise<void> {
const cfg = buildConfig()
console.log(`Loading ${cfg.owner}/${cfg.repo} CI via @struktoai/mirage-browser …`)
const resource = new GitHubCIResource(cfg)
const ws = new Workspace({ '/ci': resource }, { mode: MountMode.READ })
try {
console.log('=== BROWSER MODE: GitHubCIResource → api.github.com (direct, CORS) ===\n')
await run(ws, 'ls /ci/')
console.log('')
await run(ws, 'ls /ci/workflows/')
console.log('')
const runsOut = await run(ws, 'ls /ci/runs/')
const firstRun = runsOut.split('\n')[0]
if (firstRun !== undefined && firstRun !== '') {
const runPath = `/ci/runs/${firstRun}`
console.log('')
await run(ws, `ls "${runPath}/"`)
console.log('')
await run(ws, `head -n 10 "${runPath}/run.json"`)
}
console.log('')
await run(ws, 'tree -L 2 /ci/')
} finally {
await ws.close()
}
}
main().catch((err: unknown) => {
console.error(err)
process.exit(1)
})
examples/typescript/github_ci/github_ci_vfs.ts
-137
View File
@@ -1,137 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { createRequire } from 'node:module'
import { resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import dotenv from 'dotenv'
import {
GitHubCIResource,
MountMode,
patchNodeFs,
Workspace,
type GitHubCIConfig,
} from '@struktoai/mirage-node'
const require = createRequire(import.meta.url)
const fs = require('fs') as typeof import('fs')
const __HERE = fileURLToPath(new URL('.', import.meta.url))
dotenv.config({ path: resolve(__HERE, '../../../.env.development') })
function buildConfig(): GitHubCIConfig {
const token = process.env.GITHUB_TOKEN
if (token === undefined || token === '') throw new Error('GITHUB_TOKEN env var is required')
const owner = process.env.GITHUB_OWNER ?? 'strukto-ai'
const repo = process.env.GITHUB_REPO ?? 'mirage'
return { token, owner, repo, maxRuns: 300 }
}
async function main(): Promise<void> {
const resource = new GitHubCIResource(buildConfig())
const ws = new Workspace({ '/ci/': resource }, { mode: MountMode.READ })
const restore = patchNodeFs(ws)
try {
console.log('=== VFS MODE: fs.readFile() reads from GitHub CI transparently ===\n')
console.log('--- fs.readdir() root ---')
const entries = await fs.promises.readdir('/ci')
for (const e of entries) console.log(` ${e}`)
console.log('\n--- fs.readdir() workflows ---')
const workflows = await fs.promises.readdir('/ci/workflows')
for (const wf of workflows.slice(0, 10)) console.log(` ${wf}`)
if (workflows.length > 0) {
const wfPath = `/ci/workflows/${workflows[0]!}`
console.log(`\n--- open() + read ${wfPath} ---`)
const wfData = JSON.parse(await fs.promises.readFile(wfPath, 'utf-8')) as Record<string, unknown>
console.log(` name: ${String(wfData['name'])}`)
console.log(` path: ${String(wfData['path'])}`)
console.log(` state: ${String(wfData['state'])}`)
}
console.log('\n--- fs.readdir() runs ---')
const runs = await fs.promises.readdir('/ci/runs')
for (const r of runs.slice(0, 5)) console.log(` ${r}`)
if (runs.length > 5) console.log(` ... (${String(runs.length)} total)`)
if (runs.length > 0) {
const runDir = `/ci/runs/${runs[0]!}`
console.log(`\n--- fs.readdir() ${runDir} ---`)
const contents = await fs.promises.readdir(runDir)
for (const c of contents) console.log(` ${c}`)
const runJsonName = contents.find((c) => c === 'run.json')
if (runJsonName !== undefined) {
console.log('\n--- open() + read run.json ---')
const data = JSON.parse(
await fs.promises.readFile(`${runDir}/run.json`, 'utf-8'),
) as Record<string, unknown>
console.log(` status: ${String(data['status'])}`)
console.log(` conclusion: ${String(data['conclusion'])}`)
console.log(` event: ${String(data['event'])}`)
console.log(` branch: ${String(data['head_branch'])}`)
}
const jobsName = contents.find((c) => c === 'jobs')
if (jobsName !== undefined) {
const jobsDir = `${runDir}/jobs`
console.log('\n--- fs.readdir() jobs ---')
const jobs = await fs.promises.readdir(jobsDir)
for (const j of jobs.slice(0, 10)) console.log(` ${j}`)
const jsonJobs = jobs.filter((j) => j.endsWith('.json'))
const logJobs = jobs.filter((j) => j.endsWith('.log'))
if (jsonJobs[0] !== undefined) {
console.log('\n--- open() + read job .json ---')
const data = JSON.parse(
await fs.promises.readFile(`${jobsDir}/${jsonJobs[0]}`, 'utf-8'),
) as Record<string, unknown>
console.log(` name: ${String(data['name'])}`)
console.log(` status: ${String(data['status'])}`)
console.log(` conclusion: ${String(data['conclusion'])}`)
const steps = (data['steps'] ?? []) as Array<Record<string, unknown>>
console.log(` steps: ${String(steps.length)}`)
for (const s of steps.slice(0, 3)) {
console.log(
` ${String(s['number'])}. ${String(s['name'])} -> ${String(s['conclusion'])}`,
)
}
}
if (logJobs[0] !== undefined) {
console.log('\n--- open() + read job .log (first 10 lines) ---')
const text = await fs.promises.readFile(`${jobsDir}/${logJobs[0]}`, 'utf-8')
const lines = text.split('\n')
for (const line of lines.slice(0, 10)) console.log(` ${line.slice(0, 120)}`)
if (lines.length > 10) console.log(' ...')
}
}
}
const records = ws.records
const total = records.reduce((s, r) => s + (r.bytes ?? 0), 0)
console.log(`\nStats: ${String(records.length)} ops, ${String(total)} bytes transferred`)
} finally {
restore()
await ws.close()
}
}
main().catch((err: unknown) => {
console.error(err)
process.exit(1)
})
integ/resources/argerr/maxdepth.json
-1
View File
@@ -9,7 +9,6 @@
"argerr-email",
"argerr-gdocs",
"argerr-gdrive",
"argerr-github_ci",
"argerr-gmail",
"argerr-gsheets",
"argerr-gslides",
integ/resources/argerr/mindepth.json
-1
View File
@@ -9,7 +9,6 @@
"argerr-email",
"argerr-gdocs",
"argerr-gdrive",
"argerr-github_ci",
"argerr-gmail",
"argerr-gsheets",
"argerr-gslides",
integ/resources/argerr/mtime.json
-1
View File
@@ -9,7 +9,6 @@
"argerr-email",
"argerr-gdocs",
"argerr-gdrive",
"argerr-github_ci",
"argerr-gmail",
"argerr-gsheets",
"argerr-gslides",
integ/resources/argerr/size.json
-1
View File
@@ -9,7 +9,6 @@
"argerr-email",
"argerr-gdocs",
"argerr-gdrive",
"argerr-github_ci",
"argerr-gmail",
"argerr-gsheets",
"argerr-gslides",
integ/resources/dify/range.json
+80
View File
@@ -0,0 +1,80 @@
{
"cases": [
{
"id": "df_range_head",
"seq": 950030,
"targets": [
"dify"
],
"command": "true",
"check": {
"read": "/knowledge/guides/auth",
"offset": 0,
"size": 14
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "Authentication"
}
},
{
"id": "df_range_middle",
"seq": 950031,
"targets": [
"dify"
],
"command": "true",
"check": {
"read": "/knowledge/guides/auth",
"offset": 64,
"size": 8
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "Requests"
}
},
{
"id": "df_range_to_eof",
"seq": 950032,
"targets": [
"dify"
],
"command": "true",
"check": {
"read": "/knowledge/guides/auth",
"offset": 161,
"size": 200
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "HTTP 429 and must back off."
}
},
{
"id": "df_range_past_eof",
"seq": 950033,
"targets": [
"dify"
],
"command": "true",
"check": {
"read": "/knowledge/guides/auth",
"offset": 500,
"size": 10
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": ""
}
}
]
}
integ/resources/discord/range.json
+80
View File
@@ -0,0 +1,80 @@
{
"cases": [
{
"id": "dc_range_attachment_head",
"seq": 950020,
"targets": [
"discord"
],
"command": "true",
"check": {
"read": "/discord/Mirage HQ__100000000000000001/channels/general__200000000000000001/2026-06-01/files/budget__500000000000000001.csv",
"offset": 0,
"size": 7
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "quarter"
}
},
{
"id": "dc_range_attachment_middle",
"seq": 950021,
"targets": [
"discord"
],
"command": "true",
"check": {
"read": "/discord/Mirage HQ__100000000000000001/channels/general__200000000000000001/2026-06-01/files/budget__500000000000000001.csv",
"offset": 8,
"size": 6
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "amount"
}
},
{
"id": "dc_range_rendered_history",
"seq": 950022,
"targets": [
"discord"
],
"command": "true",
"check": {
"read": "/discord/Mirage HQ__100000000000000001/channels/general__200000000000000001/2026-06-01/chat.jsonl",
"offset": 0,
"size": 1
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "{"
}
},
{
"id": "dc_range_rendered_member",
"seq": 950023,
"targets": [
"discord"
],
"command": "true",
"check": {
"read": "/discord/Mirage HQ__100000000000000001/members/alice__300000000000000001.json",
"offset": 0,
"size": 7
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "{\"user\""
}
}
]
}
integ/resources/ranges/read.json
+238
View File
@@ -0,0 +1,238 @@
{
"cases": [
{
"id": "range_window_middle",
"seq": 950000,
"targets": [
"ram",
"disk",
"redis",
"s3",
"s3-prefix",
"gridfs",
"gridfs-prefix",
"databricks",
"databricks-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"ssh",
"nextcloud",
"sharepoint",
"sharepoint-prefix",
"hf",
"hf-prefix",
"box",
"gdrive",
"opfs"
],
"command": "printf '0123456789' > {mount}/range.txt",
"check": {
"read": "{mount}/range.txt",
"offset": 2,
"size": 3
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "234"
}
},
{
"id": "range_window_to_eof",
"seq": 950001,
"targets": [
"ram",
"disk",
"redis",
"s3",
"s3-prefix",
"gridfs",
"gridfs-prefix",
"databricks",
"databricks-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"ssh",
"nextcloud",
"sharepoint",
"sharepoint-prefix",
"hf",
"hf-prefix",
"box",
"gdrive",
"opfs"
],
"command": "printf '0123456789' > {mount}/range_eof.txt",
"check": {
"read": "{mount}/range_eof.txt",
"offset": 7,
"size": null
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "789"
}
},
{
"id": "range_head_of_file",
"seq": 950002,
"targets": [
"ram",
"disk",
"redis",
"s3",
"s3-prefix",
"gridfs",
"gridfs-prefix",
"databricks",
"databricks-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"ssh",
"nextcloud",
"sharepoint",
"sharepoint-prefix",
"hf",
"hf-prefix",
"box",
"gdrive",
"opfs"
],
"command": "printf '0123456789' > {mount}/range_head.txt",
"check": {
"read": "{mount}/range_head.txt",
"offset": 0,
"size": 4
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "0123"
}
},
{
"id": "range_size_past_eof_is_short",
"seq": 950003,
"targets": [
"ram",
"disk",
"redis",
"s3",
"s3-prefix",
"gridfs",
"gridfs-prefix",
"databricks",
"databricks-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"ssh",
"nextcloud",
"sharepoint",
"sharepoint-prefix",
"hf",
"hf-prefix",
"box",
"gdrive",
"opfs"
],
"command": "printf 'abc' > {mount}/range_short.txt",
"check": {
"read": "{mount}/range_short.txt",
"offset": 0,
"size": 100
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "abc"
}
},
{
"id": "range_offset_past_eof_is_empty",
"seq": 950004,
"targets": [
"ram",
"disk",
"redis",
"s3",
"s3-prefix",
"gridfs",
"gridfs-prefix",
"databricks",
"databricks-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"ssh",
"nextcloud",
"sharepoint",
"sharepoint-prefix",
"hf",
"hf-prefix",
"box",
"gdrive",
"opfs"
],
"command": "printf 'abc' > {mount}/range_past.txt",
"check": {
"read": "{mount}/range_past.txt",
"offset": 99,
"size": 5
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": ""
}
},
{
"id": "range_zero_length_is_empty",
"seq": 950005,
"targets": [
"ram",
"disk",
"redis",
"s3",
"s3-prefix",
"gridfs",
"gridfs-prefix",
"databricks",
"databricks-prefix",
"dropbox",
"dropbox-root",
"onedrive",
"ssh",
"nextcloud",
"sharepoint",
"sharepoint-prefix",
"hf",
"hf-prefix",
"box",
"gdrive",
"opfs"
],
"command": "printf 'abc' > {mount}/range_zero.txt",
"check": {
"read": "{mount}/range_zero.txt",
"offset": 1,
"size": 0
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": ""
}
}
]
}
integ/resources/sizes/stat_read.json
-143
View File
@@ -678,136 +678,6 @@
"stderr": ""
}
},
{
"id": "sz_ci_workflow_stat",
"seq": 680078,
"targets": [
"github_ci"
],
"command": "stat -c %s /ci/workflows/CI_101.json",
"expect": {
"exit": 0,
"stdout": "192\n",
"stderr": ""
}
},
{
"id": "sz_ci_workflow_read",
"seq": 680079,
"targets": [
"github_ci"
],
"command": "cat /ci/workflows/CI_101.json | wc -c",
"expect": {
"exit": 0,
"stdout": "192\n",
"stderr": ""
}
},
{
"id": "sz_ci_run_stat",
"seq": 680080,
"targets": [
"github_ci"
],
"command": "stat -c %s /ci/runs/CI_9001/run.json",
"expect": {
"exit": 0,
"stdout": "228\n",
"stderr": ""
}
},
{
"id": "sz_ci_run_read",
"seq": 680081,
"targets": [
"github_ci"
],
"command": "cat /ci/runs/CI_9001/run.json | wc -c",
"expect": {
"exit": 0,
"stdout": "228\n",
"stderr": ""
}
},
{
"id": "sz_ci_job_stat",
"seq": 680082,
"targets": [
"github_ci"
],
"command": "stat -c %s /ci/runs/CI_9001/jobs/build_7001.json",
"expect": {
"exit": 0,
"stdout": "321\n",
"stderr": ""
}
},
{
"id": "sz_ci_job_read",
"seq": 680083,
"targets": [
"github_ci"
],
"command": "cat /ci/runs/CI_9001/jobs/build_7001.json | wc -c",
"expect": {
"exit": 0,
"stdout": "321\n",
"stderr": ""
}
},
{
"id": "sz_ci_artifact_stat",
"seq": 680084,
"targets": [
"github_ci"
],
"command": "stat -c %s /ci/runs/CI_9001/artifacts/dist_5001.zip",
"expect": {
"exit": 0,
"stdout": "148\n",
"stderr": ""
}
},
{
"id": "sz_ci_artifact_read",
"seq": 680085,
"targets": [
"github_ci"
],
"command": "cat /ci/runs/CI_9001/artifacts/dist_5001.zip | wc -c",
"expect": {
"exit": 0,
"stdout": "148\n",
"stderr": ""
}
},
{
"id": "sz_ci_log_read",
"seq": 680086,
"targets": [
"github_ci"
],
"command": "cat /ci/runs/CI_9001/jobs/build_7001.log | wc -c",
"expect": {
"exit": 0,
"stdout": "71\n",
"stderr": ""
}
},
{
"id": "sz_ci_annotations_read",
"seq": 680087,
"targets": [
"github_ci"
],
"command": "cat /ci/runs/CI_9001/annotations.jsonl | wc -c",
"expect": {
"exit": 0,
"stdout": "105\n",
"stderr": ""
}
},
{
"id": "sz_qdrant_row_json_stat",
"seq": 680088,
@@ -1380,19 +1250,6 @@
"stderr": ""
}
},
{
"id": "sz_ci_every_json_size_matches_read",
"seq": 700111,
"targets": [
"github_ci"
],
"command": "n=0; t=0; for f in /ci/workflows/*.json /ci/runs/*/run.json /ci/runs/*/jobs/*.json; do t=$((t+1)); test \"$(stat -c %s \"$f\")\" = \"$(cat \"$f\" | wc -c)\" && n=$((n+1)); done; echo \"$n/$t\"",
"expect": {
"exit": 0,
"stdout": "3/3\n",
"stderr": ""
}
},
{
"id": "sz_trello_sized_kinds_size_matches_read",
"seq": 700112,
integ/resources/slack/range.json
+80
View File
@@ -0,0 +1,80 @@
{
"cases": [
{
"id": "slack_range_upload_head",
"seq": 950010,
"targets": [
"slack"
],
"command": "true",
"check": {
"read": "/slack/channels/eng-agents__C8/2026-02-06/files/model_eval_results__F7.csv",
"offset": 0,
"size": 5
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "suite"
}
},
{
"id": "slack_range_upload_middle",
"seq": 950011,
"targets": [
"slack"
],
"command": "true",
"check": {
"read": "/slack/channels/eng-agents__C8/2026-02-06/files/model_eval_results__F7.csv",
"offset": 6,
"size": 5
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "cases"
}
},
{
"id": "slack_range_rendered_history",
"seq": 950012,
"targets": [
"slack"
],
"command": "true",
"check": {
"read": "/slack/dms/dave__D7/2026-03-10/chat.jsonl",
"offset": 9,
"size": 7
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "message"
}
},
{
"id": "slack_range_rendered_user",
"seq": 950013,
"targets": [
"slack"
],
"command": "true",
"check": {
"read": "/slack/users/priya__U7.json",
"offset": 0,
"size": 5
},
"expect": {
"exit": 0,
"stdout": "",
"stderr": "",
"check": "{\"id\""
}
}
]
}
integ/runners/python/adapters.py
-46
View File
@@ -73,8 +73,6 @@ from mirage.resource.gdocs.gdocs import GDocsResource
from mirage.resource.gdrive.config import GoogleDriveConfig
from mirage.resource.gdrive.gdrive import GoogleDriveResource
from mirage.resource.github import GitHubConfig, GitHubResource
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.resource.github_ci.github_ci import GitHubCIResource
from mirage.resource.gmail.config import GmailConfig
from mirage.resource.gmail.gmail import GmailResource
from mirage.resource.gridfs import GridFSConfig, GridFSResource
@@ -1104,35 +1102,6 @@ class GitHubService:
return None
class GitHubCIService:
"""Points github_ci mounts at the fake api.github.com server.
Reuses the external github_server.py process on GITHUB_URL, which also
serves the fixed Actions dataset (workflows/runs/jobs/artifacts).
Args:
url (str): GITHUB_URL origin the fake is listening on.
"""
def __init__(self, url: str) -> None:
self.url = url
@classmethod
async def create(cls) -> "GitHubCIService":
return cls(os.environ["GITHUB_URL"].rstrip("/"))
def resource(self, mount: dict) -> GitHubCIResource:
owner, _, repo = mount["repo"].partition("/")
return GitHubCIResource(
GitHubCIConfig(token="ghp-integ",
owner=owner,
repo=repo,
base_url=self.url))
async def teardown(self) -> None:
return None
class DifyService:
def __init__(self, runner, base: str, dataset: str) -> None:
@@ -1980,13 +1949,6 @@ async def build_github(
return await service.resource(mount), _noop
def build_github_ci(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, GitHubCIService)
return service.resource(mount), _noop
def build_slack(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
@@ -2025,11 +1987,6 @@ ARG_ERROR_RESOURCES: dict[str, tuple[type, type, dict[str, object]]] = {
"client_id": "c",
"refresh_token": "r"
}),
"github_ci": (GitHubCIResource, GitHubCIConfig, {
"token": "t",
"owner": "o",
"repo": "r"
}),
"gmail": (GmailResource, GmailConfig, {
"client_id": "c",
"refresh_token": "r"
@@ -2119,7 +2076,6 @@ BUILDERS = {
"box": build_box,
"dropbox": build_dropbox,
"github": build_github,
"github_ci": build_github_ci,
"slack": build_slack,
"trello": build_trello,
"discord": build_discord,
@@ -2179,8 +2135,6 @@ async def make_service(target: dict, run_id: str) -> "Service | None":
if "gh" in (target.get("clis") or []):
await github.reset()
return github
if target.get("service") == "github_ci":
return await GitHubCIService.create()
if target.get("service") == "slack":
return await SlackService.create()
if target.get("service") == "trello":
integ/runners/python/harness.py
+27 -1
View File
@@ -249,6 +249,24 @@ def _check_field(st: FileStat, name: str) -> str:
async def stat_check(ws, check: dict) -> str:
"""The probe a case runs beside its command, as one printable line.
Two forms. ``stat`` names a path and the FileStat fields to print.
``read`` names a path and a byte window, and prints what that window
returned: no shell command asks for one, because commands read whole
files, so the ranged read op is only reachable through the same door
FUSE and the ops facade use.
Args:
ws: the workspace the case runs against.
check (dict): the case's ``check`` block.
"""
if "read" in check:
data, _ = await ws.dispatch("read",
PathSpec.from_str_path(check["read"]),
offset=check.get("offset", 0),
size=check.get("size"))
return data.decode("utf-8", "replace")
try:
st, _ = await ws.dispatch("stat",
PathSpec.from_str_path(check["stat"]))
@@ -292,8 +310,16 @@ def bind_mount(case: dict, mount_path: str) -> dict:
if "command" in bound:
for token, value in tokens.items():
bound["command"] = bound["command"].replace(token, value)
check = bound.get("check")
if isinstance(check, dict):
bound["check"] = dict(check)
for name in ("stat", "read"):
if isinstance(check.get(name), str):
for token, value in tokens.items():
bound["check"][name] = bound["check"][name].replace(
token, value)
expect = dict(bound["expect"])
for name in ("stdout", "stderr"):
for name in ("stdout", "stderr", "check"):
if isinstance(expect.get(name), str):
for token, value in tokens.items():
expect[name] = expect[name].replace(token, value)
integ/runners/typescript/adapters.ts
-24
View File
@@ -44,7 +44,6 @@ import {
GCalResource,
GCSResource,
GitHubResource,
GitHubCIResource,
GridFSResource,
GSheetsResource,
GSlidesResource,
@@ -1467,27 +1466,6 @@ async function openGitHub(target: Target): Promise<Open> {
return { ws: ws as unknown as ExecWorkspace, cleanup: () => ws.close() }
}
// Reuses the external github_server.py process on GITHUB_URL, which also
// serves the fixed Actions dataset (workflows/runs/jobs/artifacts).
async function openGitHubCI(target: Target): Promise<Open> {
let base = process.env.GITHUB_URL ?? ''
while (base.endsWith('/')) base = base.slice(0, -1)
if (base === '') throw new Error('github_ci target requires GITHUB_URL')
const mounts: Record<string, GitHubCIResource | [GitHubCIResource, MountMode]> = {}
for (const m of target.mounts) {
const [owner, repo] = String(m.repo).split('/')
const resource = new GitHubCIResource({
token: 'ghp-integ',
owner: owner ?? '',
repo: repo ?? '',
baseUrl: base,
})
mounts[m.path] = m.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
return { ws: ws as unknown as ExecWorkspace, cleanup: () => ws.close() }
}
async function openDify(target: Target): Promise<Open> {
const endpoint = process.env.DIFY_ENDPOINT
if (!endpoint) throw new Error('dify target requires DIFY_ENDPOINT')
@@ -1644,7 +1622,6 @@ const ARG_ERROR_RESOURCES: Record<string, () => Resource> = {
}),
gdocs: () => new GDocsResource({ clientId: 'c', refreshToken: 'r' }),
gdrive: () => new GDriveResource({ clientId: 'c', refreshToken: 'r' }),
github_ci: () => new GitHubCIResource({ token: 't', owner: 'o', repo: 'r' }),
gmail: () => new GmailResource({ clientId: 'c', refreshToken: 'r' }),
gsheets: () => new GSheetsResource({ clientId: 'c', refreshToken: 'r' }),
gslides: () => new GSlidesResource({ clientId: 'c', refreshToken: 'r' }),
@@ -1721,7 +1698,6 @@ export const ADAPTERS: Record<
lancedb: openLancedb,
notion: openNotion,
github: openGitHub,
github_ci: openGitHubCI,
slack: openSlack,
trello: openTrello,
discord: openDiscord,
integ/runners/typescript/harness.ts
+38 -6
View File
@@ -105,8 +105,11 @@ export interface Expect {
}
export interface StatCheck {
stat: string
fields: string[]
stat?: string
fields?: string[]
read?: string
offset?: number
size?: number | null
}
export interface Case {
@@ -157,7 +160,7 @@ export interface HarnessStat {
export interface ExecWorkspace {
execute(cmd: string, opts?: { stdin?: Uint8Array; sessionId?: string }): Promise<ExecResult>
dispatch(opName: string, path: string): Promise<unknown>
dispatch(opName: string, path: string, args?: readonly unknown[], kwargs?: Record<string, unknown>): Promise<unknown>
cache: { clear(): Promise<void> }
mounts(): readonly { resource: { index?: { clear(): Promise<void> } } }[]
createSession(sessionId: string, options: { mounts: Record<string, string> | string[] }): unknown
@@ -393,15 +396,31 @@ function checkField(st: HarnessStat, name: string): string {
return `${name}=${value}`
}
/**
* The probe a case runs beside its command, as one printable line.
*
* Two forms. `stat` names a path and the FileStat fields to print. `read`
* names a path and a byte window, and prints what that window returned: no
* shell command asks for one, because commands read whole files, so the
* ranged read op is only reachable through the same door FUSE and the ops
* facade use.
*/
export async function statCheck(ws: ExecWorkspace, check: StatCheck): Promise<string> {
if (check.read !== undefined) {
const data = (await ws.dispatch('read', check.read, [], {
offset: check.offset ?? 0,
size: check.size ?? null,
})) as Uint8Array
return new TextDecoder().decode(data)
}
let st: HarnessStat
try {
st = (await ws.dispatch('stat', check.stat)) as HarnessStat
st = (await ws.dispatch('stat', check.stat ?? '')) as HarnessStat
} catch (err) {
if ((err as { code?: string }).code === 'ENOENT') return 'absent\n'
throw err
}
return check.fields.map((name) => checkField(st, name)).join(' ') + '\n'
return (check.fields ?? []).map((name) => checkField(st, name)).join(' ') + '\n'
}
function provisionLine(r: ProvisionInfo): string {
@@ -433,16 +452,29 @@ export function bindMount(c: Case, mountPath: string): Case {
([token]) =>
c.command?.includes(token) === true ||
c.expect.stdout.includes(token) ||
c.expect.stderr.includes(token),
c.expect.stderr.includes(token) ||
c.check?.stat?.includes(token) === true ||
c.check?.read?.includes(token) === true ||
c.expect.check?.includes(token) === true,
)
if (!present) return c
const check =
c.check === undefined
? undefined
: {
...c.check,
...(c.check.stat !== undefined ? { stat: subst(c.check.stat) } : {}),
...(c.check.read !== undefined ? { read: subst(c.check.read) } : {}),
}
return {
...c,
...(c.command !== undefined ? { command: subst(c.command) } : {}),
...(check !== undefined ? { check } : {}),
expect: {
...c.expect,
stdout: subst(c.expect.stdout),
stderr: subst(c.expect.stderr),
...(c.expect.check !== undefined ? { check: subst(c.expect.check) } : {}),
},
}
}
integ/server/dropbox_server.py
+38 -1
View File
@@ -33,6 +33,26 @@ def _content_hit(query: str, text: str) -> bool:
text) is not None
def _parse_range(header: str | None, size: int) -> tuple[int, int] | None:
"""The byte window an HTTP ``Range`` header asks for, clamped to `size`.
Dropbox's content endpoints honor ``Range`` on ``/2/files/download``,
so the fake has to as well: without it a windowed read reads whole and
the push-down looks correct while moving every byte.
Args:
header (str | None): the request's ``Range`` value.
size (int): length of the stored content.
"""
if not header or not header.startswith("bytes="):
return None
spec = header[len("bytes="):]
start_s, _, end_s = spec.partition("-")
start = int(start_s) if start_s else 0
end = int(end_s) + 1 if end_s else size
return start, min(end, size)
class FakeDropbox:
"""One fake Dropbox account with explicit folder objects.
@@ -157,7 +177,24 @@ class FakeDropbox:
if stored is None:
return web.json_response({"error_summary": "path/not_found/..."},
status=409)
return web.Response(body=stored[0],
content = stored[0]
rng = _parse_range(request.headers.get("Range"), len(content))
if rng is None:
return web.Response(body=content,
content_type="application/octet-stream")
start, end = rng
if start >= len(content):
return web.Response(
body=b"",
status=416,
headers={"Content-Range": f"bytes */{len(content)}"},
content_type="application/octet-stream")
return web.Response(body=content[start:end],
status=206,
headers={
"Content-Range":
f"bytes {start}-{end - 1}/{len(content)}"
},
content_type="application/octet-stream")
async def handle_upload(self, request: web.Request) -> web.Response:
integ/server/github_server.py
-206
View File
@@ -16,11 +16,9 @@ import argparse
import asyncio
import base64
import hashlib
import io
import json
import re
import sys
import zipfile
from pathlib import Path
from aiohttp import web
@@ -50,89 +48,6 @@ DEFAULT_LOGIN = "integ-user"
ROOT_COMMIT_DATE = "1970-01-01T00:00:00Z"
def _build_artifact_zip() -> bytes:
# ZIP_STORED with a pinned date_time keeps the archive byte-identical
# across runs, so size_in_bytes below is a stable contract.
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf:
info = zipfile.ZipInfo("dist/report.txt",
date_time=(2026, 5, 3, 0, 0, 0))
zf.writestr(info, "ci artifact payload\n")
return buf.getvalue()
# One fixed Actions dataset served for every seeded repository. The list
# payloads and the single-object payloads share the same dicts on purpose:
# the github_ci backend sizes files from list responses and renders reads
# from the GET responses, and GitHub serves the same object shape on both.
CI_ARTIFACT_ZIP = _build_artifact_zip()
CI_JOB_LOG = (b"2026-05-03T00:05:00Z build started\n"
b"2026-05-03T00:08:00Z build finished\n")
CI_WORKFLOWS = [{
"id": 101,
"node_id": "W_101",
"name": "CI",
"path": ".github/workflows/ci.yml",
"state": "active",
"created_at": "2026-05-01T00:00:00Z",
"updated_at": "2026-05-02T00:00:00Z",
}]
CI_RUNS = [{
"id": 9001,
"name": "CI",
"run_number": 42,
"event": "push",
"status": "completed",
"conclusion": "success",
"head_branch": "main",
"created_at": "2026-05-03T00:00:00Z",
"updated_at": "2026-05-03T00:10:00Z",
}]
CI_JOBS = {
"9001": [{
"id":
7001,
"run_id":
9001,
"name":
"build",
"status":
"completed",
"conclusion":
"success",
"started_at":
"2026-05-03T00:05:00Z",
"completed_at":
"2026-05-03T00:08:00Z",
"steps": [{
"name": "checkout",
"status": "completed",
"conclusion": "success",
"number": 1,
}],
}],
}
CI_ARTIFACTS = {
"9001": [{
"id": 5001,
"name": "dist",
"size_in_bytes": len(CI_ARTIFACT_ZIP),
"expired": False,
"created_at": "2026-05-03T00:09:00Z",
"updated_at": "2026-05-03T00:09:00Z",
}],
}
CI_ANNOTATIONS = {
"7001": [{
"path": "src/app.py",
"start_line": 3,
"end_line": 3,
"annotation_level": "warning",
"message": "unused import",
}],
}
def _blob_sha(data: bytes) -> str:
# Real git object id so shas look plausible and stay stable across runs.
header = f"blob {len(data)}\0".encode()
@@ -1673,99 +1588,6 @@ class GitHubServer:
"items": items,
})
async def _ci_guard(self, request: web.Request) -> web.Response | None:
if not self._authed(request):
return _error(401, "Requires authentication")
if self._lookup(request) is None:
return _error(404, "Not Found")
return None
async def ci_workflows(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
return web.json_response({
"total_count": len(CI_WORKFLOWS),
"workflows": CI_WORKFLOWS,
})
async def ci_workflow(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
wanted = request.match_info["workflow_id"]
for wf in CI_WORKFLOWS:
if str(wf["id"]) == wanted:
return web.json_response(wf)
return _error(404, "Not Found")
async def ci_runs(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
return web.json_response({
"total_count": len(CI_RUNS),
"workflow_runs": CI_RUNS,
})
async def ci_run(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
wanted = request.match_info["run_id"]
for run in CI_RUNS:
if str(run["id"]) == wanted:
return web.json_response(run)
return _error(404, "Not Found")
async def ci_jobs(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
jobs = CI_JOBS.get(request.match_info["run_id"], [])
return web.json_response({"total_count": len(jobs), "jobs": jobs})
async def ci_job(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
wanted = request.match_info["job_id"]
for jobs in CI_JOBS.values():
for job in jobs:
if str(job["id"]) == wanted:
return web.json_response(job)
return _error(404, "Not Found")
async def ci_job_logs(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
return web.Response(body=CI_JOB_LOG, content_type="text/plain")
async def ci_artifacts(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
artifacts = CI_ARTIFACTS.get(request.match_info["run_id"], [])
return web.json_response({
"total_count": len(artifacts),
"artifacts": artifacts,
})
async def ci_artifact_zip(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
return web.Response(body=CI_ARTIFACT_ZIP,
content_type="application/zip")
async def ci_annotations(self, request: web.Request) -> web.Response:
guard = await self._ci_guard(request)
if guard is not None:
return guard
anns = CI_ANNOTATIONS.get(request.match_info["check_run_id"], [])
return web.json_response(anns)
def _add_routes(app: web.Application, server: "GitHubServer",
prefix: str) -> None:
@@ -1844,34 +1666,6 @@ def _add_routes(app: web.Application, server: "GitHubServer",
server.blob)
app.router.add_get(f"{prefix}/search/code", server.search_code)
app.router.add_get(f"{prefix}/search/repositories", server.search_repos)
app.router.add_get(f"{prefix}/repos/{{owner}}/{{repo}}/actions/workflows",
server.ci_workflows)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/workflows/{{workflow_id}}",
server.ci_workflow)
app.router.add_get(f"{prefix}/repos/{{owner}}/{{repo}}/actions/runs",
server.ci_runs)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/runs/{{run_id}}",
server.ci_run)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/runs/{{run_id}}/jobs",
server.ci_jobs)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/jobs/{{job_id}}",
server.ci_job)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/jobs/{{job_id}}/logs",
server.ci_job_logs)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/runs/{{run_id}}/artifacts",
server.ci_artifacts)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/actions/artifacts/"
f"{{artifact_id}}/zip", server.ci_artifact_zip)
app.router.add_get(
f"{prefix}/repos/{{owner}}/{{repo}}/check-runs/"
f"{{check_run_id}}/annotations", server.ci_annotations)
def build_app(server: GitHubServer) -> web.Application:
integ/server/gws_server.ts
+22 -2
View File
@@ -1924,6 +1924,25 @@ interface Ctx {
query: URLSearchParams
body: Buffer
contentType: string
range: string
}
/**
* Serve `content` for an `alt=media` read, honoring an HTTP `Range`.
*
* Drive honors `Range` on media downloads, so the fake has to as well:
* without it a windowed read reads whole and the push-down looks correct
* while moving every byte. A window starting past EOF is a 416, which the
* ops factory turns back into the empty read POSIX expects.
*/
function media(content: Buffer, range: string): [number, Buffer, string] {
const octet = 'application/octet-stream'
if (!range.startsWith('bytes=')) return [200, content, octet]
const [startText, endText] = range.slice('bytes='.length).split('-')
const start = startText === '' ? 0 : Number(startText)
const end = endText === undefined || endText === '' ? content.length : Number(endText) + 1
if (start >= content.length) return [416, Buffer.alloc(0), octet]
return [206, content.subarray(start, Math.min(end, content.length)), octet]
}
function json(ctx: Ctx): Record<string, unknown> {
@@ -2097,7 +2116,7 @@ function route(ctx: Ctx): [number, object | Buffer | null, string?] {
const item = state.files.get(m[1] as string)
if (item === undefined) return NOT_FOUND
if (method === 'GET' && query.get('alt') === 'media') {
return [200, item.content, 'application/octet-stream']
return media(item.content, ctx.range)
}
if (method === 'GET') return [200, fmtFile(item)]
if (method === 'PATCH') {
@@ -2190,7 +2209,7 @@ function route(ctx: Ctx): [number, object | Buffer | null, string?] {
const item = state.files.get(m[1] as string)
const revision = item?.revisions.find((r) => r.id === m?.[2])
if (item === undefined || revision === undefined) return NOT_FOUND
if (query.get('alt') === 'media') return [200, revision.content, 'application/octet-stream']
if (query.get('alt') === 'media') return media(revision.content, ctx.range)
return [
200,
{
@@ -2887,6 +2906,7 @@ export function startServer(port: number): Promise<http.Server> {
query: url.searchParams,
body: Buffer.concat(chunks),
contentType: req.headers['content-type'] ?? '',
range: req.headers.range ?? '',
})
} catch (err) {
console.error('gws_server: unhandled route error', err)
integ/targets.json
-34
View File
@@ -8,7 +8,6 @@
"dropbox": { "python": [], "typescript": [] },
"email": { "python": ["EMAIL_HOST"], "typescript": ["EMAIL_HOST"] },
"github": { "python": ["GITHUB_URL"], "typescript": ["GITHUB_URL"] },
"github_ci": { "python": ["GITHUB_URL"], "typescript": ["GITHUB_URL"] },
"gridfs": { "python": [], "typescript": [] },
"gws": { "python": ["GWS_URL"], "typescript": ["GWS_URL"] },
"hf": { "python": [], "typescript": ["HF_ENDPOINT"] },
@@ -1200,23 +1199,6 @@
}
]
},
{
"id": "github_ci",
"hosts": [
"python",
"typescript-node"
],
"service": "github_ci",
"mounts": [
{
"path": "/ci",
"resource": "github_ci",
"backend": "github_ci",
"repo": "integ/repo-v1",
"mode": "read"
}
]
},
{
"id": "hf-prefix",
"hosts": [
@@ -1846,22 +1828,6 @@
}
]
},
{
"id": "argerr-github_ci",
"facet": "argerr",
"hosts": [
"python",
"typescript-node"
],
"mounts": [
{
"path": "/github_ci",
"resource": "arg_error",
"backend": "github_ci",
"mode": "read"
}
]
},
{
"id": "argerr-gmail",
"facet": "argerr",
python/mirage/accessor/github_ci.py
-22
View File
@@ -1,22 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.base import Accessor
from mirage.resource.github_ci.config import GitHubCIConfig
class GitHubCIAccessor(Accessor):
def __init__(self, config: GitHubCIConfig) -> None:
self.config = config
python/mirage/commands/builtin/dify/io.py
+1
View File
@@ -29,6 +29,7 @@ from mirage.core.dify.stat import stat as _stat
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
read_range=_read,
read_stream=_read_stream,
stat=_stat,
is_mounted=lambda a: True,
python/mirage/commands/builtin/discord/io.py
+1
View File
@@ -27,6 +27,7 @@ from mirage.core.discord.stat import stat as _stat
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
read_range=_read,
read_stream=partial(stream_from_bytes, _read),
stat=_stat,
is_mounted=lambda a: True,
python/mirage/commands/builtin/github_ci/__init__.py
-32
View File
@@ -1,32 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic_bind import make_generic_commands
from mirage.commands.builtin.github_ci.find import find
from mirage.commands.builtin.github_ci.grep import grep
from mirage.commands.builtin.github_ci.io import IO as _IO
from mirage.commands.builtin.github_ci.rg import rg
_GITHUB_CI_OVERRIDES = {"find", "grep", "rg"}
COMMANDS = [
*make_generic_commands(
"github_ci",
_IO,
overrides=_GITHUB_CI_OVERRIDES,
),
find,
grep,
rg,
]
python/mirage/commands/builtin/github_ci/_provision.py
-21
View File
@@ -1,21 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.generic_bind.provision import (
exact_zero_provision, index_hit_read_provision)
file_read_provision = index_hit_read_provision
metadata_provision = exact_zero_provision
__all__ = ["file_read_provision", "metadata_provision"]
python/mirage/commands/builtin/github_ci/find.py
-93
View File
@@ -1,93 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from dataclasses import replace
from functools import partial
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.commands.builtin.generic.find import (is_link, parse_find_args,
resolve_start, walk_find)
from mirage.commands.builtin.github_ci._provision import metadata_provision
from mirage.commands.builtin.github_ci.io import resolve_glob
from mirage.commands.builtin.utils.output import format_records
from mirage.commands.config import CommandOpts
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagView
from mirage.core.github_ci.readdir import is_cross_run_root
from mirage.core.github_ci.readdir import readdir as _readdir
from mirage.core.github_ci.stat import stat as _stat
from mirage.io.types import ByteSource, IOResult
from mirage.provision.types import ProvisionResult
from mirage.types import PathSpec
async def find_provision(accessor: GitHubCIAccessor, paths: list[PathSpec],
texts: list[str],
opts: CommandOpts) -> ProvisionResult:
return await metadata_provision(
accessor, paths, texts,
replace(opts, command="find " + " ".join(p.virtual for p in paths)))
@command("find",
resource="github_ci",
spec=SPECS["find"],
provision=find_provision)
async def find(
accessor: GitHubCIAccessor,
paths: list[PathSpec],
texts: list[str],
opts: CommandOpts,
) -> tuple[ByteSource | None, IOResult]:
# The wrapper only exists for the cross-run guard: walking every run
# would fetch every run's logs. Filtering is the shared generic walk.
fl = FlagView(opts.flags, spec=SPECS["find"])
paths = await resolve_glob(accessor, paths, index=opts.index)
searches = paths if paths else [
PathSpec(virtual="/", directory="/", resource_path="")
]
args = parse_find_args(tuple(texts),
name=fl.as_str("name"),
type=fl.as_str("type"),
size=fl.as_str("size"),
mtime=fl.as_str("mtime"),
maxdepth=fl.as_str("maxdepth"),
iname=fl.as_str("iname"),
path=fl.as_str("path"),
mindepth=fl.as_str("mindepth"),
empty=fl.as_bool("empty"))
results: list[str] = []
links = opts.ns.links if opts.ns is not None else None
for search in searches:
# Same start-point rule as every other find path: only a
# directory has a subtree to walk.
start = await resolve_start(search,
args,
opts.stat_path,
is_link=is_link(links, search))
if not start.walk:
results.extend(start.results)
continue
if is_cross_run_root(search):
raise ValueError("find: recursive search across runs is disabled;"
" target a specific run (e.g. /ci/runs/<run>)")
results.extend(await walk_find(search,
readdir=partial(_readdir, accessor),
stat=partial(_stat, accessor),
index=opts.index,
args=args,
links=links,
follow=fl.as_bool("L")))
return format_records(results), IOResult()
python/mirage/commands/builtin/github_ci/grep.py
-50
View File
@@ -1,50 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.commands.builtin.generic.grep import grep as generic_grep
from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.github_ci.io import resolve_glob
from mirage.commands.config import CommandOpts
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagView
from mirage.core.github_ci.read import read as ci_read
from mirage.core.github_ci.readdir import is_cross_run_root
from mirage.core.github_ci.readdir import readdir as _readdir
from mirage.core.github_ci.stat import stat as _stat
from mirage.io.types import ByteSource, IOResult
from mirage.types import PathSpec
@command("grep", resource="github_ci", spec=SPECS["grep"])
async def grep(accessor: GitHubCIAccessor, paths: list[PathSpec],
texts: list[str],
opts: CommandOpts) -> tuple[ByteSource | None, IOResult]:
fl = FlagView(opts.flags, spec=SPECS["grep"])
resolved = await resolve_glob(accessor, paths, opts.index) if paths else []
recursive = fl.as_bool("r") or fl.as_bool("R")
if recursive and any(is_cross_run_root(p) for p in resolved):
raise ValueError("grep: recursive search across runs is disabled; "
"target a specific run (e.g. /ci/runs/<run>/jobs)")
return await generic_grep(
resolved,
texts,
opts.flags,
readdir=bound_op(_readdir, accessor, opts.index),
stat=bound_op(_stat, accessor, opts.index),
read_bytes=bound_op(ci_read, accessor, opts.index),
read_stream=None,
stdin=opts.stdin,
)
python/mirage/commands/builtin/github_ci/io.py
-37
View File
@@ -1,37 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from functools import partial
from mirage.commands.builtin.generic_bind import CommandIO
from mirage.commands.builtin.utils.wrap import stream_from_bytes
from mirage.core.github_ci.read import read as _read
from mirage.core.github_ci.readdir import readdir as _readdir
from mirage.core.github_ci.stat import stat as _stat
# GitHub CI logs/artifacts are read through the generic factory; find, grep
# and rg keep wrappers because they reject recursive search across runs (which
# would fetch every run's logs). GitHub CI is read-only, so the generic
# byte-mutation commands are intentionally absent (no write op wired). There is
# no native streaming read, so the stream is synthesized from the object read.
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
read_stream=partial(stream_from_bytes, _read),
stat=_stat,
is_mounted=lambda a: True,
local=False,
)
resolve_glob = IO.resolve_glob
python/mirage/commands/builtin/github_ci/rg.py
-47
View File
@@ -1,47 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.commands.builtin.generic.rg import rg as generic_rg
from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.github_ci.io import resolve_glob
from mirage.commands.config import CommandOpts
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
from mirage.core.github_ci.read import read as ci_read
from mirage.core.github_ci.readdir import is_cross_run_root
from mirage.core.github_ci.readdir import readdir as _readdir
from mirage.core.github_ci.stat import stat as _stat
from mirage.io.types import ByteSource, IOResult
from mirage.types import PathSpec
@command("rg", resource="github_ci", spec=SPECS["rg"])
async def rg(accessor: GitHubCIAccessor, paths: list[PathSpec],
texts: list[str],
opts: CommandOpts) -> tuple[ByteSource | None, IOResult]:
resolved = await resolve_glob(accessor, paths, opts.index) if paths else []
if any(is_cross_run_root(p) for p in resolved):
raise ValueError("rg: recursive search across runs is disabled; "
"target a specific run (e.g. /ci/runs/<run>/jobs)")
return await generic_rg(
resolved,
texts,
opts.flags,
readdir=bound_op(_readdir, accessor, opts.index),
stat=bound_op(_stat, accessor, opts.index),
read_bytes=bound_op(ci_read, accessor, opts.index),
read_stream=None,
stdin=opts.stdin,
)
python/mirage/commands/builtin/ram/io.py
+1
View File
@@ -37,6 +37,7 @@ from mirage.core.ram.write import write_bytes as _write
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
read_range=_read,
read_stream=_read_stream,
stat=_stat,
is_mounted=lambda a: a.store is not None,
python/mirage/commands/builtin/redis/io.py
+1
View File
@@ -37,6 +37,7 @@ from mirage.core.redis.write import write_bytes as _write
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
read_range=_read,
read_stream=_read_stream,
stat=_stat,
is_mounted=lambda a: a.store is not None,
python/mirage/commands/builtin/slack/io.py
+1
View File
@@ -28,6 +28,7 @@ from mirage.core.slack.stat import stat as _stat
IO = CommandIO(
readdir=_readdir,
read_bytes=_read,
read_range=_read,
read_stream=partial(stream_from_bytes, _read),
stat=_stat,
is_mounted=lambda a: True,
python/mirage/core/dify/read.py
+19 -2
View File
@@ -7,16 +7,33 @@ from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.core.dify._client import get_document_segments, iter_segment_pages
from mirage.core.dify.path import resolve_path
from mirage.types import PathSpec
from mirage.utils.ranges import slice_window
async def read_bytes(accessor: DifyAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX) -> bytes:
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None) -> bytes:
"""Read a document, optionally only a byte range of it.
A document is rendered here from its segments, so its bytes do not
exist until we make them and the window can only be taken
afterwards, the same way the rendered branches of gdrive, slack and
discord take theirs.
Args:
accessor (DifyAccessor): Dify accessor.
path (PathSpec): the path to read.
index (IndexCacheStore): listing cache, consulted for the entry.
offset (int): first byte to read.
size (int | None): how many bytes, or None for the rest.
"""
resolved = await resolve_path(accessor, path, index)
if resolved.is_dir:
raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
segments = await get_document_segments(accessor, resolved.entry.id)
return segments_to_bytes(segments)
return slice_window(segments_to_bytes(segments), offset, size)
async def read_stream(
python/mirage/core/discord/files.py
+5 -2
View File
@@ -37,7 +37,7 @@ def file_blob_name(att: dict[str, Any]) -> str:
return f"{path_safe_name(raw_name)}__{aid}"
async def download_file(url: str) -> bytes:
async def download_file(url: str, range_header: str | None = None) -> bytes:
"""Download a Discord-hosted file blob.
Discord CDN URLs (``cdn.discordapp.com`` for ``url``,
@@ -47,11 +47,14 @@ async def download_file(url: str) -> bytes:
Args:
url (str): Discord attachment URL (typically ``url`` from the
attachment object).
range_header (str | None): an HTTP ``Range`` value, or None for
the whole file.
Returns:
bytes: raw file content.
"""
headers = {"Range": range_header} if range_header else None
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
async with session.get(url, headers=headers) as resp:
resp.raise_for_status()
return await resp.read()
python/mirage/core/discord/read.py
+22 -4
View File
@@ -22,6 +22,7 @@ from mirage.core.discord.render import member_json_bytes
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
from mirage.utils.ranges import range_header, slice_window
async def _ensure_channel(
@@ -41,7 +42,23 @@ async def read(
accessor: DiscordAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None,
) -> bytes:
"""Read a Discord path, optionally only a byte range of it.
Only an attachment has a remote range to ask for. A channel's
history and a member profile are rendered here into JSON, so their
bytes do not exist until we make them and the window can only be
taken afterwards.
Args:
accessor (DiscordAccessor): Discord accessor.
path (PathSpec): the path to read.
index (IndexCacheStore): listing cache, consulted for the entry.
offset (int): first byte to read.
size (int | None): how many bytes, or None for the rest.
"""
virtual = path.virtual if isinstance(path, PathSpec) else path
prefix = mount_prefix_of(path.virtual, path.resource_path)
key = path.resource_path
@@ -52,8 +69,9 @@ async def read(
and parts[4] == "chat.jsonl"):
ch_key = f"{parts[0]}/{parts[1]}/{parts[2]}"
ch_lookup = await _ensure_channel(index, prefix, ch_key, virtual)
return await get_history_jsonl(accessor.config, ch_lookup.entry.id,
parts[3])
history = await get_history_jsonl(accessor.config, ch_lookup.entry.id,
parts[3])
return slice_window(history, offset, size)
# <guild>/channels/<ch>/<date>/files/<blob>
if (len(parts) == 6 and parts[1] == "channels" and parts[4] == "files"):
@@ -76,7 +94,7 @@ async def read(
or {}).get("proxy_url") or ""
if not url:
raise enoent(virtual)
return await download_file(url)
return await download_file(url, range_header(offset, size))
# <guild>/members/<user>.json
if len(parts) == 3 and parts[1] == "members":
@@ -92,7 +110,7 @@ async def read(
for m in members:
user = m.get("user", {})
if user.get("id") == entry_lookup.entry.id:
return member_json_bytes(m)
return slice_window(member_json_bytes(m), offset, size)
raise enoent(virtual)
raise enoent(virtual)
python/mirage/core/github_ci/__init__.py
-13
View File
@@ -1,13 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
python/mirage/core/github_ci/_client.py
-80
View File
@@ -1,80 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any
import aiohttp
from pydantic import SecretStr
from mirage.core.github._client import github_headers, github_url
async def ci_get(token: SecretStr,
path: str,
params: dict[str, Any] | None = None,
*,
base_url: str | None = None,
**kwargs: str) -> dict[str, Any]:
url = github_url(path, base_url, **kwargs)
headers = github_headers(token)
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers, params=params) as resp:
resp.raise_for_status()
return await resp.json()
async def ci_get_bytes(token: SecretStr,
path: str,
*,
base_url: str | None = None,
**kwargs: str) -> bytes:
url = github_url(path, base_url, **kwargs)
headers = github_headers(token)
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers,
allow_redirects=True) as resp:
resp.raise_for_status()
return await resp.read()
async def ci_get_paginated(token: SecretStr,
path: str,
list_key: str,
params: dict[str, Any] | None = None,
max_results: int | None = None,
*,
base_url: str | None = None,
**kwargs: str) -> list[dict[str, Any]]:
params = dict(params or {})
params.setdefault("per_page", 100)
page = 1
results: list[dict[str, Any]] = []
url = github_url(path, base_url, **kwargs)
headers = github_headers(token)
async with aiohttp.ClientSession() as session:
while True:
params["page"] = page
async with session.get(url, headers=headers,
params=params) as resp:
resp.raise_for_status()
data = await resp.json()
batch = data[list_key]
results.extend(batch)
if max_results is not None and len(results) >= max_results:
results = results[:max_results]
break
if len(batch) < params["per_page"]:
break
page += 1
return results
python/mirage/core/github_ci/annotations.py
-31
View File
@@ -1,31 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any, cast
from mirage.core.github_ci._client import ci_get
from mirage.resource.github_ci.config import GitHubCIConfig
async def list_annotations(config: GitHubCIConfig,
check_run_id: str) -> list[dict[str, Any]]:
return cast(
list[dict[str, Any]], await ci_get(
config.token,
"/repos/{owner}/{repo}/check-runs/{check_run_id}/annotations",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
check_run_id=check_run_id,
))
python/mirage/core/github_ci/artifacts.py
-42
View File
@@ -1,42 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any
from mirage.core.github_ci._client import ci_get_bytes, ci_get_paginated
from mirage.resource.github_ci.config import GitHubCIConfig
async def list_artifacts(config: GitHubCIConfig,
run_id: str) -> list[dict[str, Any]]:
return await ci_get_paginated(
config.token,
"/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts",
list_key="artifacts",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
run_id=run_id,
)
async def download_artifact(config: GitHubCIConfig, artifact_id: str) -> bytes:
return await ci_get_bytes(
config.token,
"/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
artifact_id=artifact_id,
)
python/mirage/core/github_ci/read.py
-104
View File
@@ -1,104 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import json
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.core.github_ci.annotations import list_annotations
from mirage.core.github_ci.artifacts import download_artifact
from mirage.core.github_ci.render import ci_json_bytes
from mirage.core.github_ci.runs import (download_job_log, get_job, get_run,
list_jobs_for_run)
from mirage.core.github_ci.workflows import get_workflow
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_prefix_of
async def read(
accessor: GitHubCIAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
) -> bytes:
virtual = path.virtual
prefix = mount_prefix_of(path.virtual, path.resource_path)
key = path.resource_path
parts = key.split("/")
# /workflows/<name>_<id>.json
if len(parts) == 2 and parts[0] == "workflows" and parts[1].endswith(
".json"):
virtual_key = prefix + "/" + key
lookup = await index.get(virtual_key)
if lookup.entry is None:
raise enoent(virtual)
wf = await get_workflow(accessor.config, lookup.entry.id)
return ci_json_bytes(wf)
# /runs/<workflow>_<run-id>/run.json
if (len(parts) == 3 and parts[0] == "runs" and parts[2] == "run.json"):
run_virtual = prefix + "/" + f"{parts[0]}/{parts[1]}"
lookup = await index.get(run_virtual)
if lookup.entry is None:
raise enoent(virtual)
run = await get_run(accessor.config, lookup.entry.id)
return ci_json_bytes(run)
# /runs/<workflow>_<run-id>/annotations.jsonl
if (len(parts) == 3 and parts[0] == "runs"
and parts[2] == "annotations.jsonl"):
run_virtual = prefix + "/" + f"{parts[0]}/{parts[1]}"
lookup = await index.get(run_virtual)
if lookup.entry is None:
raise enoent(virtual)
jobs = await list_jobs_for_run(accessor.config, lookup.entry.id)
lines = []
for j in jobs:
anns = await list_annotations(accessor.config, str(j["id"]))
for a in anns:
lines.append(
json.dumps(a, ensure_ascii=False, separators=(",", ":")))
if lines:
return ("\n".join(lines) + "\n").encode()
return b""
# /runs/<workflow>_<run-id>/jobs/<job>_<job-id>.json
if (len(parts) == 4 and parts[0] == "runs" and parts[2] == "jobs"
and parts[3].endswith(".json")):
virtual_key = prefix + "/" + key
lookup = await index.get(virtual_key)
if lookup.entry is None:
raise enoent(virtual)
job = await get_job(accessor.config, lookup.entry.id)
return ci_json_bytes(job)
# /runs/<workflow>_<run-id>/jobs/<job>_<job-id>.log
if (len(parts) == 4 and parts[0] == "runs" and parts[2] == "jobs"
and parts[3].endswith(".log")):
virtual_key = prefix + "/" + key
lookup = await index.get(virtual_key)
if lookup.entry is None:
raise enoent(virtual)
return await download_job_log(accessor.config, lookup.entry.id)
# /runs/<workflow>_<run-id>/artifacts/<name>_<id>.zip
if (len(parts) == 4 and parts[0] == "runs" and parts[2] == "artifacts"):
virtual_key = prefix + "/" + key
lookup = await index.get(virtual_key)
if lookup.entry is None:
raise enoent(virtual)
return await download_artifact(accessor.config, lookup.entry.id)
raise enoent(virtual)
python/mirage/core/github_ci/readdir.py
-256
View File
@@ -1,256 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore, IndexEntry
from mirage.core.github_ci.artifacts import list_artifacts
from mirage.core.github_ci.render import ci_json_bytes
from mirage.core.github_ci.runs import list_jobs_for_run, list_runs
from mirage.core.github_ci.workflows import list_workflows
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
def _safe_name(name: str) -> str:
if not name:
return "unknown"
return name.replace("/", "\u2215")
def is_cross_run_root(path: PathSpec) -> bool:
original = path.virtual if isinstance(path, PathSpec) else path
prefix = mount_prefix_of(path.virtual, path.resource_path) if isinstance(
path, PathSpec) else ""
if prefix and original.startswith(prefix):
rest = original[len(prefix):]
if prefix.endswith("/") or rest == "" or rest.startswith("/"):
original = rest or "/"
return original.strip("/") in ("", "runs")
async def readdir(
accessor: GitHubCIAccessor,
path_spec: PathSpec,
index: IndexCacheStore = NULL_INDEX,
) -> list[str]:
virtual = path_spec.virtual
prefix = mount_prefix_of(path_spec.virtual, path_spec.resource_path)
path = (path_spec.dir if path_spec.pattern else path_spec).mount_path
key = path.strip("/")
virtual_key = prefix + "/" + key if key else prefix or "/"
if not key:
return [f"{prefix}/workflows", f"{prefix}/runs"]
parts = key.split("/")
# /workflows
if len(parts) == 1 and parts[0] == "workflows":
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
return listing.entries
workflows = await list_workflows(accessor.config)
entries = []
names = []
for wf in workflows:
name = _safe_name(wf.get("name", str(wf["id"])))
filename = f"{name}_{wf['id']}.json"
entry = IndexEntry(
id=str(wf["id"]),
name=wf.get("name", ""),
resource_type="ci/workflow",
vfs_name=filename,
size=len(ci_json_bytes(wf)),
remote_time=wf.get("updated_at", ""),
)
entries.append((filename, entry))
names.append(f"{prefix}/{key}/{filename}")
await index.set_dir(virtual_key, entries)
return names
# /runs
if len(parts) == 1 and parts[0] == "runs":
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
return listing.entries
runs = await list_runs(accessor.config, days=accessor.config.days)
entries = []
names = []
for r in runs:
wf_name = _safe_name(r.get("name", str(r["id"])))
dirname = f"{wf_name}_{r['id']}"
entry = IndexEntry(
id=str(r["id"]),
name=r.get("name", ""),
resource_type="ci/run",
vfs_name=dirname,
remote_time=r.get("updated_at", ""),
)
entries.append((dirname, entry))
names.append(f"{prefix}/{key}/{dirname}")
# run.json renders the run object this listing already fetched,
# so its exact size is free here; annotations.jsonl has no length
# anywhere in the API and stays size-unknown.
await index.set_dir(f"{virtual_key}/{dirname}", [
("run.json",
IndexEntry(
id=str(r["id"]),
name="run.json",
resource_type="ci/run_json",
vfs_name="run.json",
size=len(ci_json_bytes(r)),
remote_time=r.get("updated_at", ""),
)),
("jobs",
IndexEntry(
id=str(r["id"]),
name="jobs",
resource_type="ci/jobs_dir",
vfs_name="jobs",
)),
("annotations.jsonl",
IndexEntry(
id=str(r["id"]),
name="annotations.jsonl",
resource_type="ci/annotations",
vfs_name="annotations.jsonl",
remote_time=r.get("updated_at", ""),
)),
("artifacts",
IndexEntry(
id=str(r["id"]),
name="artifacts",
resource_type="ci/artifacts_dir",
vfs_name="artifacts",
)),
])
await index.set_dir(virtual_key, entries)
return names
# /runs/<workflow>_<run-id>
if len(parts) == 2 and parts[0] == "runs":
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
return listing.entries
lookup = await index.get(virtual_key)
if lookup.entry is None:
parent = PathSpec(
virtual=prefix + "/runs",
directory=prefix + "/runs",
resource_path=mount_key(prefix + "/runs", prefix),
)
await readdir(accessor, parent, index)
lookup = await index.get(virtual_key)
if lookup.entry is None:
raise enoent(virtual)
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
return listing.entries
base = f"{prefix}/{key}"
return [
f"{base}/run.json",
f"{base}/jobs",
f"{base}/annotations.jsonl",
f"{base}/artifacts",
]
# /runs/<workflow>_<run-id>/jobs
if len(parts) == 3 and parts[0] == "runs" and parts[2] == "jobs":
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
return listing.entries
run_virtual = prefix + "/" + f"{parts[0]}/{parts[1]}"
run_lookup = await index.get(run_virtual)
if run_lookup.entry is None:
parent = PathSpec(
virtual=prefix + "/runs",
directory=prefix + "/runs",
resource_path=mount_key(prefix + "/runs", prefix),
)
await readdir(accessor, parent, index)
run_lookup = await index.get(run_virtual)
if run_lookup.entry is None:
raise enoent(virtual)
run_id = run_lookup.entry.id
jobs = await list_jobs_for_run(accessor.config, run_id)
entries = []
names = []
for j in jobs:
name = _safe_name(j.get("name", str(j["id"])))
json_filename = f"{name}_{j['id']}.json"
log_filename = f"{name}_{j['id']}.log"
entry_json = IndexEntry(
id=str(j["id"]),
name=j.get("name", ""),
resource_type="ci/job",
vfs_name=json_filename,
size=len(ci_json_bytes(j)),
remote_time=j.get("completed_at", ""),
)
entry_log = IndexEntry(
id=str(j["id"]),
name=j.get("name", ""),
resource_type="ci/job_log",
vfs_name=log_filename,
remote_time=j.get("completed_at", ""),
)
entries.append((json_filename, entry_json))
entries.append((log_filename, entry_log))
names.append(f"{prefix}/{key}/{json_filename}")
names.append(f"{prefix}/{key}/{log_filename}")
await index.set_dir(virtual_key, entries)
return names
# /runs/<workflow>_<run-id>/artifacts
if len(parts) == 3 and parts[0] == "runs" and parts[2] == "artifacts":
listing = await index.list_dir(virtual_key)
if listing.entries is not None:
return listing.entries
run_virtual = prefix + "/" + f"{parts[0]}/{parts[1]}"
run_lookup = await index.get(run_virtual)
if run_lookup.entry is None:
parent = PathSpec(
virtual=prefix + "/runs",
directory=prefix + "/runs",
resource_path=mount_key(prefix + "/runs", prefix),
)
await readdir(accessor, parent, index)
run_lookup = await index.get(run_virtual)
if run_lookup.entry is None:
raise enoent(virtual)
run_id = run_lookup.entry.id
artifacts = await list_artifacts(accessor.config, run_id)
entries = []
names = []
for a in artifacts:
name = _safe_name(a.get("name", str(a["id"])))
filename = f"{name}_{a['id']}.zip"
entry = IndexEntry(
id=str(a["id"]),
name=a.get("name", ""),
resource_type="ci/artifact",
vfs_name=filename,
# size_in_bytes is the zip archive's exact byte length
# (verified live against the GitHub API), not the
# uncompressed total.
size=a.get("size_in_bytes"),
remote_time=a.get("updated_at", ""),
)
entries.append((filename, entry))
names.append(f"{prefix}/{key}/{filename}")
await index.set_dir(virtual_key, entries)
return names
return []
python/mirage/core/github_ci/render.py
-23
View File
@@ -1,23 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import json
from typing import Any
def ci_json_bytes(obj: dict[str, Any]) -> bytes:
# Single renderer for workflow/run/job JSON files: read() and the
# readdir-time sizing must produce the same bytes for the same payload,
# so the advertised size is exact by construction.
return json.dumps(obj, indent=2, ensure_ascii=False).encode()
python/mirage/core/github_ci/runs.py
-82
View File
@@ -1,82 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from datetime import datetime, timedelta, timezone
from typing import Any
from mirage.core.github_ci._client import (ci_get, ci_get_bytes,
ci_get_paginated)
from mirage.resource.github_ci.config import GitHubCIConfig
async def list_runs(config: GitHubCIConfig,
days: int = 30) -> list[dict[str, Any]]:
since = (datetime.now(timezone.utc) -
timedelta(days=days)).strftime("%Y-%m-%d")
return await ci_get_paginated(
config.token,
"/repos/{owner}/{repo}/actions/runs",
list_key="workflow_runs",
params={"created": f">={since}"},
max_results=config.max_runs,
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
)
async def get_run(config: GitHubCIConfig, run_id: str) -> dict[str, Any]:
return await ci_get(
config.token,
"/repos/{owner}/{repo}/actions/runs/{run_id}",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
run_id=run_id,
)
async def list_jobs_for_run(config: GitHubCIConfig,
run_id: str) -> list[dict[str, Any]]:
return await ci_get_paginated(
config.token,
"/repos/{owner}/{repo}/actions/runs/{run_id}/jobs",
list_key="jobs",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
run_id=run_id,
)
async def get_job(config: GitHubCIConfig, job_id: str) -> dict[str, Any]:
return await ci_get(
config.token,
"/repos/{owner}/{repo}/actions/jobs/{job_id}",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
job_id=job_id,
)
async def download_job_log(config: GitHubCIConfig, job_id: str) -> bytes:
return await ci_get_bytes(
config.token,
"/repos/{owner}/{repo}/actions/jobs/{job_id}/logs",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
job_id=job_id,
)
python/mirage/core/github_ci/stat.py
-165
View File
@@ -1,165 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import logging
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.core.github_ci.readdir import readdir as _readdir
from mirage.types import FileStat, FileType, PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
logger = logging.getLogger(__name__)
VIRTUAL_DIRS = {"workflows", "runs", "jobs", "artifacts"}
async def _lookup_with_fallback(
accessor: GitHubCIAccessor,
virtual_key: str,
prefix: str,
index: IndexCacheStore = NULL_INDEX,
):
result = await index.get(virtual_key)
if result.entry is not None:
return result
parent_virtual = virtual_key.rsplit("/", 1)[0] or "/"
try:
await _readdir(
accessor,
PathSpec(virtual=parent_virtual,
directory=parent_virtual,
resource_path=mount_key(parent_virtual, prefix)),
index=index,
)
except FileNotFoundError as exc:
logger.debug("stat populate failed for %s: %s", virtual_key, exc)
return await index.get(virtual_key)
async def stat(
accessor: GitHubCIAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
) -> FileStat:
virtual = path.virtual
prefix = mount_prefix_of(path.virtual, path.resource_path)
key = path.resource_path
if not key:
return FileStat(name="/", type=FileType.DIRECTORY)
parts = key.split("/")
virtual_key = prefix + "/" + key
if len(parts) == 1 and parts[0] in VIRTUAL_DIRS:
return FileStat(name=parts[0], type=FileType.DIRECTORY)
if len(parts) == 2 and parts[0] == "workflows" and parts[1].endswith(
".json"):
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name=lookup.entry.vfs_name or lookup.entry.name,
type=FileType.JSON,
size=lookup.entry.size,
modified=lookup.entry.remote_time or None,
extra={"workflow_id": lookup.entry.id},
)
if len(parts) == 2 and parts[0] == "runs":
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name=lookup.entry.vfs_name or lookup.entry.name,
type=FileType.DIRECTORY,
modified=lookup.entry.remote_time or None,
extra={"run_id": lookup.entry.id},
)
if len(parts) == 3 and parts[0] == "runs" and parts[2] in VIRTUAL_DIRS:
return FileStat(name=parts[2], type=FileType.DIRECTORY)
if len(parts) == 3 and parts[0] == "runs" and parts[2] == "run.json":
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name="run.json",
type=FileType.JSON,
size=lookup.entry.size,
modified=lookup.entry.remote_time or None,
extra={"run_id": lookup.entry.id},
)
if (len(parts) == 3 and parts[0] == "runs"
and parts[2] == "annotations.jsonl"):
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name="annotations.jsonl",
type=FileType.TEXT,
modified=lookup.entry.remote_time or None,
extra={"run_id": lookup.entry.id},
)
if (len(parts) == 4 and parts[0] == "runs" and parts[2] == "jobs"
and parts[3].endswith(".json")):
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name=lookup.entry.vfs_name or lookup.entry.name,
type=FileType.JSON,
size=lookup.entry.size,
modified=lookup.entry.remote_time or None,
extra={"job_id": lookup.entry.id},
)
if (len(parts) == 4 and parts[0] == "runs" and parts[2] == "jobs"
and parts[3].endswith(".log")):
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name=lookup.entry.vfs_name or lookup.entry.name,
type=FileType.TEXT,
modified=lookup.entry.remote_time or None,
extra={"job_id": lookup.entry.id},
)
if (len(parts) == 4 and parts[0] == "runs" and parts[2] == "artifacts"):
lookup = await _lookup_with_fallback(accessor, virtual_key, prefix,
index)
if lookup.entry is None:
raise enoent(virtual)
return FileStat(
name=lookup.entry.vfs_name or lookup.entry.name,
type=FileType.ZIP,
size=lookup.entry.size,
modified=lookup.entry.remote_time or None,
extra={"artifact_id": lookup.entry.id},
)
raise enoent(virtual)
python/mirage/core/github_ci/workflows.py
-41
View File
@@ -1,41 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any
from mirage.core.github_ci._client import ci_get, ci_get_paginated
from mirage.resource.github_ci.config import GitHubCIConfig
async def list_workflows(config: GitHubCIConfig) -> list[dict[str, Any]]:
return await ci_get_paginated(
config.token,
"/repos/{owner}/{repo}/actions/workflows",
list_key="workflows",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
)
async def get_workflow(config: GitHubCIConfig,
workflow_id: str) -> dict[str, Any]:
return await ci_get(
config.token,
"/repos/{owner}/{repo}/actions/workflows/{workflow_id}",
base_url=config.base_url,
owner=config.owner,
repo=config.repo,
workflow_id=workflow_id,
)
python/mirage/core/ram/read.py
+25 -3
View File
@@ -20,9 +20,27 @@ from mirage.observe.context import record
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.path import norm
from mirage.utils.ranges import slice_window
async def read_bytes(accessor: RAMAccessor, path_spec: PathSpec) -> bytes:
async def read_bytes(accessor: RAMAccessor,
path_spec: PathSpec,
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None) -> bytes:
"""Read a file, optionally only a byte range of it.
The bytes are already in memory, so the window is a slice rather
than a smaller fetch. It is taken here anyway so a RAM mount answers
a windowed read the same way every other backend does.
Args:
accessor (RAMAccessor): RAM accessor.
path_spec (PathSpec): the path to read.
index (IndexCacheStore): unused; the store is the listing.
offset (int): first byte to read.
size (int | None): how many bytes, or None for the rest.
"""
virtual = path_spec.virtual
path = path_spec.mount_path
store = accessor.store
@@ -31,14 +49,18 @@ async def read_bytes(accessor: RAMAccessor, path_spec: PathSpec) -> bytes:
if key not in store.files:
raise enoent(virtual)
data = store.files[key]
if offset or size is not None:
data = slice_window(data, offset, size)
record("read", path, "ram", len(data), start_ms)
return data
async def read(accessor: RAMAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX) -> bytes:
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None) -> bytes:
try:
return await read_bytes(accessor, path)
return await read_bytes(accessor, path, index, offset, size)
except FileNotFoundError as exc:
raise enoent(path.virtual) from exc
python/mirage/core/redis/read.py
+21 -3
View File
@@ -22,13 +22,29 @@ from mirage.utils.errors import enoent
from mirage.utils.path import norm
async def read_bytes(accessor: RedisAccessor, path_spec: PathSpec) -> bytes:
async def read_bytes(accessor: RedisAccessor,
path_spec: PathSpec,
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None) -> bytes:
"""Read a file, optionally only a byte range of it.
Args:
accessor (RedisAccessor): Redis accessor.
path_spec (PathSpec): the path to read.
index (IndexCacheStore): unused; the key space is the listing.
offset (int): first byte to read.
size (int | None): how many bytes, or None for the rest.
"""
virtual = path_spec.virtual
path = path_spec.mount_path
store = accessor.store
start_ms = int(time.monotonic() * 1000)
key = norm(path)
data = await store.get_file(key)
if offset or size is not None:
data = await store.get_file_range(key, offset, size)
else:
data = await store.get_file(key)
if data is None:
raise enoent(virtual)
record("read", path, "redis", len(data), start_ms)
@@ -39,8 +55,10 @@ async def read(
accessor: RedisAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None,
) -> bytes:
try:
return await read_bytes(accessor, path)
return await read_bytes(accessor, path, index, offset, size)
except FileNotFoundError as exc:
raise enoent(path.virtual) from exc
python/mirage/core/slack/files.py
+8 -2
View File
@@ -39,17 +39,23 @@ def file_blob_name(file_meta: dict[str, Any]) -> str:
return f"{path_safe_name(raw_name)}__{fid}"
async def download_file(config: SlackConfig, url: str) -> bytes:
"""Download a Slack-hosted file blob.
async def download_file(config: SlackConfig,
url: str,
range_header: str | None = None) -> bytes:
"""Download a Slack-hosted file blob, optionally only a byte range.
Args:
config (SlackConfig): Slack credentials.
url (str): Slack file URL (typically url_private_download).
range_header (str | None): an HTTP ``Range`` value, or None for
the whole file.
Returns:
bytes: raw file content.
"""
headers = {"Authorization": f"Bearer {reveal_secret(config.token)}"}
if range_header:
headers["Range"] = range_header
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers) as resp:
resp.raise_for_status()
python/mirage/core/slack/read.py
+23 -3
View File
@@ -20,13 +20,30 @@ from mirage.core.slack.users import get_user_profile, user_json_bytes
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_prefix_of
from mirage.utils.ranges import range_header, slice_window
async def read(
accessor: SlackAccessor,
path: PathSpec,
index: IndexCacheStore = NULL_INDEX,
offset: int = 0,
size: int | None = None,
) -> bytes:
"""Read a Slack path, optionally only a byte range of it.
Only an uploaded file has a remote range to ask for. A channel's
history and a user profile are rendered here into JSON, so their
bytes do not exist until we make them and the window can only be
taken afterwards.
Args:
accessor (SlackAccessor): Slack accessor.
path (PathSpec): the path to read.
index (IndexCacheStore): listing cache, consulted for the entry.
offset (int): first byte to read.
size (int | None): how many bytes, or None for the rest.
"""
virtual = path.virtual
prefix = mount_prefix_of(path.virtual, path.resource_path) if isinstance(
path, PathSpec) else ""
@@ -45,7 +62,9 @@ async def read(
raise enoent(virtual)
channel_id = lookup.entry.id
date_str = parts[2]
return await get_history_jsonl(accessor.config, channel_id, date_str)
history = await get_history_jsonl(accessor.config, channel_id,
date_str)
return slice_window(history, offset, size)
if (len(parts) == 5 and parts[0] in ("channels", "dms")
and parts[3] == "files"):
@@ -56,7 +75,8 @@ async def read(
url = lookup.entry.extra.get("url_private_download")
if not url:
raise enoent(virtual)
return await slack_files.download_file(accessor.config, url)
return await slack_files.download_file(accessor.config, url,
range_header(offset, size))
if len(parts) == 2 and parts[0] == "users":
virtual_key = prefix + "/" + key
@@ -64,6 +84,6 @@ async def read(
if lookup.entry is None:
raise enoent(virtual)
user = await get_user_profile(accessor.config, lookup.entry.id)
return user_json_bytes(user)
return slice_window(user_json_bytes(user), offset, size)
raise enoent(virtual)
python/mirage/ops/github_ci/__init__.py
-18
View File
@@ -1,18 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.github_ci.io import IO
from mirage.ops.generic import make_generic_ops
OPS = make_generic_ops("github_ci", IO)
python/mirage/resource/dify/dify.py
+1 -1
View File
@@ -27,7 +27,7 @@ class DifyResource(BaseResource):
accessor: DifyAccessor
name: str = ResourceName.DIFY
caches_reads: bool = True
_ops = _DIFY_OPS
_ops: dict[str, Any] = _DIFY_OPS
PROMPT: str = PROMPT
SUPPORTS_SNAPSHOT: bool = False
python/mirage/resource/github_ci/__init__.py
-24
View File
@@ -1,24 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.resource.github_ci.config import GitHubCIConfig
__all__ = ["GitHubCIConfig", "GitHubCIResource"]
def __getattr__(name: str):
if name == "GitHubCIResource":
from mirage.resource.github_ci.github_ci import GitHubCIResource
return GitHubCIResource
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
python/mirage/resource/github_ci/config.py
-24
View File
@@ -1,24 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from pydantic import BaseModel, SecretStr
class GitHubCIConfig(BaseModel):
token: SecretStr
owner: str
repo: str
days: int = 30
max_runs: int = 300
base_url: str | None = None
python/mirage/resource/github_ci/github_ci.py
-58
View File
@@ -1,58 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.core.github_ci.readdir import readdir
from mirage.resource.base import BaseResource
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.resource.github_ci.prompt import PROMPT
from mirage.types import ResourceName
from mirage.utils.glob_walk import make_resolve_glob
_resolve_glob = make_resolve_glob(readdir)
class GitHubCIResource(BaseResource):
accessor: GitHubCIAccessor
name: str = ResourceName.GITHUB_CI
caches_reads: bool = True
# An API-backed tree that changes rarely; a day-long index spares the
# provider a full re-walk every 10 minutes. Mirrors the TypeScript
# resource.
index_ttl: float = 86_400
PROMPT: str = PROMPT
def __init__(self, config: GitHubCIConfig) -> None:
super().__init__()
self.config = config
self.accessor = GitHubCIAccessor(self.config)
from mirage.commands.builtin.github_ci import COMMANDS
from mirage.ops.github_ci import OPS
for fn in COMMANDS:
self.register(fn)
for fn in OPS:
self.register_op(fn)
async def resolve_glob(self, paths, prefix: str = ""):
return await _resolve_glob(self.accessor, paths, index=self._index)
def get_state(self) -> dict[str, Any]:
return self.config_state(self.config)
def load_state(self, state: dict[str, Any]) -> None:
pass
python/mirage/resource/github_ci/prompt.py
-27
View File
@@ -1,27 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
PROMPT = """\
{prefix}
workflows/
<workflow-name>_<workflow-id>.json
runs/
<workflow-name>_<run-id>/
run.json
jobs/
<job-name>_<job-id>.json
<job-name>_<job-id>.log
annotations.jsonl
artifacts/
<artifact-name>_<artifact-id>.zip"""
python/mirage/resource/redis/store.py
+29
View File
@@ -79,6 +79,35 @@ class RedisStore:
return await cast("Awaitable[bytes | None]",
self._client.get(self._fk(path)))
async def get_file_range(self, path: str, offset: int,
size: int | None) -> bytes | None:
"""A byte window of a stored file, or None when the key is absent.
``GETRANGE`` slices server-side, so a window costs the window
rather than the whole value on the wire. Its bounds are
inclusive and ``-1`` means the last byte, which is how "to the
end" is spelled.
``EXISTS`` rides along in the same pipeline because ``GETRANGE``
answers an empty string for a missing key, for an empty file and
for a window past the end alike; without it a read of a deleted
path would return b"" instead of raising.
Args:
path (str): mount-relative path of the file.
offset (int): first byte to read.
size (int | None): how many bytes, or None for the rest.
"""
key = self._fk(path)
end = -1 if size is None else offset + size - 1
async with self._client.pipeline(transaction=False) as pipe:
pipe.exists(key)
pipe.getrange(key, offset, end)
exists, data = await pipe.execute()
if not exists:
return None
return data if isinstance(data, bytes) else str(data).encode()
async def set_file(self, path: str, data: bytes) -> None:
await self._client.set(self._fk(path), data)
python/mirage/resource/registry.py
-3
View File
@@ -114,9 +114,6 @@ REGISTRY: dict[str, ResourceEntry] = {
"github":
ResourceEntry("mirage.resource.github:GitHubResource",
"mirage.resource.github:GitHubConfig"),
"github_ci":
ResourceEntry("mirage.resource.github_ci:GitHubCIResource",
"mirage.resource.github_ci:GitHubCIConfig"),
"linear":
ResourceEntry("mirage.resource.linear:LinearResource",
"mirage.resource.linear:LinearConfig"),
python/mirage/types.py
-1
View File
@@ -435,7 +435,6 @@ class ResourceName(str, Enum):
JAEGER = "jaeger"
SSH = "ssh"
REDIS = "redis"
GITHUB_CI = "github_ci"
GCS = "gcs"
EMAIL = "email"
DIFY = "dify"
python/mirage/utils/ranges.py
+9 -1
View File
@@ -102,6 +102,11 @@ def is_unsatisfiable_range(exc: BaseException) -> bool:
predicate lives here so the ops factory can turn all of them into the
empty read the caller expects, rather than each backend re-deciding.
A reader that seeks rather than sending a header (OpenDAL's file
object, which hf and nextcloud both open) raises an OSError from the
seek itself instead of surfacing a status. It is the same condition,
so it is matched here rather than guarded in each of those backends.
Args:
exc (BaseException): whatever the backend reader raised.
"""
@@ -109,4 +114,7 @@ def is_unsatisfiable_range(exc: BaseException) -> bool:
return True
if _code_of(exc) in ("InvalidRange", "RequestedRangeNotSatisfiable"):
return True
return "range not satisfiable" in str(exc).lower()
text = str(exc).lower()
if "range not satisfiable" in text:
return True
return "seek" in text and "beyond the end" in text
python/tests/commands/builtin/github_ci/test_find.py
-136
View File
@@ -1,136 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import AsyncMock, patch
import pytest
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.commands.builtin.github_ci.find import find
from mirage.commands.config import CommandOpts
from mirage.commands.errors import FindParseError
from mirage.io.stream import materialize
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.types import FileStat, FileType, PathSpec
from mirage.utils.key_prefix import mount_key
@pytest.fixture
def accessor():
return GitHubCIAccessor(
config=GitHubCIConfig(token="t", owner="o", repo="r"))
@pytest.fixture
def index():
return RAMIndexCacheStore()
def _scope(path: str, prefix: str = "") -> PathSpec:
return PathSpec(resource_path=mount_key(path, prefix),
virtual=path,
directory=path)
@pytest.mark.asyncio
async def test_find_runs_root_rejected(accessor, index):
with pytest.raises(ValueError, match="across runs is disabled"):
await find(accessor, [_scope('/runs')], [],
CommandOpts(index=index, flags={'name': '*.log'}))
@pytest.mark.asyncio
async def test_find_root_rejected(accessor, index):
with pytest.raises(ValueError, match="across runs is disabled"):
await find(accessor, [], [],
CommandOpts(index=index, flags={'name': '*.log'}))
@pytest.mark.asyncio
async def test_find_invalid_maxdepth_raises_find_parse_error(accessor, index):
with pytest.raises(FindParseError,
match=r"invalid argument 'abc' to '-maxdepth'"):
await find(accessor, [_scope('/runs/wf_1')], [],
CommandOpts(index=index, flags={'maxdepth': 'abc'}))
@pytest.mark.asyncio
async def test_find_single_run_allowed(accessor, index):
async def fake_readdir(_acc, p, index=None):
if p.virtual == "/runs/wf_1":
return ["/runs/wf_1/run.json", "/runs/wf_1/jobs"]
if p.virtual == "/runs/wf_1/jobs":
return ["/runs/wf_1/jobs/build_1.log"]
return []
with patch("mirage.commands.builtin.github_ci.find._readdir",
new=AsyncMock(side_effect=fake_readdir)), \
patch("mirage.core.github_ci.stat._readdir",
new=AsyncMock(side_effect=fake_readdir)):
out, io = await find(accessor, [_scope('/runs/wf_1')], [],
CommandOpts(index=index, flags={'name': '*.log'}))
data = await materialize(out)
assert b"/runs/wf_1/jobs/build_1.log" in data
@pytest.mark.asyncio
async def test_find_path_pattern_is_honored(accessor, index):
async def fake_readdir(_acc, p, index=None):
if p.virtual == "/runs/wf_1":
return ["/runs/wf_1/run.json", "/runs/wf_1/jobs"]
if p.virtual == "/runs/wf_1/jobs":
return ["/runs/wf_1/jobs/build_1.log"]
return []
with patch("mirage.commands.builtin.github_ci.find._readdir",
new=AsyncMock(side_effect=fake_readdir)), \
patch("mirage.core.github_ci.stat._readdir",
new=AsyncMock(side_effect=fake_readdir)):
out, _io = await find(
accessor, [_scope('/runs/wf_1')], [],
CommandOpts(index=index, flags={'path': '*jobs*'}))
data = await materialize(out)
assert data.decode().splitlines() == [
"/runs/wf_1/jobs", "/runs/wf_1/jobs/build_1.log"
]
@pytest.mark.asyncio
async def test_find_size_counts_sizeless_entries_as_zero(accessor, index):
async def fake_readdir(_acc, p, index=None):
if p.virtual == "/runs/wf_1":
return ["/runs/wf_1/run.json", "/runs/wf_1/jobs"]
if p.virtual == "/runs/wf_1/jobs":
return ["/runs/wf_1/jobs/build_1.log"]
return []
async def fake_stat(_acc, p, _idx=None):
virtual = p.virtual if isinstance(p, PathSpec) else p
name = virtual.rsplit("/", 1)[-1]
if "." in name:
return FileStat(name=name, type=FileType.TEXT, size=None)
return FileStat(name=name, type=FileType.DIRECTORY)
with patch("mirage.commands.builtin.github_ci.find._readdir",
new=AsyncMock(side_effect=fake_readdir)), \
patch("mirage.commands.builtin.github_ci.find._stat",
new=AsyncMock(side_effect=fake_stat)):
out, _io = await find(accessor, [_scope('/runs/wf_1')], [],
CommandOpts(index=index, flags={'size': '+0c'}))
data = await materialize(out)
assert data == b""
python/tests/commands/builtin/github_ci/test_grep.py
-126
View File
@@ -1,126 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import AsyncMock, patch
import pytest
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.commands.builtin.github_ci.grep import grep
from mirage.commands.config import CommandOpts
from mirage.io.stream import materialize
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.types import FileStat, FileType, PathSpec
from mirage.utils.key_prefix import mount_key
@pytest.fixture
def accessor():
return GitHubCIAccessor(
config=GitHubCIConfig(token="t", owner="o", repo="r"))
@pytest.fixture
def index():
return RAMIndexCacheStore()
def _scope(path: str, prefix: str = "") -> PathSpec:
return PathSpec(resource_path=mount_key(path, prefix),
virtual=path,
directory=path)
@pytest.mark.asyncio
async def test_grep_single_file_match(accessor, index):
file_stat = FileStat(name="run.json", type=FileType.JSON)
with (
patch("mirage.commands.builtin.github_ci.grep._stat",
new=AsyncMock(return_value=file_stat)),
patch("mirage.commands.builtin.github_ci.grep.ci_read",
new=AsyncMock(return_value=b"hello world\nbye world\n")),
):
out, io = await grep(accessor, [_scope('/runs/wf_1/run.json')],
['hello'], CommandOpts(index=index))
data = await materialize(out)
assert b"hello world" in data
assert io.exit_code == 0
@pytest.mark.asyncio
async def test_grep_no_match_exit_one(accessor, index):
file_stat = FileStat(name="run.json", type=FileType.JSON)
with (
patch("mirage.commands.builtin.github_ci.grep._stat",
new=AsyncMock(return_value=file_stat)),
patch("mirage.commands.builtin.github_ci.grep.ci_read",
new=AsyncMock(return_value=b"abc\ndef\n")),
):
out, io = await grep(accessor, [_scope('/runs/wf_1/run.json')],
['missing'], CommandOpts(index=index))
await materialize(out)
assert io.exit_code == 1
@pytest.mark.asyncio
async def test_grep_recursive_directory(accessor, index):
dir_stat = FileStat(name="workflows", type=FileType.DIRECTORY)
file_stat = FileStat(name="ci.json", type=FileType.JSON)
async def fake_stat(_acc, p, index=None):
if p.virtual.endswith(".json"):
return file_stat
return dir_stat
async def fake_readdir(_acc, p, index=None):
if p.virtual == "/workflows":
return ["/workflows/ci_1.json", "/workflows/build_2.json"]
return []
async def fake_read(_acc, p, index=None):
if "ci_1" in p.virtual:
return b"name: Test\non: push\n"
return b"name: Build\non: push\n"
with (
patch("mirage.commands.builtin.github_ci.grep._stat",
new=AsyncMock(side_effect=fake_stat)),
patch("mirage.commands.builtin.github_ci.grep._readdir",
new=AsyncMock(side_effect=fake_readdir)),
patch("mirage.commands.builtin.github_ci.grep.ci_read",
new=AsyncMock(side_effect=fake_read)),
):
out, io = await grep(accessor, [_scope('/workflows')], ['Test'],
CommandOpts(index=index, flags={'r': True}))
data = await materialize(out)
assert b"name: Test" in data
assert io.exit_code == 0
@pytest.mark.asyncio
async def test_grep_recursive_runs_rejected(accessor, index):
with pytest.raises(ValueError, match="across runs is disabled"):
await grep(accessor, [_scope('/runs')], ['x'],
CommandOpts(index=index, flags={'r': True}))
@pytest.mark.asyncio
async def test_grep_stdin(accessor, index):
out, io = await grep(
accessor, [], ['two'],
CommandOpts(stdin=b'line one\nline two\nline three\n', index=index))
data = await materialize(out)
assert b"line two" in data
assert io.exit_code == 0
python/tests/commands/builtin/github_ci/test_rg.py
-125
View File
@@ -1,125 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import AsyncMock, patch
import pytest
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.commands.builtin.github_ci.rg import rg
from mirage.commands.config import CommandOpts
from mirage.io.stream import materialize
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.types import FileStat, FileType, PathSpec
from mirage.utils.key_prefix import mount_key
@pytest.fixture
def accessor():
return GitHubCIAccessor(
config=GitHubCIConfig(token="t", owner="o", repo="r"))
@pytest.fixture
def index():
return RAMIndexCacheStore()
def _scope(path: str, prefix: str = "") -> PathSpec:
return PathSpec(resource_path=mount_key(path, prefix),
virtual=path,
directory=path)
@pytest.mark.asyncio
async def test_rg_single_file_match(accessor, index):
file_stat = FileStat(name="run.json", type=FileType.JSON)
with (
patch("mirage.commands.builtin.github_ci.rg._stat",
new=AsyncMock(return_value=file_stat)),
patch("mirage.commands.builtin.github_ci.rg.ci_read",
new=AsyncMock(return_value=b"alpha\nbeta\ngamma\n")),
):
out, io = await rg(accessor, [_scope('/runs/wf_1/run.json')], ['beta'],
CommandOpts(index=index))
data = await materialize(out)
assert b"beta" in data
assert io.exit_code == 0
@pytest.mark.asyncio
async def test_rg_no_match_exit_one(accessor, index):
file_stat = FileStat(name="run.json", type=FileType.JSON)
with (
patch("mirage.commands.builtin.github_ci.rg._stat",
new=AsyncMock(return_value=file_stat)),
patch("mirage.commands.builtin.github_ci.rg.ci_read",
new=AsyncMock(return_value=b"abc\ndef\n")),
):
out, io = await rg(accessor, [_scope('/runs/wf_1/run.json')],
['missing'], CommandOpts(index=index))
await materialize(out)
assert io.exit_code == 1
@pytest.mark.asyncio
async def test_rg_directory_implicit_recursive(accessor, index):
dir_stat = FileStat(name="workflows", type=FileType.DIRECTORY)
file_stat = FileStat(name="ci.json", type=FileType.JSON)
async def fake_stat(_acc, p, index=None):
if p.virtual.endswith(".json"):
return file_stat
return dir_stat
async def fake_readdir(_acc, p, index=None):
if p.virtual == "/workflows":
return ["/workflows/ci_1.json", "/workflows/build_2.json"]
return []
async def fake_read(_acc, p, index=None):
if "ci_1" in p.virtual:
return b"name: Test\non: push\n"
return b"name: Build\non: push\n"
with (
patch("mirage.commands.builtin.github_ci.rg._stat",
new=AsyncMock(side_effect=fake_stat)),
patch("mirage.commands.builtin.github_ci.rg._readdir",
new=AsyncMock(side_effect=fake_readdir)),
patch("mirage.commands.builtin.github_ci.rg.ci_read",
new=AsyncMock(side_effect=fake_read)),
):
out, io = await rg(accessor, [_scope('/workflows')], ['Test'],
CommandOpts(index=index))
data = await materialize(out)
assert b"name: Test" in data
assert io.exit_code == 0
@pytest.mark.asyncio
async def test_rg_runs_root_rejected(accessor, index):
with pytest.raises(ValueError, match="across runs is disabled"):
await rg(accessor, [_scope('/runs')], ['x'], CommandOpts(index=index))
@pytest.mark.asyncio
async def test_rg_stdin(accessor, index):
out, io = await rg(
accessor, [], ['two'],
CommandOpts(stdin=b'line one\nline two\nline three\n', index=index))
data = await materialize(out)
assert b"line two" in data
assert io.exit_code == 0
python/tests/commands/test_links_optin.py
+2 -3
View File
@@ -30,9 +30,8 @@ LINK_AWARE = ("ls", "stat", "find", "du", "file")
# The two spellings of delegation: the family's full-command generic
# entry, or find's walk primitives for the wrappers with custom guards
# (email pushes a folder-level -name down to IMAP search, github_ci
# refuses cross-run walks) that still route filtering through the
# shared walk.
# (email pushes a folder-level -name down to IMAP search) that still
# route filtering through the shared walk.
GENERIC_CALLS = {
"ls": ("ls_generic(", ),
"stat": ("stat_generic(", "generic_stat("),
python/tests/core/discord/test_read.py
+23
View File
@@ -76,3 +76,26 @@ async def test_read_not_found(accessor, index):
PathSpec(resource_path="no/such/path",
virtual="/no/such/path",
directory="/no/such/path"), index)
@pytest.mark.asyncio
async def test_read_jsonl_window_is_sliced_locally(accessor, index):
"""A rendered branch has no remote range, so the window is taken after."""
fake_data = b'{"id":"100","content":"hello"}\n'
with patch(
"mirage.core.discord.read.get_history_jsonl",
new_callable=AsyncMock,
return_value=fake_data,
):
path = "/My Server/channels/general/2024-01-15/chat.jsonl"
result = await read(
accessor,
PathSpec(virtual=path,
directory=path,
resource_path=path.strip("/")),
index,
offset=1,
size=4,
)
assert result == b'"id"'
python/tests/core/github_ci/test_readdir.py
-94
View File
@@ -1,94 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import AsyncMock, patch
import pytest
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.core.github_ci.readdir import readdir
from mirage.core.github_ci.render import ci_json_bytes
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.types import PathSpec
@pytest.fixture
def accessor():
return GitHubCIAccessor(GitHubCIConfig(token="t", owner="o", repo="r"))
@pytest.fixture
def index():
return RAMIndexCacheStore()
def _spec(original: str) -> PathSpec:
return PathSpec(virtual=original,
directory=original,
resource_path=original.strip("/"))
@pytest.mark.asyncio
async def test_readdir_workflows_stores_rendered_size(accessor, index):
wf = {"id": 7, "name": "CI", "state": "active", "updated_at": "2026-01-01"}
with patch("mirage.core.github_ci.readdir.list_workflows",
new_callable=AsyncMock,
return_value=[wf]):
await readdir(accessor, _spec("/workflows"), index)
lookup = await index.get("/workflows/CI_7.json")
assert lookup.entry is not None
assert lookup.entry.size == len(ci_json_bytes(wf))
@pytest.mark.asyncio
async def test_readdir_runs_seeds_run_dir_with_sized_run_json(accessor, index):
run = {"id": 11, "name": "CI", "status": "completed", "updated_at": "u"}
with patch("mirage.core.github_ci.readdir.list_runs",
new_callable=AsyncMock,
return_value=[run]):
await readdir(accessor, _spec("/runs"), index)
listing = await index.list_dir("/runs/CI_11")
assert listing.entries == [
"/runs/CI_11/run.json",
"/runs/CI_11/jobs",
"/runs/CI_11/annotations.jsonl",
"/runs/CI_11/artifacts",
]
lookup = await index.get("/runs/CI_11/run.json")
assert lookup.entry is not None
assert lookup.entry.size == len(ci_json_bytes(run))
annotations = await index.get("/runs/CI_11/annotations.jsonl")
assert annotations.entry is not None
assert annotations.entry.size is None
@pytest.mark.asyncio
async def test_readdir_jobs_stores_rendered_json_size_only(accessor, index):
run = {"id": 11, "name": "CI", "updated_at": "u"}
job = {"id": 21, "name": "build", "completed_at": "c", "steps": []}
with patch("mirage.core.github_ci.readdir.list_runs",
new_callable=AsyncMock,
return_value=[run]):
await readdir(accessor, _spec("/runs"), index)
with patch("mirage.core.github_ci.readdir.list_jobs_for_run",
new_callable=AsyncMock,
return_value=[job]):
await readdir(accessor, _spec("/runs/CI_11/jobs"), index)
json_lookup = await index.get("/runs/CI_11/jobs/build_21.json")
assert json_lookup.entry is not None
assert json_lookup.entry.size == len(ci_json_bytes(job))
log_lookup = await index.get("/runs/CI_11/jobs/build_21.log")
assert log_lookup.entry is not None
assert log_lookup.entry.size is None
python/tests/core/github_ci/test_runs_cap.py
-174
View File
@@ -1,174 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any
import pytest
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.commands.builtin.github_ci import COMMANDS
from mirage.commands.config import CommandOpts
from mirage.core.github_ci import _client as ci_client
from mirage.core.github_ci.runs import list_runs
from mirage.io.stream import materialize
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.types import PathSpec
class _MockResponse:
def __init__(self, data: Any):
self._data = data
def raise_for_status(self) -> None:
return None
async def json(self) -> Any:
return self._data
async def __aenter__(self) -> "_MockResponse":
return self
async def __aexit__(self, *_a: object) -> None:
return None
class _MockSession:
def __init__(self, list_key: str, total: int, per_page: int = 100):
self.list_key = list_key
self.total = total
self.per_page = per_page
self.calls: list[dict[str, str]] = []
def get(self,
_url: str,
headers: Any = None,
params: Any = None) -> _MockResponse:
params = dict(params or {})
self.calls.append(params)
page = int(params.get("page", 1))
per_page = int(params.get("per_page", self.per_page))
start = (page - 1) * per_page
end = min(start + per_page, self.total)
batch = [{"id": i, "name": f"item-{i}"} for i in range(start, end)]
return _MockResponse({self.list_key: batch})
async def __aenter__(self) -> "_MockSession":
return self
async def __aexit__(self, *_a: object) -> None:
return None
def _patch_session(monkeypatch: pytest.MonkeyPatch,
session: _MockSession) -> None:
monkeypatch.setattr(ci_client.aiohttp, "ClientSession", lambda: session)
def test_config_default_max_runs():
cfg = GitHubCIConfig(token="t", owner="o", repo="r")
assert cfg.max_runs == 300
def test_config_override_max_runs():
cfg = GitHubCIConfig(token="t", owner="o", repo="r", max_runs=42)
assert cfg.max_runs == 42
@pytest.mark.asyncio
async def test_paginator_truncates_to_max_results(monkeypatch):
session = _MockSession("workflow_runs", total=1000)
_patch_session(monkeypatch, session)
out = await ci_client.ci_get_paginated(
"tok",
"/repos/{owner}/{repo}/actions/runs",
list_key="workflow_runs",
max_results=300,
owner="o",
repo="r",
)
assert len(out) == 300
assert len(session.calls) == 3
@pytest.mark.asyncio
async def test_paginator_no_max_returns_all(monkeypatch):
session = _MockSession("workflow_runs", total=250)
_patch_session(monkeypatch, session)
out = await ci_client.ci_get_paginated(
"tok",
"/repos/{owner}/{repo}/actions/runs",
list_key="workflow_runs",
owner="o",
repo="r",
)
assert len(out) == 250
@pytest.mark.asyncio
async def test_paginator_stops_when_batch_short(monkeypatch):
session = _MockSession("workflow_runs", total=50)
_patch_session(monkeypatch, session)
out = await ci_client.ci_get_paginated(
"tok",
"/repos/{owner}/{repo}/actions/runs",
list_key="workflow_runs",
max_results=300,
owner="o",
repo="r",
)
assert len(out) == 50
assert len(session.calls) == 1
@pytest.mark.asyncio
async def test_list_runs_uses_config_max_runs(monkeypatch):
session = _MockSession("workflow_runs", total=1000)
_patch_session(monkeypatch, session)
cfg = GitHubCIConfig(token="t", owner="o", repo="r", max_runs=150)
out = await list_runs(cfg)
assert len(out) == 150
assert len(session.calls) == 2
@pytest.mark.asyncio
async def test_list_runs_default_caps_at_300(monkeypatch):
session = _MockSession("workflow_runs", total=10_000)
_patch_session(monkeypatch, session)
cfg = GitHubCIConfig(token="t", owner="o", repo="r")
out = await list_runs(cfg)
assert len(out) == 300
assert len(session.calls) == 3
@pytest.mark.asyncio
async def test_ls_runs_listing_capped(monkeypatch):
session = _MockSession("workflow_runs", total=1000)
_patch_session(monkeypatch, session)
cfg = GitHubCIConfig(token="t", owner="o", repo="r", max_runs=5)
accessor = GitHubCIAccessor(config=cfg)
index = RAMIndexCacheStore()
runs_path = PathSpec(resource_path="runs",
virtual="/runs",
directory="/runs")
ls_cmd = next(c for c in COMMANDS
if any(rc.name == "ls" for rc in c._registered_commands))
out, _ = await ls_cmd.__wrapped__(
accessor, [runs_path], [],
CommandOpts(index=index, flags={"args_1": True}))
data = await materialize(out)
names = [line for line in data.decode().splitlines() if line.strip()]
assert len(names) == 5
python/tests/core/github_ci/test_stat.py
-75
View File
@@ -1,75 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import pytest
from mirage.accessor.github_ci import GitHubCIAccessor
from mirage.cache.index import IndexEntry
from mirage.cache.index.ram import RAMIndexCacheStore
from mirage.core.github_ci.stat import stat
from mirage.resource.github_ci.config import GitHubCIConfig
from mirage.types import FileType, PathSpec
@pytest.fixture
def accessor():
return GitHubCIAccessor(GitHubCIConfig(token="t", owner="o", repo="r"))
@pytest.fixture
def index():
return RAMIndexCacheStore()
def _spec(original: str) -> PathSpec:
return PathSpec(virtual=original,
directory=original,
resource_path=original.strip("/"))
@pytest.mark.asyncio
async def test_stat_run_returns_modified(accessor, index):
await index.put(
"/runs/CI__123",
IndexEntry(
id="123",
name="CI",
resource_type="ci/run",
remote_time="2026-04-05T00:00:00Z",
vfs_name="CI__123",
),
)
result = await stat(accessor, _spec("/runs/CI__123"), index)
assert result.type == FileType.DIRECTORY
assert result.extra["run_id"] == "123"
assert result.modified == "2026-04-05T00:00:00Z"
@pytest.mark.asyncio
async def test_stat_run_json_serves_seeded_size(accessor, index):
await index.put(
"/runs/CI__123/run.json",
IndexEntry(
id="123",
name="run.json",
resource_type="ci/run_json",
vfs_name="run.json",
size=57,
remote_time="2026-04-05T00:00:00Z",
),
)
result = await stat(accessor, _spec("/runs/CI__123/run.json"), index)
assert result.type == FileType.JSON
assert result.size == 57
assert result.extra["run_id"] == "123"
python/tests/core/redis/test_read.py
+34
View File
@@ -113,3 +113,37 @@ async def test_read_bytes_normalizes_path():
assert result == b"data"
await s.clear()
await s.close()
@pytest.mark.asyncio
async def test_read_bytes_window_uses_getrange(accessor):
"""A window is sliced by redis, and the bounds are inclusive."""
spec = PathSpec(resource_path="hello.txt",
virtual="/hello.txt",
directory="/hello.txt")
assert await read_bytes(accessor, offset=6, size=5,
path_spec=spec) == b"world"
assert await read_bytes(accessor, offset=6, size=None,
path_spec=spec) == b"world"
assert await read_bytes(accessor, offset=0, size=5,
path_spec=spec) == b"hello"
@pytest.mark.asyncio
async def test_read_bytes_window_past_eof_is_empty(accessor):
spec = PathSpec(resource_path="hello.txt",
virtual="/hello.txt",
directory="/hello.txt")
assert await read_bytes(accessor, offset=99, size=5, path_spec=spec) == b""
@pytest.mark.asyncio
async def test_read_bytes_window_on_a_missing_key_still_raises(accessor):
"""GETRANGE answers "" for a missing key, so EXISTS decides absence."""
with pytest.raises(FileNotFoundError):
await read_bytes(accessor,
PathSpec(resource_path="nope.txt",
virtual="/nope.txt",
directory="/nope.txt"),
offset=1,
size=2)
python/tests/core/slack/test_read.py
+78
View File
@@ -210,3 +210,81 @@ async def test_download_file_uses_bot_token():
await download_file(SlackConfig(token="xoxb-bot"), "http://x")
assert seen[0] == {"Authorization": "Bearer xoxb-bot"}
assert seen[1] == {"Authorization": "Bearer xoxb-bot"}
@pytest.mark.asyncio
async def test_read_file_blob_pushes_the_window_down(accessor, index):
"""An upload is stored bytes, so the window becomes a Range header."""
await index.set_dir("/channels", [
(
"general__C001",
IndexEntry(
id="C001",
name="general",
resource_type="slack/channel",
vfs_name="general__C001",
),
),
])
await index.set_dir(
"/channels/general__C001/2026-04-10/files",
[
(
"report__F1.pdf",
IndexEntry(
id="F1",
name="report.pdf",
resource_type="slack/file",
vfs_name="report__F1.pdf",
size=4096,
extra={
"url_private_download":
"https://files.slack.com/x/report.pdf",
},
),
),
],
)
with patch("mirage.core.slack.files.download_file",
new_callable=AsyncMock,
return_value=b"1.4 f") as mock_dl:
data = await read(
accessor,
PathSpec(resource_path=("channels/general__C001/2026-04-10"
"/files/report__F1.pdf"),
virtual=("/channels/general__C001/2026-04-10"
"/files/report__F1.pdf"),
directory=("/channels/general__C001/2026-04-10"
"/files/report__F1.pdf")),
index=index,
offset=5,
size=5,
)
assert data == b"1.4 f"
mock_dl.assert_called_once_with(accessor.config,
"https://files.slack.com/x/report.pdf",
"bytes=5-9")
@pytest.mark.asyncio
async def test_read_jsonl_window_is_sliced_locally(accessor, index):
"""Rendered history has no remote range, so the window is taken after."""
await _populate_index(index)
with patch(
"mirage.core.slack.read.get_history_jsonl",
new_callable=AsyncMock,
return_value=b'{"text":"hello"}\n',
):
result = await read(
accessor,
PathSpec(
resource_path=(
"/channels/general__C001/2023-11-14/chat.jsonl"),
virtual="/channels/general__C001/2023-11-14/chat.jsonl",
directory="/channels/general__C001/2023-11-14/chat.jsonl",
),
index=index,
offset=1,
size=6,
)
assert result == b'"text"'
python/tests/ops/test_inventory.py
-5
View File
@@ -111,11 +111,6 @@ OPS_INVENTORY = {
("readdir", "github", "", False),
("stat", "github", "", False),
],
"github_ci": [
("read", "github_ci", "", False),
("readdir", "github_ci", "", False),
("stat", "github_ci", "", False),
],
"gmail": [
("read", "gmail", "", False),
("readdir", "gmail", "", False),
python/tests/ops/test_read_range_roster.py
+80
View File
@@ -0,0 +1,80 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import importlib
import pkgutil
import pytest
import mirage.commands.builtin
# Every backend that takes the window itself instead of leaving it to the
# generic read-and-slice fallback. Most push it down to the store (one
# ranged GET rather than the whole object); the ones that render their
# content or already hold it in memory take the window right after
# building the bytes, so a windowed read is answered the same way
# everywhere. Losing a name here is not a test failure anywhere else:
# the fallback keeps the backend correct while it silently starts
# reading whole objects again.
NATIVE_RANGE = {
"box",
"databricks_volume",
"dify",
"discord",
"disk",
"dropbox",
"gdrive",
"gridfs",
"hf_buckets",
"nextcloud",
"onedrive",
"ram",
"redis",
"s3",
"sharepoint",
"slack",
"ssh",
}
def _io_tables() -> dict[str, object]:
"""Every backend's ``CommandIO``, keyed by backend package name."""
tables: dict[str, object] = {}
for mod in pkgutil.iter_modules(mirage.commands.builtin.__path__):
if not mod.ispkg:
continue
try:
io = importlib.import_module(
f"mirage.commands.builtin.{mod.name}.io")
except ImportError:
continue
table = getattr(io, "IO", None)
if table is not None and hasattr(table, "read_range"):
tables[mod.name] = table
return tables
def test_native_range_roster_is_exactly_the_declared_set():
tables = _io_tables()
assert tables, "no backend IO tables were discovered"
actual = {n for n, t in tables.items() if t.read_range is not None}
assert actual == NATIVE_RANGE & set(tables)
@pytest.mark.parametrize("name", sorted(NATIVE_RANGE))
def test_declared_backends_expose_the_slot(name):
tables = _io_tables()
if name not in tables:
pytest.skip(f"{name} has no importable IO table here")
assert tables[name].read_range is not None
python/tests/resource/test_state_round_trip.py
-2
View File
@@ -243,8 +243,6 @@ REDACTION_CASES = [
("mirage.resource.trello", "TrelloResource", "TrelloConfig",
dict(api_key="TRELLO-KEY-LEAK", api_token="TRELLO-TOKEN-LEAK"),
["TRELLO-KEY-LEAK", "TRELLO-TOKEN-LEAK"]),
("mirage.resource.github_ci", "GitHubCIResource", "GitHubCIConfig",
dict(token="GHCI-TOKEN-LEAK", owner="o", repo="r"), ["GHCI-TOKEN-LEAK"]),
("mirage.resource.email", "EmailResource", "EmailConfig",
dict(imap_host="h",
smtp_host="h",
python/tests/test_tests_mirror_sources.py
+1 -1
View File
@@ -61,7 +61,7 @@ UNMIRRORED_DIRS = {
# would count 816 today. What the ratchet buys is narrower than it looks:
# a module whose name appears nowhere in the suite cannot be added
# silently.
MIRROR_BASELINE = 201
MIRROR_BASELINE = 194
def _test_dirs() -> list[pathlib.Path]:
python/tests/utils/test_ranges.py
+11
View File
@@ -120,3 +120,14 @@ def test_a_real_failure_is_not_swallowed():
assert not is_unsatisfiable_range(_BotoError("NoSuchKey", 404))
assert not is_unsatisfiable_range(_AiohttpError(500))
assert not is_unsatisfiable_range(RuntimeError("AccessDenied"))
def test_the_opendal_seek_shape_is_unsatisfiable():
"""hf and nextcloud seek instead of sending a header, and the seek
itself raises rather than surfacing a status."""
assert is_unsatisfiable_range(
OSError("invalid seek to a position beyond the end of the range"))
def test_an_ordinary_seek_error_is_not_unsatisfiable():
assert not is_unsatisfiable_range(OSError("invalid seek: bad whence"))
spec/layout_exceptions.json
+1 -1
View File
@@ -1,5 +1,5 @@
{
"baseline": 300,
"baseline": 299,
"baseline_reason": "Every divergence below the excused ones predates the gate and each needs its own decision, so --strict fails on a rise rather than demanding zero. Lower this number whenever a divergence is closed; the gate fails on a drop too, so an improvement cannot be silently spent. Items 31-37 of the cleanup plan are scoped from this report. The unit is one module, never one directory: a one-sided directory counts once per module inside it, so it cannot absorb new modules without moving the number. An excused directory is the deliberate exception -- it excuses its whole subtree, because the excuse is that there is no counterpart to mirror, which makes growth inside it expected rather than drift.",
"directories": {
"python_only": {
spec/parity_exceptions.json
-18
View File
@@ -1,28 +1,10 @@
{
"command_io": {
"box": {
"slots": "python range-reads; the typescript core read has no window argument."
},
"dropbox": {
"slots": "python range-reads; the typescript core read has no window argument."
},
"gdrive": {
"slots": "python range-reads; the typescript core read has no window argument."
},
"hf": {
"slots": "find is a slot in python and a bespoke command in typescript (see notion) \u2014 the same query, expressed at a different layer."
},
"notion": {
"slots": "find is a slot in python and a bespoke command in typescript (notion/find.ts). Both pass stat=None, so the -mtime behavior matches; only the layer differs."
},
"onedrive": {
"slots": "python range-reads; typescript has no core/onedrive/read.ts to add a window to."
},
"sharepoint": {
"slots": "python range-reads; typescript has no core/sharepoint/read.ts to add a window to."
},
"ssh": {
"slots": "python range-reads over SFTP; typescript's read is whole-file because of the ssh2 256 KiB packet cap."
}
},
"command_io_aliases": {
spec/python/general/awk.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/base64.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/basename.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/cat.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -241,7 +235,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/cmp.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/column.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/comm.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/cut.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/diff.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/dirname.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/du.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/expand.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/file.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/find.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -241,7 +235,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/fmt.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/fold.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/grep.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": false,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -241,7 +235,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/head.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -241,7 +235,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/join.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",
spec/python/general/jq.json
-7
View File
@@ -73,12 +73,6 @@
"has_provision": true,
"has_write": false
},
"github_ci": {
"filetypes": [],
"has_aggregate": false,
"has_provision": true,
"has_write": false
},
"gmail": {
"filetypes": [],
"has_aggregate": false,
@@ -235,7 +229,6 @@
"gdocs",
"gdrive",
"github",
"github_ci",
"gmail",
"gridfs",
"gsheets",

Some files were not shown because too many files have changed in this diff Show More