github-featured/strukto-ai--mirage
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat(db): semantic virtualization + JSON-harness integ for all DB backends (#644)

Browse Source 
* feat(db): semantic virtualization + JSON-harness integ for all DB backends

Postgres/Mongo DB virtualization work plus a full integ-harness migration.

Core:
- Postgres semantic.json (dimensions/time_dimensions/facts/relationships),
  byte-identical py+ts.
- Fix SQL identifier injection via quote_ident/qualified (py; ts already safe).
- Fix grep/rg push-down: honor a literal+no-shaping-flags gate so -v/-c/-l/-n
  fall through to the generic scan; rg now mirrors grep per backend in py+ts.
- Fix a hidden py/ts divergence: whole-valued double renders as `5` (Postgres
  canonical + node driver) not `5.0`; shared canonicalize_row in
  utils/json_canonical, applied to postgres rows and mongodb docs/schema.
- Mongodb schema type inference classifies whole-valued doubles as int
  (matches the JS driver) and samples deterministically (sorted _id).

Integ:
- Migrate postgres, mongodb, chroma, qdrant, lancedb, notion off the bespoke
  .txt truth-file harness onto the shared JSON case harness (targets.json +
  resources/<backend>/*.json), byte-identical py==ts; delete the bespoke
  *.py/*.ts scripts and truth_*.txt; CI runs `main.py/ts --target <backend>`.
- notion's TS-only MCP/REST transport parity kept as a standalone
  self-asserting notion_mcp_parity.ts (no golden file).

* fix(search): faithful grep/rg push-down (Codex review)

- Postgres push-down was case-INsensitive (ILIKE) and left % / _ unescaped,
  so `grep Ada` matched `ada` and `rg -F user_id` matched `userXid`. Now
  case-sensitive LIKE by default, ILIKE only under -i, and LIKE wildcards are
  escaped. Threads case sensitivity through the row + metadata search chain
  (py + ts); metadata scan is case-sensitive unless -i.
- search_pushdown_ok rejects newline-joined patterns (-F with multiple -e are
  independent alternatives LIKE cannot express) so they take the generic path.
- has_search_shaping_flags now recognizes rg's -I (no filename) and the
  file-filtering --glob/--type, forcing those onto the generic scan.
This commit is contained in:
Zecheng Zhang
2026-07-26 22:12:28 -07:00
committed by GitHub
parent e6767aa0d2
commit 087ed195c0
105 changed files with 7642 additions and 3373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test_integ.yml
+58 -68
View File
@@ -51,15 +51,11 @@ jobs:
- '.github/workflows/test_integ.yml'
- '.github/actions/integ-battery-setup/action.yml'
database:
- 'integ/mongodb.py'
- 'integ/mongodb.ts'
- 'integ/truth_mongodb.txt'
- 'integ/postgres.py'
- 'integ/postgres.ts'
- 'integ/truth_postgres.txt'
- 'integ/chroma.py'
- 'integ/chroma.ts'
- 'integ/truth_chroma.txt'
- 'integ/resources/mongodb/**'
- 'integ/resources/postgres/**'
- 'integ/targets.json'
- 'integ/runners/**'
- 'integ/resources/chroma/**'
- 'integ/check_lines.sh'
- 'python/mirage/accessor/mongodb.py'
- 'python/mirage/core/mongodb/**'
@@ -73,9 +69,7 @@ jobs:
- 'python/mirage/core/chroma/**'
- 'python/mirage/commands/builtin/chroma/**'
- 'python/mirage/resource/chroma/**'
- 'integ/qdrant.py'
- 'integ/qdrant.ts'
- 'integ/truth_qdrant.txt'
- 'integ/resources/qdrant/**'
- 'python/mirage/accessor/qdrant.py'
- 'python/mirage/core/qdrant/**'
- 'python/mirage/commands/builtin/qdrant/**'
@@ -172,10 +166,10 @@ jobs:
./python/.venv/bin/python integ/history.py > /tmp/history.out
diff integ/truth_history.txt /tmp/history.out
- name: Run lancedb integ (embedded) and diff against truth
run: |
./python/.venv/bin/python integ/lancedb.py > /tmp/lancedb.out
diff integ/truth_lancedb.txt /tmp/lancedb.out
- name: Run lancedb integ (embedded, JSON harness)
env:
LANCEDB_ENABLED: "1"
run: ./python/.venv/bin/python integ/runners/python/main.py --target lancedb
- name: Run find arg-error suite (SaaS backends, dummy creds) and diff
run: |
@@ -315,17 +309,21 @@ jobs:
working-directory: integ
run: pnpm exec tsx cross_commands.ts
- name: Run notion integ (TS, mock server) and diff against truth
- name: Run notion integ (TS, mock server, JSON harness)
working-directory: integ
run: |
pnpm exec tsx notion.ts > /tmp/notion_ts.out
diff truth_notion.txt /tmp/notion_ts.out
env:
NOTION_ENABLED: "1"
run: pnpm exec tsx runners/typescript/main.ts --target notion
- name: Run lancedb integ (TS, embedded) and diff against truth
- name: Run notion MCP/REST parity check (TS)
working-directory: integ
run: |
pnpm exec tsx lancedb.ts > /tmp/ts-lancedb.out
diff truth_lancedb.txt /tmp/ts-lancedb.out
run: pnpm exec tsx notion_mcp_parity.ts
- name: Run lancedb integ (TS, embedded, JSON harness)
working-directory: integ
env:
LANCEDB_ENABLED: "1"
run: pnpm exec tsx runners/typescript/main.ts --target lancedb
- name: Run find arg-error suite (SaaS backends, dummy creds) and diff
working-directory: integ
@@ -536,29 +534,23 @@ jobs:
sleep 1
done
- name: Run MongoDB backend and check against truth
shell: bash
run: |
set -o pipefail
./python/.venv/bin/python integ/mongodb.py 2>&1 \
| bash integ/check_lines.sh integ/truth_mongodb.txt
- name: Run MongoDB backend (JSON harness)
run: ./python/.venv/bin/python integ/runners/python/main.py --target mongodb
- name: Run Postgres backend and check against truth
shell: bash
run: |
set -o pipefail
./python/.venv/bin/python integ/postgres.py 2>&1 \
| bash integ/check_lines.sh integ/truth_postgres.txt
- name: Run Postgres backend (JSON harness)
run: ./python/.venv/bin/python integ/runners/python/main.py --target postgres
- name: Run chroma integ (ChromaDB OSS) and diff against truth
run: |
./python/.venv/bin/python integ/chroma.py > /tmp/chroma.out
diff integ/truth_chroma.txt /tmp/chroma.out
- name: Run chroma integ (ChromaDB OSS, JSON harness)
env:
CHROMA_HOST: localhost
CHROMA_PORT: "8000"
run: ./python/.venv/bin/python integ/runners/python/main.py --target chroma
- name: Run qdrant integ and diff against truth
run: |
./python/.venv/bin/python integ/qdrant.py > /tmp/qdrant.out
diff integ/truth_qdrant.txt /tmp/qdrant.out
- name: Run qdrant integ (JSON harness)
env:
QDRANT_HOST: localhost
QDRANT_PORT: "6333"
run: ./python/.venv/bin/python integ/runners/python/main.py --target qdrant
integ-database-ts:
needs: changes
@@ -614,6 +606,10 @@ jobs:
working-directory: typescript
run: pnpm --filter @struktoai/mirage-node build
- name: Build mirage-browser
working-directory: typescript
run: pnpm --filter @struktoai/mirage-browser build
- name: Wait for ChromaDB OSS
run: |
for i in $(seq 1 30); do
@@ -647,33 +643,27 @@ jobs:
echo "Qdrant did not become healthy" >&2
exit 1
- name: Run MongoDB backend (TS) and check against truth
shell: bash
- name: Run MongoDB backend (TS, JSON harness)
working-directory: integ
run: |
set -o pipefail
pnpm exec tsx mongodb.ts 2>&1 \
| bash check_lines.sh truth_mongodb.txt
run: pnpm exec tsx runners/typescript/main.ts --target mongodb
- name: Run Postgres backend (TS) and check against truth
shell: bash
- name: Run Postgres backend (TS, JSON harness)
working-directory: integ
run: |
set -o pipefail
pnpm exec tsx postgres.ts 2>&1 \
| bash check_lines.sh truth_postgres.txt
run: pnpm exec tsx runners/typescript/main.ts --target postgres
- name: Run TS chroma integ (ChromaDB OSS) and diff against truth
- name: Run TS chroma integ (ChromaDB OSS, JSON harness)
working-directory: integ
run: |
pnpm exec tsx chroma.ts > /tmp/ts-chroma.out
diff truth_chroma.txt /tmp/ts-chroma.out
env:
CHROMA_HOST: localhost
CHROMA_PORT: "8000"
run: pnpm exec tsx runners/typescript/main.ts --target chroma
- name: Run TS qdrant integ and diff against truth
- name: Run TS qdrant integ (JSON harness)
working-directory: integ
run: |
pnpm exec tsx qdrant.ts > /tmp/ts-qdrant.out
diff truth_qdrant.txt /tmp/ts-qdrant.out
env:
QDRANT_HOST: localhost
QDRANT_PORT: "6333"
run: pnpm exec tsx runners/typescript/main.ts --target qdrant
integ-data:
needs: changes
@@ -710,10 +700,10 @@ jobs:
working-directory: python
run: uv sync --all-extras --no-extra camel
- name: Run notion integ (mock server) and diff against truth
run: |
./python/.venv/bin/python integ/notion.py > /tmp/notion.out
diff integ/truth_notion.txt /tmp/notion.out
- name: Run notion integ (mock server, JSON harness)
env:
NOTION_ENABLED: "1"
run: ./python/.venv/bin/python integ/runners/python/main.py --target notion
- name: Wait for Nextcloud install
shell: bash
integ/chroma.py
-305
View File
@@ -1,305 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import base64
import gzip
import json
import os
import sys
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
import chromadb # noqa: E402
from cases import run_provision_probe # noqa: E402
from cases import run_not_found, run_sed_readonly_probe # noqa: E402
from mirage import MountMode, Workspace # noqa: E402
from mirage.resource.chroma import ChromaConfig, ChromaResource # noqa: E402
CHROMA_HOST = os.environ.get("CHROMA_HOST", "localhost")
CHROMA_PORT = int(os.environ.get("CHROMA_PORT", "8000"))
EMBED_DIM = 8
MOUNT = "/knowledge/"
PATH_TREE: dict[str, dict] = {
"guides/quickstart.md": {
"size": 180,
"created_at": "2026-01-01T00:00:00Z",
"updated_at": "2026-02-01T00:00:00Z",
},
"guides/auth.md": {
"size": 190,
"created_at": "2026-01-15T00:00:00Z",
"updated_at": "2026-02-15T00:00:00Z",
},
"policies/refunds.md": {
"size": 150,
"created_at": "2026-02-01T00:00:00Z",
"updated_at": "2026-03-01T00:00:00Z",
},
"policies/privacy.md": {
"size": 120,
"created_at": "2026-02-10T00:00:00Z",
"updated_at": "2026-03-10T00:00:00Z",
},
"CHANGELOG.md": {
"size": 90,
},
# No size metadata: sizeless files count as 0 for -size/du, never dropped.
"policies/archived.md": {
"created_at": "2026-03-01T00:00:00Z",
},
}
CHUNKS: dict[str, list[dict]] = {
"policies/archived.md": [
{
"document": "Archived policy retained for records.",
"metadata": {
"page_slug": "policies/archived.md",
"chunk_index": 0
},
},
],
"guides/quickstart.md": [
{
"document":
"Welcome to Acme. This quickstart gets you running fast.",
"metadata": {
"page_slug": "guides/quickstart.md",
"chunk_index": 0
},
},
{
"document":
"Install the CLI with npm i -g acme then run acme login.",
"metadata": {
"page_slug": "guides/quickstart.md",
"chunk_index": 1
},
},
{
"document":
"Set your token in the ACME_TOKEN environment variable.",
"metadata": {
"page_slug": "guides/quickstart.md",
"chunk_index": 2
},
},
],
"guides/auth.md": [
{
"document":
"Authentication uses bearer tokens via the Authorization header.",
"metadata": {
"page_slug": "guides/auth.md",
"chunk_index": 0
},
},
{
"document":
"Requests are rate limited to 100 calls per minute per token.",
"metadata": {
"page_slug": "guides/auth.md",
"chunk_index": 1
},
},
{
"document":
"If you exceed the limit you receive HTTP 429 and must back off.",
"metadata": {
"page_slug": "guides/auth.md",
"chunk_index": 2
},
},
],
"policies/refunds.md": [
{
"document": "Refunds are available within 30 days of purchase.",
"metadata": {
"page_slug": "policies/refunds.md",
"chunk_index": 0
},
},
{
"document": "Email support to start a refund with your order id.",
"metadata": {
"page_slug": "policies/refunds.md",
"chunk_index": 1
},
},
{
"document":
"Approved refunds are processed within five business days.",
"metadata": {
"page_slug": "policies/refunds.md",
"chunk_index": 2
},
},
],
"policies/privacy.md": [
{
"document":
"Customer data is stored encrypted at rest and in transit.",
"metadata": {
"page_slug": "policies/privacy.md",
"chunk_index": 0
},
},
{
"document": "We never sell personal information to third parties.",
"metadata": {
"page_slug": "policies/privacy.md",
"chunk_index": 1
},
},
],
"CHANGELOG.md": [
{
"document": "v2.0 added rate limit headers and refund automation.",
"metadata": {
"page_slug": "CHANGELOG.md",
"chunk_index": 0
},
},
],
}
CASES: list[tuple[str, str]] = [
("ls", "ls {root}"),
("ls_guides", "ls {root}guides/"),
("find_md", "find {root} -name '*.md'"),
("find_type_f", "find {root} -type f | sort"),
("find_root_maxdepth0", "find {root} -maxdepth 0"),
("find_root_name", "find {root} -name knowledge"),
("find_size_plus_100c", "find {root} -type f -size +100c | sort"),
("find_size_archived_kept",
"find {root}policies/ -name 'archived*' -size -1k"),
# cold (bespoke) then warm (cache-mount generic) must be identical
("grep_cold_single", "grep bearer {root}guides/auth.md"),
("grep_warm_single", "grep bearer {root}guides/auth.md"),
("cat_auth", "cat {root}guides/auth.md"),
("cat_quickstart", "cat {root}guides/quickstart.md"),
("head_1", "head -n 1 {root}guides/quickstart.md"),
("tail_1", "tail -n 1 {root}guides/quickstart.md"),
("grep_429", "grep 429 {root}guides/auth.md"),
("grep_e_multi", "grep -e bearer -e 429 {root}guides/auth.md"),
("grep_c_rate", "grep -c rate {root}guides/auth.md"),
("grep_r_refund", "grep -r refund {root}policies/"),
("grep_cold_count", "grep -c sell {root}policies/privacy.md"),
("grep_warm_count", "grep -c sell {root}policies/privacy.md"),
("grep_rl_encrypted", "grep -rl encrypted {root}"),
("grep_v_bearer", "grep -v bearer {root}guides/auth.md"),
("grep_rE_alternation", 'grep -rE "rate limited|refund" {root}'),
("wc_l_auth", "wc -l {root}guides/auth.md"),
("sort_auth", "sort {root}guides/auth.md"),
("uniq_auth", "uniq {root}guides/auth.md"),
("uniq_w0_auth", "uniq -w 0 {root}guides/auth.md"),
("stat_name_auth", 'stat -c "%n" {root}guides/auth.md'),
("cut_d_f1", "cut -d ' ' -f 1 {root}guides/quickstart.md"),
("awk_first_word", "awk '{{print $1}}' {root}guides/quickstart.md"),
("sed_upper_acme", "sed s/Acme/ACME/ {root}guides/quickstart.md"),
("rg_l_token", "rg -l token {root}"),
("pipe_cat_wc", "cat {root}guides/auth.md | wc -l"),
("pipe_sort_uniq_wc", "cat {root}policies/refunds.md | sort | uniq"
" | wc -l"),
# cat cache poisoning: per-file cache keys after a concat cat
("poison_concat", "cat {root}guides/quickstart.md {root}guides/auth.md"),
("poison_first_intact", "cat {root}guides/quickstart.md"),
("poison_second_intact", "cat {root}guides/auth.md"),
("pipe_concat_head",
"cat {root}guides/quickstart.md {root}guides/auth.md | head -n 1"),
# du has no native op -> exercises the stat/readdir walk fallback,
# which must match the Python du builder byte for byte.
("du_guides", "du {root}guides"),
("du_root", "du {root}"),
("du_c_multi", "du -c {root}guides {root}policies"),
# symlink into the mount: links are namespace state, so they work on a
# read-only backend (no backend write happens)
("sym_ln", "ln -s {root}guides/auth.md {root}meta_link"),
("sym_readlink", "readlink {root}meta_link"),
("sym_cat", "cat {root}meta_link"),
("sym_wc", "wc -l {root}meta_link"),
("sym_ls", "ls -F {root} | grep meta_link"),
("sym_rm", "rm {root}meta_link && ls {root}"),
]
def encoded_path_tree() -> str:
# gzip+base64 variant so the integ covers parse_path_tree's
# encoded branch (unit tests cover the plain JSON branch)
raw = json.dumps(PATH_TREE).encode()
return base64.b64encode(gzip.compress(raw)).decode()
def embedding_for(position: int) -> list[float]:
vector = [0.0] * EMBED_DIM
vector[position % EMBED_DIM] = 1.0
return vector
async def seed_collection(collection_name: str) -> None:
client = await chromadb.AsyncHttpClient(host=CHROMA_HOST, port=CHROMA_PORT)
collection = await client.create_collection(collection_name)
ids = ["__path_tree__"]
documents = [encoded_path_tree()]
metadatas: list[dict] = [{"kind": "path_tree"}]
embeddings = [embedding_for(0)]
position = 1
for chunks in CHUNKS.values():
for chunk in chunks:
slug = chunk["metadata"]["page_slug"]
index = chunk["metadata"]["chunk_index"]
ids.append(f"{slug}#{index}")
documents.append(chunk["document"])
metadatas.append(chunk["metadata"])
embeddings.append(embedding_for(position))
position += 1
await collection.add(ids=ids,
documents=documents,
metadatas=metadatas,
embeddings=embeddings)
async def run_case(ws: Workspace, name: str, cmd: str) -> None:
result = await ws.execute(cmd)
out = await result.stdout_str()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
async def main() -> None:
collection_name = f"mirage-integ-{uuid.uuid4().hex[:8]}"
await seed_collection(collection_name)
config = ChromaConfig(
host=CHROMA_HOST,
port=CHROMA_PORT,
collection_name=collection_name,
)
ws = Workspace({MOUNT: ChromaResource(config=config)}, mode=MountMode.READ)
# Vector search (the `search` command) is not exercised here: the thin
# client ships no embedding function for query_texts. It is covered by
# unit tests (tests/core/chroma/test_search.py).
for name, tmpl in CASES:
await run_case(ws, name, tmpl.format(root=MOUNT))
await run_not_found(ws, MOUNT)
await run_provision_probe(ws, f"{MOUNT}guides/auth.md")
await run_sed_readonly_probe(ws, f"{MOUNT}guides/auth.md")
if __name__ == "__main__":
asyncio.run(main())
integ/chroma.ts
-270
View File
@@ -1,270 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { randomBytes } from "node:crypto";
import { gzipSync } from "node:zlib";
import { ChromaClient } from "chromadb";
import { ChromaResource, MountMode, Workspace } from "@struktoai/mirage-node";
import { runNotFound, runProvisionProbe, runSedReadonlyProbe } from "./cases.ts";
const CHROMA_HOST = process.env.CHROMA_HOST ?? "localhost";
const CHROMA_PORT = Number.parseInt(process.env.CHROMA_PORT ?? "8000", 10);
const EMBED_DIM = 8;
const MOUNT = "/knowledge/";
const DEC = new TextDecoder();
const PATH_TREE: Record<string, Record<string, unknown>> = {
"guides/quickstart.md": {
size: 180,
created_at: "2026-01-01T00:00:00Z",
updated_at: "2026-02-01T00:00:00Z",
},
"guides/auth.md": {
size: 190,
created_at: "2026-01-15T00:00:00Z",
updated_at: "2026-02-15T00:00:00Z",
},
"policies/refunds.md": {
size: 150,
created_at: "2026-02-01T00:00:00Z",
updated_at: "2026-03-01T00:00:00Z",
},
"policies/privacy.md": {
size: 120,
created_at: "2026-02-10T00:00:00Z",
updated_at: "2026-03-10T00:00:00Z",
},
"CHANGELOG.md": {
size: 90,
},
// No size metadata: sizeless files count as 0 for -size/du, never dropped.
"policies/archived.md": {
created_at: "2026-03-01T00:00:00Z",
},
};
interface SeedChunk {
document: string;
metadata: { page_slug: string; chunk_index: number };
}
const CHUNKS: Record<string, SeedChunk[]> = {
"policies/archived.md": [
{
document: "Archived policy retained for records.",
metadata: { page_slug: "policies/archived.md", chunk_index: 0 },
},
],
"guides/quickstart.md": [
{
document: "Welcome to Acme. This quickstart gets you running fast.",
metadata: { page_slug: "guides/quickstart.md", chunk_index: 0 },
},
{
document: "Install the CLI with npm i -g acme then run acme login.",
metadata: { page_slug: "guides/quickstart.md", chunk_index: 1 },
},
{
document: "Set your token in the ACME_TOKEN environment variable.",
metadata: { page_slug: "guides/quickstart.md", chunk_index: 2 },
},
],
"guides/auth.md": [
{
document:
"Authentication uses bearer tokens via the Authorization header.",
metadata: { page_slug: "guides/auth.md", chunk_index: 0 },
},
{
document: "Requests are rate limited to 100 calls per minute per token.",
metadata: { page_slug: "guides/auth.md", chunk_index: 1 },
},
{
document:
"If you exceed the limit you receive HTTP 429 and must back off.",
metadata: { page_slug: "guides/auth.md", chunk_index: 2 },
},
],
"policies/refunds.md": [
{
document: "Refunds are available within 30 days of purchase.",
metadata: { page_slug: "policies/refunds.md", chunk_index: 0 },
},
{
document: "Email support to start a refund with your order id.",
metadata: { page_slug: "policies/refunds.md", chunk_index: 1 },
},
{
document: "Approved refunds are processed within five business days.",
metadata: { page_slug: "policies/refunds.md", chunk_index: 2 },
},
],
"policies/privacy.md": [
{
document: "Customer data is stored encrypted at rest and in transit.",
metadata: { page_slug: "policies/privacy.md", chunk_index: 0 },
},
{
document: "We never sell personal information to third parties.",
metadata: { page_slug: "policies/privacy.md", chunk_index: 1 },
},
],
"CHANGELOG.md": [
{
document: "v2.0 added rate limit headers and refund automation.",
metadata: { page_slug: "CHANGELOG.md", chunk_index: 0 },
},
],
};
const CASES: ReadonlyArray<readonly [string, string]> = [
["ls", "ls {root}"],
["ls_guides", "ls {root}guides/"],
["find_md", "find {root} -name '*.md'"],
["find_type_f", "find {root} -type f | sort"],
["find_root_maxdepth0", "find {root} -maxdepth 0"],
["find_root_name", "find {root} -name knowledge"],
["find_size_plus_100c", "find {root} -type f -size +100c | sort"],
["find_size_archived_kept", "find {root}policies/ -name 'archived*' -size -1k"],
// cold (bespoke) then warm (cache-mount generic) must be identical
["grep_cold_single", "grep bearer {root}guides/auth.md"],
["grep_warm_single", "grep bearer {root}guides/auth.md"],
["cat_auth", "cat {root}guides/auth.md"],
["cat_quickstart", "cat {root}guides/quickstart.md"],
["head_1", "head -n 1 {root}guides/quickstart.md"],
["tail_1", "tail -n 1 {root}guides/quickstart.md"],
["grep_429", "grep 429 {root}guides/auth.md"],
["grep_e_multi", "grep -e bearer -e 429 {root}guides/auth.md"],
["grep_c_rate", "grep -c rate {root}guides/auth.md"],
["grep_r_refund", "grep -r refund {root}policies/"],
["grep_cold_count", "grep -c sell {root}policies/privacy.md"],
["grep_warm_count", "grep -c sell {root}policies/privacy.md"],
["grep_rl_encrypted", "grep -rl encrypted {root}"],
["grep_v_bearer", "grep -v bearer {root}guides/auth.md"],
["grep_rE_alternation", 'grep -rE "rate limited|refund" {root}'],
["wc_l_auth", "wc -l {root}guides/auth.md"],
["sort_auth", "sort {root}guides/auth.md"],
["uniq_auth", "uniq {root}guides/auth.md"],
["uniq_w0_auth", "uniq -w 0 {root}guides/auth.md"],
["stat_name_auth", 'stat -c "%n" {root}guides/auth.md'],
["cut_d_f1", "cut -d ' ' -f 1 {root}guides/quickstart.md"],
["awk_first_word", "awk '{print $1}' {root}guides/quickstart.md"],
["sed_upper_acme", "sed s/Acme/ACME/ {root}guides/quickstart.md"],
["rg_l_token", "rg -l token {root}"],
["pipe_cat_wc", "cat {root}guides/auth.md | wc -l"],
["pipe_sort_uniq_wc", "cat {root}policies/refunds.md | sort | uniq | wc -l"],
// cat cache poisoning: per-file cache keys after a concat cat
["poison_concat", "cat {root}guides/quickstart.md {root}guides/auth.md"],
["poison_first_intact", "cat {root}guides/quickstart.md"],
["poison_second_intact", "cat {root}guides/auth.md"],
[
"pipe_concat_head",
"cat {root}guides/quickstart.md {root}guides/auth.md | head -n 1",
],
// du has no native op -> exercises the stat/readdir walk fallback,
// which must match the Python du builder byte for byte.
["du_guides", "du {root}guides"],
["du_root", "du {root}"],
["du_c_multi", "du -c {root}guides {root}policies"],
// symlink into the mount: links are namespace state, so they work on a
// read-only backend (no backend write happens)
["sym_ln", "ln -s {root}guides/auth.md {root}meta_link"],
["sym_readlink", "readlink {root}meta_link"],
["sym_cat", "cat {root}meta_link"],
["sym_wc", "wc -l {root}meta_link"],
["sym_ls", "ls -F {root} | grep meta_link"],
["sym_rm", "rm {root}meta_link && ls {root}"],
];
function encodedPathTree(): string {
// gzip+base64 variant so the integ covers parsePathTree's
// encoded branch (unit tests cover the plain JSON branch)
return gzipSync(Buffer.from(JSON.stringify(PATH_TREE))).toString("base64");
}
function embeddingFor(position: number): number[] {
const vector = new Array<number>(EMBED_DIM).fill(0);
vector[position % EMBED_DIM] = 1;
return vector;
}
async function seedCollection(collectionName: string): Promise<void> {
const client = new ChromaClient({ host: CHROMA_HOST, port: CHROMA_PORT });
const collection = await client.createCollection({
name: collectionName,
embeddingFunction: null,
});
const ids = ["__path_tree__"];
const documents = [encodedPathTree()];
const metadatas: Record<string, string | number>[] = [{ kind: "path_tree" }];
const embeddings = [embeddingFor(0)];
let position = 1;
for (const chunks of Object.values(CHUNKS)) {
for (const chunk of chunks) {
const slug = chunk.metadata.page_slug;
const index = chunk.metadata.chunk_index;
ids.push(`${slug}#${String(index)}`);
documents.push(chunk.document);
metadatas.push(chunk.metadata);
embeddings.push(embeddingFor(position));
position += 1;
}
}
await collection.add({ ids, documents, metadatas, embeddings });
}
async function runCase(
ws: Workspace,
name: string,
cmd: string,
): Promise<void> {
const result = await ws.execute(cmd);
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(out.endsWith("\n") ? out : out + "\n");
}
async function main(): Promise<void> {
const collectionName = `mirage-integ-${randomBytes(4).toString("hex")}`;
await seedCollection(collectionName);
const ws = new Workspace(
{
[MOUNT]: new ChromaResource({
host: CHROMA_HOST,
port: CHROMA_PORT,
collectionName,
}),
},
{ mode: MountMode.READ },
);
// Vector search (the `chroma-query` command) is not exercised here: the
// thin client ships no embedding function for queryTexts. It is covered
// by unit tests (src/core/chroma and src/util/score tests).
try {
for (const [name, tmpl] of CASES) {
await runCase(ws, name, tmpl.replaceAll("{root}", MOUNT));
}
await runNotFound(ws, MOUNT);
await runProvisionProbe(ws, `${MOUNT}guides/auth.md`);
await runSedReadonlyProbe(ws, `${MOUNT}guides/auth.md`);
} finally {
await ws.close();
}
}
main().catch((err: unknown) => {
process.stderr.write(String(err) + "\n");
process.exit(1);
});
integ/lancedb.py
-171
View File
@@ -1,171 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import sys
from pathlib import Path
_INTEG_DIR = str(Path(__file__).parent)
sys.path[:] = [p for p in sys.path if p not in (_INTEG_DIR, "")]
import asyncio # noqa: E402
import shutil # noqa: E402
import tempfile # noqa: E402
import lancedb # noqa: E402
from mirage import MountMode, Workspace # noqa: E402
from mirage.resource.lancedb import LanceDBConfig # noqa: E402
from mirage.resource.lancedb import LanceDBResource # noqa: E402
MOUNT = "/db/"
ROWS = [
{
"id": 1,
"label": "cat",
"kind": "big",
"name": "a big orange cat"
},
{
"id": 2,
"label": "cat",
"kind": "small",
"name": "a small grey cat"
},
{
"id": 3,
"label": "dog",
"kind": "big",
"name": "a big brown dog"
},
{
"id": 4,
"label": "dog",
"kind": "small",
"name": "a small white dog"
},
]
CASES: list[tuple[str, str]] = [
("ls_root", "ls {root}"),
("ls_table", "ls {root}animals"),
("ls_group", "ls {root}animals/cat"),
("find_md", "find {root}animals -name '*.md'"),
("cat_card", "cat {root}animals/cat/big/1.md"),
("wc_c_card", "wc -c {root}animals/cat/big/1.md"),
# cold (bespoke) then warm (cache-mount generic) must be identical
("grep_cold_single", "grep orange {root}animals/cat/big/1.md"),
("grep_warm_single", "grep orange {root}animals/cat/big/1.md"),
("grep_i", "grep -i ORANGE {root}animals/cat/big/1.md"),
("grep_n", "grep -n label {root}animals/cat/big/1.md"),
("grep_v", "grep -v cat {root}animals/cat/big/1.md"),
("grep_c", "grep -c cat {root}animals/cat/big/1.md"),
("grep_o", "grep -o cat {root}animals/cat/big/1.md"),
("grep_w", "grep -w cat {root}animals/cat/big/1.md"),
("grep_F_literal", 'grep -F "id: 1" {root}animals/cat/big/1.md'),
("grep_m1", "grep -m 1 cat {root}animals/cat/big/1.md"),
("grep_A1", "grep -A 1 id {root}animals/cat/big/1.md"),
("grep_B1", "grep -B 1 label {root}animals/cat/big/1.md"),
("grep_C1", "grep -C 1 label {root}animals/cat/big/1.md"),
("grep_multi",
"grep small {root}animals/cat/small/2.md {root}animals/dog/small/4.md"),
("grep_r_table", "grep -r orange {root}animals"),
("grep_r_multipath", "grep -r small {root}animals/cat {root}animals/dog"),
("grep_rl", "grep -rl cat {root}animals"),
("grep_E_alt", 'grep -E "orange|brown" {root}animals/cat/big/1.md'),
("pipe_grep_stdin", "cat {root}animals/cat/big/1.md | grep orange"),
("rg_basic", "rg orange {root}animals/cat/big/1.md"),
# du has no native op on lancedb -> exercises the stat/readdir walk
# fallback, which must match the Python du builder byte for byte.
("du_file", "du {root}animals/cat/big/1.md"),
("du_group", "du {root}animals/cat"),
("du_table", "du {root}animals"),
("du_c_multi", "du -c {root}animals/cat {root}animals/dog"),
]
EXIT_CODE_CASES: list[tuple[str, str]] = [
("grep_q_match", "grep -q cat {root}animals/cat/big/1.md"),
("grep_q_no_match", "grep -q zebra {root}animals/cat/big/1.md"),
("grep_no_match", "grep zebra {root}animals/cat/big/1.md"),
]
async def run_cases(ws: Workspace) -> None:
for name, tmpl in CASES:
result = await ws.execute(tmpl.format(root=MOUNT))
out = await result.stdout_str()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
for name, tmpl in EXIT_CODE_CASES:
result = await ws.execute(tmpl.format(root=MOUNT))
out = await result.stdout_str()
print(f"=== {name} ===")
print(f"exit={result.exit_code}")
if out:
print(out, end="" if out.endswith("\n") else "\n")
nf_target = f"{MOUNT.rstrip('/')}/__nf_missing__.txt"
for nf_name, nf_prog in (("nf_cat", "cat"), ("nf_head", "head"),
("nf_tail", "tail"), ("nf_wc", "wc"),
("nf_stat", "stat"), ("nf_grep", "grep x")):
result = await ws.execute(f"{nf_prog} {nf_target}")
err = (await result.stderr_str()).strip()
print(f"=== {nf_name} ===")
print(f"exit={result.exit_code}")
if err:
print(err)
prov_target = f"{MOUNT}animals/cat/big/1.md"
for pv_name, pv_cmd in (("prov_probe_cat", f"cat {prov_target}"),
("prov_probe_grep", f"grep x {prov_target}"),
("prov_probe_ls", f"ls {MOUNT}animals/cat/big")):
result = await ws.execute(pv_cmd, provision=True)
print(f"=== {pv_name} ===")
print(f"net={result.network_read} write={result.network_write} "
f"cache={result.cache_read} ops={result.read_ops} "
f"hits={result.cache_hits} "
f"precision={result.precision.value}")
result = await ws.execute(f"sed -n 1p {prov_target}")
out = await result.stdout_str()
print("=== sed_stream_1p ===")
print(out, end="" if out.endswith("\n") else "\n")
result = await ws.execute(f"sed -i s/x/y/ {prov_target}")
err = (await result.stderr_str()).strip()
print("=== sed_i_readonly ===")
print(f"exit={result.exit_code}")
if err:
print(err)
async def main() -> None:
uri = tempfile.mkdtemp(prefix="mirage-integ-lancedb-")
try:
db = lancedb.connect(uri)
db.create_table("animals", data=ROWS)
config = LanceDBConfig(
uri=uri,
group_by=["label", "kind"],
id_column="id",
title_column="name",
text_column="name",
)
ws = Workspace({MOUNT: LanceDBResource(config)}, mode=MountMode.READ)
# Vector search (the `search` command) is not exercised here: the
# seed table ships no vector column. It is covered by unit tests
# (tests/core/lancedb/test_search.py).
await run_cases(ws)
finally:
shutil.rmtree(uri, ignore_errors=True)
if __name__ == "__main__":
asyncio.run(main())
integ/lancedb.ts
-130
View File
@@ -1,130 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import * as lancedb from "@lancedb/lancedb";
import { LanceDBResource, MountMode, Workspace } from "@struktoai/mirage-node";
import { runNotFound, runProvisionProbe, runSedReadonlyProbe } from "./cases.ts";
const MOUNT = "/db/";
const DEC = new TextDecoder();
const ROWS = [
{ id: 1, label: "cat", kind: "big", name: "a big orange cat" },
{ id: 2, label: "cat", kind: "small", name: "a small grey cat" },
{ id: 3, label: "dog", kind: "big", name: "a big brown dog" },
{ id: 4, label: "dog", kind: "small", name: "a small white dog" },
];
const CASES: [string, string][] = [
["ls_root", "ls {root}"],
["ls_table", "ls {root}animals"],
["ls_group", "ls {root}animals/cat"],
["find_md", "find {root}animals -name '*.md'"],
["cat_card", "cat {root}animals/cat/big/1.md"],
["wc_c_card", "wc -c {root}animals/cat/big/1.md"],
// cold (bespoke) then warm (cache-mount generic) must be identical
["grep_cold_single", "grep orange {root}animals/cat/big/1.md"],
["grep_warm_single", "grep orange {root}animals/cat/big/1.md"],
["grep_i", "grep -i ORANGE {root}animals/cat/big/1.md"],
["grep_n", "grep -n label {root}animals/cat/big/1.md"],
["grep_v", "grep -v cat {root}animals/cat/big/1.md"],
["grep_c", "grep -c cat {root}animals/cat/big/1.md"],
["grep_o", "grep -o cat {root}animals/cat/big/1.md"],
["grep_w", "grep -w cat {root}animals/cat/big/1.md"],
["grep_F_literal", 'grep -F "id: 1" {root}animals/cat/big/1.md'],
["grep_m1", "grep -m 1 cat {root}animals/cat/big/1.md"],
["grep_A1", "grep -A 1 id {root}animals/cat/big/1.md"],
["grep_B1", "grep -B 1 label {root}animals/cat/big/1.md"],
["grep_C1", "grep -C 1 label {root}animals/cat/big/1.md"],
[
"grep_multi",
"grep small {root}animals/cat/small/2.md {root}animals/dog/small/4.md",
],
["grep_r_table", "grep -r orange {root}animals"],
["grep_r_multipath", "grep -r small {root}animals/cat {root}animals/dog"],
["grep_rl", "grep -rl cat {root}animals"],
["grep_E_alt", 'grep -E "orange|brown" {root}animals/cat/big/1.md'],
["pipe_grep_stdin", "cat {root}animals/cat/big/1.md | grep orange"],
["rg_basic", "rg orange {root}animals/cat/big/1.md"],
// du has no native op on lancedb -> exercises the stat/readdir walk
// fallback, which must match the Python du builder byte for byte.
["du_file", "du {root}animals/cat/big/1.md"],
["du_group", "du {root}animals/cat"],
["du_table", "du {root}animals"],
["du_c_multi", "du -c {root}animals/cat {root}animals/dog"],
];
const EXIT_CODE_CASES: [string, string][] = [
["grep_q_match", "grep -q cat {root}animals/cat/big/1.md"],
["grep_q_no_match", "grep -q zebra {root}animals/cat/big/1.md"],
["grep_no_match", "grep zebra {root}animals/cat/big/1.md"],
];
async function runCases(ws: Workspace): Promise<void> {
for (const [name, tmpl] of CASES) {
const result = await ws.execute(tmpl.replaceAll("{root}", MOUNT));
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(out.endsWith("\n") ? out : out + "\n");
}
for (const [name, tmpl] of EXIT_CODE_CASES) {
const result = await ws.execute(tmpl.replaceAll("{root}", MOUNT));
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(`exit=${result.exitCode}\n`);
if (out) process.stdout.write(out.endsWith("\n") ? out : out + "\n");
}
}
async function main(): Promise<void> {
const uri = mkdtempSync(join(tmpdir(), "mirage-integ-lancedb-"));
try {
const db = await lancedb.connect(uri);
await db.createTable("animals", ROWS);
const ws = new Workspace(
{
[MOUNT]: new LanceDBResource({
uri,
groupBy: ["label", "kind"],
idColumn: "id",
titleColumn: "name",
textColumn: "name",
}),
},
{ mode: MountMode.READ },
);
// Vector search (the `search` command) is not exercised here: the seed
// table ships no vector column. It is covered by unit tests
// (src/core/lancedb tests).
try {
await runCases(ws);
await runNotFound(ws, MOUNT);
await runProvisionProbe(ws, `${MOUNT}animals/cat/big/1.md`);
await runSedReadonlyProbe(ws, `${MOUNT}animals/cat/big/1.md`);
} finally {
await ws.close();
}
} finally {
rmSync(uri, { recursive: true, force: true });
}
}
main().catch((err: unknown) => {
process.stderr.write(String(err) + "\n");
process.exit(1);
});
integ/mongodb.py
-232
View File
@@ -1,232 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import os
from cases import run_not_found, run_provision_probe, run_sed_readonly_probe
from pymongo import AsyncMongoClient
from mirage import MountMode, Workspace
from mirage.resource.mongodb import MongoDBConfig, MongoDBResource
from mirage.types import CommandSafeguard
MONGODB_URI = os.environ.get("MONGODB_URI", "mongodb://localhost:27017")
DB = "mirage_integ"
MOUNT = "/mongodb"
BOOKS = [
{
"_id": 1,
"title": "alpha",
"author": "ada",
"year": 2020,
"tags": ["fiction", "classic"],
"rating": 4.5,
},
{
"_id": 2,
"title": "beta",
"author": "ben",
"year": 2021,
"tags": ["fiction"],
"rating": 3.2,
},
{
"_id": 3,
"title": "gamma",
"author": "cara",
"year": 2022,
"rating": 5.0,
},
{
"_id": 4,
"title": "delta",
"author": "ada",
"year": 2023,
"tags": ["history"],
"rating": 4.0,
},
{
"_id": 5,
"title": "epsilon",
"author": "ben",
"year": 2024,
"rating": 2.5,
},
]
AUTHORS = [
{
"_id": 1,
"name": "ada",
"books": 2
},
{
"_id": 2,
"name": "ben",
"books": 2
},
{
"_id": 3,
"name": "cara",
"books": 1
},
]
CASES: list[tuple[str, str]] = [
("ls_root", f"ls {MOUNT}/"),
("ls_db", f"ls {MOUNT}/{DB}/"),
("ls_collections", f"ls {MOUNT}/{DB}/collections/"),
("ls_views", f"ls {MOUNT}/{DB}/views/"),
("ls_entity", f"ls {MOUNT}/{DB}/collections/books/"),
("tree", f"tree -L 3 {MOUNT}/{DB}/"),
("cat_database_json", f"cat {MOUNT}/{DB}/database.json"),
("cat_schema_json", f"cat {MOUNT}/{DB}/collections/books/schema.json"),
("cat_docs", f"cat {MOUNT}/{DB}/collections/books/documents.jsonl"),
("head_2", f"head -n 2 {MOUNT}/{DB}/collections/books/documents.jsonl"),
("tail_2", f"tail -n 2 {MOUNT}/{DB}/collections/books/documents.jsonl"),
("wc_l_books", f"wc -l {MOUNT}/{DB}/collections/books/documents.jsonl"),
("wc_default_books", f"wc {MOUNT}/{DB}/collections/books/documents.jsonl"),
("wc_l_authors",
f"wc -l {MOUNT}/{DB}/collections/authors/documents.jsonl"),
("stat_docs", f"stat {MOUNT}/{DB}/collections/books/documents.jsonl"),
("grep_c_title",
f"grep -c title {MOUNT}/{DB}/collections/books/documents.jsonl"),
("grep_ada", f"grep ada {MOUNT}/{DB}/collections/books/documents.jsonl"),
("grep_e_multi",
f"grep -n -e ada -e ben {MOUNT}/{DB}/collections/books/documents.jsonl"),
("grep_r_e_multi",
f"grep -r -e alpha -e beta {MOUNT}/{DB}/collections/books"),
("grep_db_scope", f"grep ada {MOUNT}/{DB}/"),
("grep_root_scope", f"grep ada {MOUNT}/"),
("rg_db_scope", f"rg ben {MOUNT}/{DB}/"),
("rg_e_multi", f"rg -e gamma -e cara {MOUNT}/{DB}/collections/books"),
("find_docs", f"find {MOUNT}/{DB}/ -name documents.jsonl"),
("find_schema", f"find {MOUNT}/{DB}/ -name schema.json"),
("jq_titles",
f"jq '.title' {MOUNT}/{DB}/collections/books/documents.jsonl"),
("pipe_grep_c",
f"cat {MOUNT}/{DB}/collections/books/documents.jsonl | grep -c fiction"),
("cat_view_docs", f"cat {MOUNT}/{DB}/views/recent_books/documents.jsonl"),
("wc_l_view", f"wc -l {MOUNT}/{DB}/views/recent_books/documents.jsonl"),
# ----- safeguard: per-mount cap on cat (set to 2 lines below) -----
("safeguard_cat_truncates",
f"cat {MOUNT}/{DB}/collections/books/documents.jsonl"),
("safeguard_cat_pipe_uncapped",
f"cat {MOUNT}/{DB}/collections/books/documents.jsonl | wc -l"),
# symlink to the database meta file (namespace state; read-only backend)
("sym_ln", f"ln -s {MOUNT}/{DB}/database.json {MOUNT}/meta_link"),
("sym_readlink", f"readlink {MOUNT}/meta_link"),
("sym_cat", f"cat {MOUNT}/meta_link"),
("sym_ls", f"ls -F {MOUNT} | grep meta_link"),
("sym_rm", f"rm {MOUNT}/meta_link && ls {MOUNT}"),
]
async def _seed(client: AsyncMongoClient) -> None:
await client.drop_database(DB)
db = client[DB]
await db["books"].insert_many([dict(d) for d in BOOKS])
await db["authors"].insert_many([dict(d) for d in AUTHORS])
await db.create_collection(
"recent_books",
viewOn="books",
pipeline=[{
"$match": {
"year": {
"$gte": 2022
}
}
}],
)
async def _run(ws: Workspace, name: str, cmd: str) -> None:
result = await ws.execute(cmd)
out = await result.stdout_str()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
if name.startswith("safeguard_"):
err = await result.stderr_str()
if err:
print(err, end="" if err.endswith("\n") else "\n")
async def _follow_once(ws: Workspace, cmd: str) -> str:
result = await ws.execute(cmd)
return await result.stdout_str()
async def _insert_until_done(client: AsyncMongoClient,
task: "asyncio.Task[str]") -> None:
i = 0
while not task.done():
i += 1
await client[DB]["books"].insert_one({
"_id": 100 + i,
"title": "live_insert",
})
await asyncio.sleep(0.5)
async def _run_follow(ws: Workspace, client: AsyncMongoClient) -> None:
name = "tail_f_change_stream"
cmd = (f"tail -f {MOUNT}/{DB}/collections/books/documents.jsonl "
"| head -n 1")
task = asyncio.create_task(_follow_once(ws, cmd))
inserter = asyncio.create_task(_insert_until_done(client, task))
try:
out = await asyncio.wait_for(task, timeout=30)
finally:
inserter.cancel()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
def _set_cat_safeguard(ws: Workspace, max_lines: int) -> None:
sg = CommandSafeguard(max_lines=max_lines)
mounts = list(ws._registry._mounts)
for m in mounts:
m.command_safeguards["cat"] = sg
async def main() -> None:
seed_client = AsyncMongoClient(MONGODB_URI)
try:
await _seed(seed_client)
finally:
await seed_client.close()
resource = MongoDBResource(
config=MongoDBConfig(uri=MONGODB_URI, databases=[DB]))
ws = Workspace({MOUNT: resource}, mode=MountMode.READ)
for name, cmd in CASES:
if name == "safeguard_cat_truncates":
_set_cat_safeguard(ws, max_lines=2)
await _run(ws, name, cmd)
await run_not_found(ws, MOUNT)
await run_provision_probe(
ws, f"{MOUNT}/{DB}/collections/books/documents.jsonl")
await run_sed_readonly_probe(
ws, f"{MOUNT}/{DB}/collections/books/documents.jsonl")
live_client = AsyncMongoClient(MONGODB_URI)
try:
await _run_follow(ws, live_client)
finally:
await live_client.close()
if __name__ == "__main__":
asyncio.run(main())
integ/mongodb.ts
-192
View File
@@ -1,192 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { Double, MongoClient } from "mongodb";
import {
CommandSafeguard,
MongoDBResource,
MountMode,
Workspace,
} from "@struktoai/mirage-node";
import { runNotFound, runProvisionProbe, runSedReadonlyProbe } from "./cases.ts";
const MONGODB_URI = process.env.MONGODB_URI ?? "mongodb://localhost:27017";
const DB = "mirage_integ";
const MOUNT = "/mongodb";
const DEC = new TextDecoder();
const BOOKS = [
{ _id: 1, title: "alpha", author: "ada", year: 2020, tags: ["fiction", "classic"], rating: 4.5 },
{ _id: 2, title: "beta", author: "ben", year: 2021, tags: ["fiction"], rating: 3.2 },
{ _id: 3, title: "gamma", author: "cara", year: 2022, rating: 5.0 },
{ _id: 4, title: "delta", author: "ada", year: 2023, tags: ["history"], rating: 4.0 },
{ _id: 5, title: "epsilon", author: "ben", year: 2024, rating: 2.5 },
];
const AUTHORS = [
{ _id: 1, name: "ada", books: 2 },
{ _id: 2, name: "ben", books: 2 },
{ _id: 3, name: "cara", books: 1 },
];
const CASES: ReadonlyArray<readonly [string, string]> = [
["ls_root", `ls ${MOUNT}/`],
["ls_db", `ls ${MOUNT}/${DB}/`],
["ls_collections", `ls ${MOUNT}/${DB}/collections/`],
["ls_views", `ls ${MOUNT}/${DB}/views/`],
["ls_entity", `ls ${MOUNT}/${DB}/collections/books/`],
["tree", `tree -L 3 ${MOUNT}/${DB}/`],
["cat_database_json", `cat ${MOUNT}/${DB}/database.json`],
["cat_schema_json", `cat ${MOUNT}/${DB}/collections/books/schema.json`],
["cat_docs", `cat ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["head_2", `head -n 2 ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["tail_2", `tail -n 2 ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["wc_l_books", `wc -l ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["wc_default_books", `wc ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["wc_l_authors", `wc -l ${MOUNT}/${DB}/collections/authors/documents.jsonl`],
["stat_docs", `stat ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["grep_c_title", `grep -c title ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["grep_ada", `grep ada ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["grep_e_multi", `grep -n -e ada -e ben ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["grep_r_e_multi", `grep -r -e alpha -e beta ${MOUNT}/${DB}/collections/books`],
["grep_db_scope", `grep ada ${MOUNT}/${DB}/`],
["grep_root_scope", `grep ada ${MOUNT}/`],
["rg_db_scope", `rg ben ${MOUNT}/${DB}/`],
["rg_e_multi", `rg -e gamma -e cara ${MOUNT}/${DB}/collections/books`],
["find_docs", `find ${MOUNT}/${DB}/ -name documents.jsonl`],
["find_schema", `find ${MOUNT}/${DB}/ -name schema.json`],
["jq_titles", `jq '.title' ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["pipe_grep_c", `cat ${MOUNT}/${DB}/collections/books/documents.jsonl | grep -c fiction`],
["cat_view_docs", `cat ${MOUNT}/${DB}/views/recent_books/documents.jsonl`],
["wc_l_view", `wc -l ${MOUNT}/${DB}/views/recent_books/documents.jsonl`],
["safeguard_cat_truncates", `cat ${MOUNT}/${DB}/collections/books/documents.jsonl`],
["safeguard_cat_pipe_uncapped", `cat ${MOUNT}/${DB}/collections/books/documents.jsonl | wc -l`],
// symlink to the database meta file (namespace state; read-only backend)
["sym_ln", `ln -s ${MOUNT}/${DB}/database.json ${MOUNT}/meta_link`],
["sym_readlink", `readlink ${MOUNT}/meta_link`],
["sym_cat", `cat ${MOUNT}/meta_link`],
["sym_ls", `ls -F ${MOUNT} | grep meta_link`],
["sym_rm", `rm ${MOUNT}/meta_link && ls ${MOUNT}`],
];
async function seed(client: MongoClient): Promise<void> {
const db = client.db(DB);
await db.dropDatabase();
// Python seeds floats (BSON double); insert Double so the inferred schema
// matches (JS whole numbers would otherwise serialize to BSON int32).
await db
.collection("books")
.insertMany(BOOKS.map((d) => ({ ...d, rating: new Double(d.rating) })));
await db.collection("authors").insertMany(AUTHORS.map((d) => ({ ...d })));
await db.createCollection("recent_books", {
viewOn: "books",
pipeline: [{ $match: { year: { $gte: 2022 } } }],
});
}
async function run(ws: Workspace, name: string, cmd: string): Promise<void> {
const result = await ws.execute(cmd);
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(out.endsWith("\n") ? out : out + "\n");
if (name.startsWith("safeguard_")) {
const err = DEC.decode(result.stderr);
if (err) process.stdout.write(err.endsWith("\n") ? err : err + "\n");
}
}
async function insertUntilDone(
client: MongoClient,
done: () => boolean,
): Promise<void> {
let i = 0;
while (!done()) {
i += 1;
await client
.db(DB)
.collection("books")
.insertOne({ _id: 100 + i, title: "live_insert" });
await new Promise((r) => setTimeout(r, 500));
}
}
async function runFollow(ws: Workspace, client: MongoClient): Promise<void> {
const name = "tail_f_change_stream";
const cmd = `tail -f ${MOUNT}/${DB}/collections/books/documents.jsonl | head -n 1`;
let finished = false;
const followP = ws.execute(cmd).then((result) => {
finished = true;
return DEC.decode(result.stdout);
});
const inserter = insertUntilDone(client, () => finished);
const timeout = new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error("tail -f change stream timed out")), 30000),
);
try {
const out = await Promise.race([followP, timeout]);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(out.endsWith("\n") ? out : out + "\n");
} finally {
finished = true;
await inserter;
}
}
function setCatSafeguard(ws: Workspace, maxLines: number): void {
const sg = new CommandSafeguard({ maxLines });
for (const m of ws.registry.allMounts()) m.commandSafeguards.set("cat", sg);
}
async function main(): Promise<void> {
const seedClient = new MongoClient(MONGODB_URI);
await seedClient.connect();
try {
await seed(seedClient);
} finally {
await seedClient.close();
}
const resource = new MongoDBResource({ uri: MONGODB_URI, databases: [DB] });
const ws = new Workspace({ [MOUNT]: resource }, { mode: MountMode.READ });
try {
for (const [name, cmd] of CASES) {
if (name === "safeguard_cat_truncates") setCatSafeguard(ws, 2);
await run(ws, name, cmd);
}
await runNotFound(ws, MOUNT);
await runProvisionProbe(
ws,
`${MOUNT}/${DB}/collections/books/documents.jsonl`,
);
await runSedReadonlyProbe(
ws,
`${MOUNT}/${DB}/collections/books/documents.jsonl`,
);
const liveClient = new MongoClient(MONGODB_URI);
await liveClient.connect();
try {
await runFollow(ws, liveClient);
} finally {
await liveClient.close();
}
} finally {
await ws.close();
await resource.close();
}
}
main().catch((err: unknown) => {
console.error(err);
process.exit(1);
});
integ/notion_mcp_parity.ts
+94
View File
@@ -0,0 +1,94 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// TS-only check with no golden file: the notion backend is reachable over two
// transports (the REST API and an MCP server), and both must render the exact
// same virtual filesystem. The REST rendering is asserted against the shared
// JSON harness (target `notion`); this asserts the MCP-backed resource is
// byte-identical to the REST one over the same command battery. Python has no
// MCP notion transport, so this cannot live in the cross-language harness.
import { NotionResource as BrowserNotionResource } from '@struktoai/mirage-browser'
import { MemoryOAuthClientProvider } from '@struktoai/mirage-core'
import { MountMode, NotionResource, Workspace } from '@struktoai/mirage-node'
import {
CASES,
EXIT_CODE_CASES,
startMockMcpServer,
startMockServer,
} from './server/notion_server.ts'
const MOUNT = '/notion'
const DEC = new TextDecoder()
async function render(ws: Workspace, cmd: string, withExit: boolean): Promise<string> {
const result = await ws.execute(cmd)
const out = DEC.decode(result.stdout)
const tail = out.endsWith('\n') || out === '' ? out : out + '\n'
return withExit ? `exit=${String(result.exitCode)}\n${tail}` : tail
}
async function main(): Promise<void> {
const { server, port } = await startMockServer()
const { server: mcpServer, port: mcpPort } = await startMockMcpServer()
const restWs = new Workspace(
{ [MOUNT]: new NotionResource({ apiKey: 'integ-test', baseUrl: `http://127.0.0.1:${String(port)}/v1` }) },
{ mode: MountMode.READ },
)
const authProvider = new MemoryOAuthClientProvider({
clientMetadata: { redirect_uris: ['http://127.0.0.1/cb'] },
redirect: () => {},
})
const mcpWs = new Workspace(
{
[MOUNT]: new BrowserNotionResource({
authProvider,
serverUrl: `http://127.0.0.1:${String(mcpPort)}/mcp`,
}),
},
{ mode: MountMode.READ },
)
let mismatches = 0
try {
const all: ReadonlyArray<readonly [string, string, boolean]> = [
...CASES.map(([name, cmd]) => [name, cmd, false] as const),
...EXIT_CODE_CASES.map(([name, cmd]) => [name, cmd, true] as const),
]
for (const [name, cmd, withExit] of all) {
const restOut = await render(restWs, cmd, withExit)
const mcpOut = await render(mcpWs, cmd, withExit)
if (mcpOut !== restOut) {
mismatches += 1
process.stderr.write(
`MCP/REST MISMATCH in ${name}:\n--- rest ---\n${restOut}--- mcp ---\n${mcpOut}`,
)
}
}
const n = String(all.length)
if (mismatches === 0) process.stderr.write(`notion mcp parity: ${n}/${n} cases byte-identical\n`)
} finally {
await restWs.close()
await mcpWs.close()
server.close()
mcpServer.close()
}
// The MCP transport leaves open handles, so exit explicitly rather than
// waiting for the event loop to drain (which would hang the CI step).
process.exit(mismatches > 0 ? 1 : 0)
}
main().catch((err: unknown) => {
console.error(err)
process.exit(1)
})
integ/package.json
+1 -6
View File
@@ -11,13 +11,8 @@
"s3": "tsx s3.ts",
"fuse": "tsx fuse.ts",
"s3_cases": "tsx s3_cases.ts",
"mongodb": "tsx mongodb.ts",
"notion": "tsx notion.ts",
"postgres": "tsx postgres.ts",
"notion:mcp-parity": "tsx notion_mcp_parity.ts",
"safeguard": "tsx safeguard.ts",
"chroma": "tsx chroma.ts",
"lancedb": "tsx lancedb.ts",
"qdrant": "tsx qdrant.ts",
"ssh": "tsx ssh.ts",
"runtime": "tsx runtime.ts",
"slack:server": "tsx server/slack.ts",
integ/postgres.py
-151
View File
@@ -1,151 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import os
import asyncpg
from cases import run_not_found, run_provision_probe, run_sed_readonly_probe
from mirage import MountMode, Workspace
from mirage.resource.postgres import PostgresConfig, PostgresResource
from mirage.types import CommandSafeguard
DSN = os.environ.get("POSTGRES_DSN",
"postgres://mirage:mirage@localhost:55432/mirage_integ")
MOUNT = "/pg"
BOOKS = [
(1, "alpha", "ada", 2020, 4.5),
(2, "beta", "ben", 2021, 3.2),
(3, "gamma", "cara", 2022, 5.0),
(4, "delta", "ada", 2023, 4.0),
(5, "epsilon", "ben", 2024, 2.5),
]
AUTHORS = [
(1, "ada", 2),
(2, "ben", 2),
(3, "cara", 1),
]
CASES: list[tuple[str, str]] = [
("ls_root", f"ls {MOUNT}/"),
("ls_schema", f"ls {MOUNT}/public/"),
("ls_tables", f"ls {MOUNT}/public/tables/"),
("ls_views", f"ls {MOUNT}/public/views/"),
("ls_entity", f"ls {MOUNT}/public/tables/books/"),
("tree", f"tree -L 3 {MOUNT}/public/"),
("cat_schema_json", f"cat {MOUNT}/public/tables/books/schema.json"),
("cat_rows", f"cat {MOUNT}/public/tables/books/rows.jsonl"),
("head_2", f"head -n 2 {MOUNT}/public/tables/books/rows.jsonl"),
("tail_2", f"tail -n 2 {MOUNT}/public/tables/books/rows.jsonl"),
("wc_l_books", f"wc -l {MOUNT}/public/tables/books/rows.jsonl"),
("wc_default_books", f"wc {MOUNT}/public/tables/books/rows.jsonl"),
("wc_l_authors", f"wc -l {MOUNT}/public/tables/authors/rows.jsonl"),
("stat_rows", f"stat {MOUNT}/public/tables/books/rows.jsonl"),
("grep_c_title", f"grep -c title {MOUNT}/public/tables/books/rows.jsonl"),
("grep_ada", f"grep ada {MOUNT}/public/tables/books/rows.jsonl"),
("grep_e_multi",
f"grep -n -e ada -e ben {MOUNT}/public/tables/books/rows.jsonl"),
("rg_e_multi",
f"rg -n -e ada -e ben {MOUNT}/public/tables/books/rows.jsonl"),
("grep_schema_scope", f"grep ada {MOUNT}/public/tables/"),
("rg_schema_scope", f"rg ben {MOUNT}/public/"),
("find_rows", f"find {MOUNT}/public/ -name rows.jsonl"),
("find_schema", f"find {MOUNT}/public/ -name schema.json"),
# rows.jsonl is sizeless (table_size_bytes lives in extra.size_bytes):
# stat prints size 0, -size counts it as 0, wc -c sees rendered bytes.
("stat_size_rows",
f"stat -c '%s %n' {MOUNT}/public/tables/books/rows.jsonl"),
("find_size_plus_rows",
f"find {MOUNT}/public/tables/ -name rows.jsonl -size +1c"),
("find_size_under_rows",
f"find {MOUNT}/public/tables/books/ -name rows.jsonl -size -1k"),
("jq_titles", f"jq '.title' {MOUNT}/public/tables/books/rows.jsonl"),
("pipe_grep_c",
f"cat {MOUNT}/public/tables/books/rows.jsonl | grep -c ada"),
("cat_view_rows", f"cat {MOUNT}/public/views/recent_books/rows.jsonl"),
("wc_l_view", f"wc -l {MOUNT}/public/views/recent_books/rows.jsonl"),
("safeguard_cat_truncates", f"cat {MOUNT}/public/tables/books/rows.jsonl"),
("safeguard_cat_pipe_uncapped",
f"cat {MOUNT}/public/tables/books/rows.jsonl | wc -l"),
# symlink into the mount (namespace state; works on a read-only backend)
("sym_ln", f"ln -s {MOUNT}/public/tables/books/schema.json"
f" {MOUNT}/meta_link"),
("sym_readlink", f"readlink {MOUNT}/meta_link"),
("sym_cat", f"cat {MOUNT}/meta_link"),
("sym_ls", f"ls -F {MOUNT} | grep meta_link"),
("sym_rm", f"rm {MOUNT}/meta_link && ls {MOUNT}"),
]
async def _seed(conn: asyncpg.Connection) -> None:
await conn.execute("DROP VIEW IF EXISTS recent_books")
await conn.execute("DROP TABLE IF EXISTS books")
await conn.execute("DROP TABLE IF EXISTS authors")
await conn.execute("CREATE TABLE books (id int PRIMARY KEY, title text, "
"author text, year int, rating double precision)")
await conn.execute(
"CREATE TABLE authors (id int PRIMARY KEY, name text, books int)")
await conn.executemany(
"INSERT INTO books (id, title, author, year, rating) "
"VALUES ($1, $2, $3, $4, $5)", BOOKS)
await conn.executemany(
"INSERT INTO authors (id, name, books) VALUES ($1, $2, $3)", AUTHORS)
await conn.execute(
"CREATE VIEW recent_books AS SELECT * FROM books WHERE year >= 2022")
await conn.execute("ANALYZE books")
await conn.execute("ANALYZE authors")
async def _run(ws: Workspace, name: str, cmd: str) -> None:
result = await ws.execute(cmd)
out = await result.stdout_str()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
if name.startswith("safeguard_"):
err = await result.stderr_str()
if err:
print(err, end="" if err.endswith("\n") else "\n")
def _set_cat_safeguard(ws: Workspace, max_lines: int) -> None:
sg = CommandSafeguard(max_lines=max_lines)
mounts = list(ws._registry._mounts)
for m in mounts:
m.command_safeguards["cat"] = sg
async def main() -> None:
conn = await asyncpg.connect(DSN)
try:
await _seed(conn)
finally:
await conn.close()
resource = PostgresResource(
config=PostgresConfig(dsn=DSN, max_read_rows=200))
ws = Workspace({MOUNT: resource}, mode=MountMode.READ)
for name, cmd in CASES:
if name == "safeguard_cat_truncates":
_set_cat_safeguard(ws, max_lines=2)
await _run(ws, name, cmd)
await run_not_found(ws, MOUNT)
await run_provision_probe(ws, f"{MOUNT}/public/tables/books/rows.jsonl")
await run_sed_readonly_probe(ws, f"{MOUNT}/public/tables/books/rows.jsonl")
await resource.accessor.close()
if __name__ == "__main__":
asyncio.run(main())
integ/postgres.ts
-155
View File
@@ -1,155 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import pg from "pg";
import {
CommandSafeguard,
MountMode,
PostgresResource,
Workspace,
} from "@struktoai/mirage-node";
import { runNotFound, runProvisionProbe, runSedReadonlyProbe } from "./cases.ts";
const DSN =
process.env.POSTGRES_DSN ?? "postgres://mirage:mirage@localhost:55432/mirage_integ";
const MOUNT = "/pg";
const DEC = new TextDecoder();
const BOOKS: ReadonlyArray<readonly [number, string, string, number, number]> = [
[1, "alpha", "ada", 2020, 4.5],
[2, "beta", "ben", 2021, 3.2],
[3, "gamma", "cara", 2022, 5.0],
[4, "delta", "ada", 2023, 4.0],
[5, "epsilon", "ben", 2024, 2.5],
];
const AUTHORS: ReadonlyArray<readonly [number, string, number]> = [
[1, "ada", 2],
[2, "ben", 2],
[3, "cara", 1],
];
const CASES: ReadonlyArray<readonly [string, string]> = [
["ls_root", `ls ${MOUNT}/`],
["ls_schema", `ls ${MOUNT}/public/`],
["ls_tables", `ls ${MOUNT}/public/tables/`],
["ls_views", `ls ${MOUNT}/public/views/`],
["ls_entity", `ls ${MOUNT}/public/tables/books/`],
["tree", `tree -L 3 ${MOUNT}/public/`],
["cat_schema_json", `cat ${MOUNT}/public/tables/books/schema.json`],
["cat_rows", `cat ${MOUNT}/public/tables/books/rows.jsonl`],
["head_2", `head -n 2 ${MOUNT}/public/tables/books/rows.jsonl`],
["tail_2", `tail -n 2 ${MOUNT}/public/tables/books/rows.jsonl`],
["wc_l_books", `wc -l ${MOUNT}/public/tables/books/rows.jsonl`],
["wc_default_books", `wc ${MOUNT}/public/tables/books/rows.jsonl`],
["wc_l_authors", `wc -l ${MOUNT}/public/tables/authors/rows.jsonl`],
["stat_rows", `stat ${MOUNT}/public/tables/books/rows.jsonl`],
["grep_c_title", `grep -c title ${MOUNT}/public/tables/books/rows.jsonl`],
["grep_ada", `grep ada ${MOUNT}/public/tables/books/rows.jsonl`],
["grep_e_multi", `grep -n -e ada -e ben ${MOUNT}/public/tables/books/rows.jsonl`],
["rg_e_multi", `rg -n -e ada -e ben ${MOUNT}/public/tables/books/rows.jsonl`],
["grep_schema_scope", `grep ada ${MOUNT}/public/tables/`],
["rg_schema_scope", `rg ben ${MOUNT}/public/`],
["find_rows", `find ${MOUNT}/public/ -name rows.jsonl`],
["find_schema", `find ${MOUNT}/public/ -name schema.json`],
// rows.jsonl is sizeless (tableSizeBytes lives in extra.size_bytes):
// stat prints size 0, -size counts it as 0, wc -c sees rendered bytes.
["stat_size_rows", `stat -c '%s %n' ${MOUNT}/public/tables/books/rows.jsonl`],
["find_size_plus_rows", `find ${MOUNT}/public/tables/ -name rows.jsonl -size +1c`],
["find_size_under_rows", `find ${MOUNT}/public/tables/books/ -name rows.jsonl -size -1k`],
["jq_titles", `jq '.title' ${MOUNT}/public/tables/books/rows.jsonl`],
["pipe_grep_c", `cat ${MOUNT}/public/tables/books/rows.jsonl | grep -c ada`],
["cat_view_rows", `cat ${MOUNT}/public/views/recent_books/rows.jsonl`],
["wc_l_view", `wc -l ${MOUNT}/public/views/recent_books/rows.jsonl`],
["safeguard_cat_truncates", `cat ${MOUNT}/public/tables/books/rows.jsonl`],
["safeguard_cat_pipe_uncapped", `cat ${MOUNT}/public/tables/books/rows.jsonl | wc -l`],
// symlink into the mount (namespace state; works on a read-only backend)
["sym_ln", `ln -s ${MOUNT}/public/tables/books/schema.json ${MOUNT}/meta_link`],
["sym_readlink", `readlink ${MOUNT}/meta_link`],
["sym_cat", `cat ${MOUNT}/meta_link`],
["sym_ls", `ls -F ${MOUNT} | grep meta_link`],
["sym_rm", `rm ${MOUNT}/meta_link && ls ${MOUNT}`],
];
async function seed(client: pg.Client): Promise<void> {
await client.query("DROP VIEW IF EXISTS recent_books");
await client.query("DROP TABLE IF EXISTS books");
await client.query("DROP TABLE IF EXISTS authors");
await client.query(
"CREATE TABLE books (id int PRIMARY KEY, title text, author text, year int, rating double precision)",
);
await client.query("CREATE TABLE authors (id int PRIMARY KEY, name text, books int)");
for (const [id, title, author, year, rating] of BOOKS) {
await client.query(
"INSERT INTO books (id, title, author, year, rating) VALUES ($1, $2, $3, $4, $5)",
[id, title, author, year, rating],
);
}
for (const [id, name, books] of AUTHORS) {
await client.query("INSERT INTO authors (id, name, books) VALUES ($1, $2, $3)", [
id,
name,
books,
]);
}
await client.query("CREATE VIEW recent_books AS SELECT * FROM books WHERE year >= 2022");
await client.query("ANALYZE books");
await client.query("ANALYZE authors");
}
async function run(ws: Workspace, name: string, cmd: string): Promise<void> {
const result = await ws.execute(cmd);
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(out.endsWith("\n") ? out : out + "\n");
if (name.startsWith("safeguard_")) {
const err = DEC.decode(result.stderr);
if (err) process.stdout.write(err.endsWith("\n") ? err : err + "\n");
}
}
function setCatSafeguard(ws: Workspace, maxLines: number): void {
const sg = new CommandSafeguard({ maxLines });
for (const m of ws.registry.allMounts()) m.commandSafeguards.set("cat", sg);
}
async function main(): Promise<void> {
const seedClient = new pg.Client({ connectionString: DSN });
await seedClient.connect();
try {
await seed(seedClient);
} finally {
await seedClient.end();
}
const resource = new PostgresResource({ dsn: DSN, maxReadRows: 200 });
const ws = new Workspace({ [MOUNT]: resource }, { mode: MountMode.READ });
try {
for (const [name, cmd] of CASES) {
if (name === "safeguard_cat_truncates") setCatSafeguard(ws, 2);
await run(ws, name, cmd);
}
await runNotFound(ws, MOUNT);
await runProvisionProbe(ws, `${MOUNT}/public/tables/books/rows.jsonl`);
await runSedReadonlyProbe(ws, `${MOUNT}/public/tables/books/rows.jsonl`);
} finally {
await ws.close();
await resource.close();
}
}
main().catch((err: unknown) => {
console.error(err);
process.exit(1);
});
integ/qdrant.py
-185
View File
@@ -1,185 +0,0 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import os
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from qdrant_client import AsyncQdrantClient, models # noqa: E402
from mirage import MountMode, Workspace # noqa: E402
from mirage.resource.qdrant import QdrantConfig, QdrantResource # noqa: E402
QDRANT_URL = os.environ.get("QDRANT_URL")
QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY")
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost")
QDRANT_PORT = int(os.environ.get("QDRANT_PORT", "6333"))
EMBED_DIM = 8
COLLECTION = "mirage_integ"
MOUNT = "/db/"
ROWS = [
(1, "cat", "big", "a big orange cat"),
(2, "cat", "small", "a small grey cat"),
(3, "dog", "big", "a big brown dog"),
(4, "dog", "small", "a small white dog"),
]
CASES: list[tuple[str, str]] = [
("ls_root", "ls {root}"),
("ls_group", "ls {root}cat"),
("ls_leaf", "ls {root}cat/big"),
("find_txt", "find {root} -name '*.txt'"),
("find_json", "find {root} -name '*.json'"),
("cat_txt", "cat {root}cat/big/1.txt"),
("cat_json", "cat {root}cat/big/1.json"),
("wc_c_txt", "wc -c {root}cat/big/1.txt"),
("grep_text", "grep orange {root}cat/big/1.txt"),
("grep_json_field", "grep label {root}cat/big/1.json"),
("grep_i", "grep -i ORANGE {root}cat/big/1.txt"),
("grep_n", "grep -n cat {root}cat/big/1.json"),
("grep_c", "grep -c cat {root}cat/big/1.json"),
("grep_o", "grep -o cat {root}cat/big/1.json"),
("grep_w", "grep -w big {root}cat/big/1.json"),
("grep_F_literal", "grep -F 'orange cat' {root}cat/big/1.txt"),
("grep_E_alt", 'grep -E "orange|brown" {root}cat/big/1.txt'),
("grep_v", "grep -v zebra {root}cat/big/1.txt"),
("grep_multi", "grep small {root}cat/small/2.json {root}dog/small/4.json"),
("grep_r_group", "grep -r orange {root}cat"),
("grep_rl", "grep -rl cat {root}"),
("pipe_grep_stdin", "cat {root}cat/big/1.json | grep orange"),
("rg_basic", "rg orange {root}cat/big/1.txt"),
# du has no native op -> exercises the stat/readdir walk fallback,
# which must match the Python du builder byte for byte.
("du_leaf", "du {root}cat/big"),
("du_group", "du {root}cat"),
("du_root", "du {root}"),
("du_c_multi", "du -c {root}cat {root}dog"),
# symlink into the mount (namespace state; works on a read-only backend)
("sym_ln", "ln -s {root}cat/big/1.json {root}meta_link"),
("sym_readlink", "readlink {root}meta_link"),
("sym_cat", "cat {root}meta_link"),
("sym_grep", "grep label {root}meta_link"),
("sym_ls", "ls -F {root} | grep meta_link"),
("sym_rm", "rm {root}meta_link && ls {root}"),
]
EXIT_CODE_CASES: list[tuple[str, str]] = [
("grep_q_match", "grep -q cat {root}cat/big/1.txt"),
("grep_q_no_match", "grep -q zebra {root}cat/big/1.txt"),
("grep_no_match", "grep zebra {root}cat/big/1.txt"),
]
async def run_cases(ws: Workspace) -> None:
for name, tmpl in CASES:
result = await ws.execute(tmpl.format(root=MOUNT))
out = await result.stdout_str()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
for name, tmpl in EXIT_CODE_CASES:
result = await ws.execute(tmpl.format(root=MOUNT))
out = await result.stdout_str()
print(f"=== {name} ===")
print(f"exit={result.exit_code}")
if out:
print(out, end="" if out.endswith("\n") else "\n")
nf_target = f"{MOUNT.rstrip('/')}/cat/big/__nf_missing__.json"
for nf_name, nf_prog in (("nf_cat", "cat"), ("nf_head", "head"),
("nf_tail", "tail"), ("nf_wc", "wc"),
("nf_stat", "stat"), ("nf_grep", "grep x")):
result = await ws.execute(f"{nf_prog} {nf_target}")
err = (await result.stderr_str()).strip()
print(f"=== {nf_name} ===")
print(f"exit={result.exit_code}")
if err:
print(err)
prov_target = f"{MOUNT}cat/big/1.txt"
for pv_name, pv_cmd in (("prov_probe_cat", f"cat {prov_target}"),
("prov_probe_grep", f"grep x {prov_target}"),
("prov_probe_ls", f"ls {MOUNT}cat/big")):
result = await ws.execute(pv_cmd, provision=True)
print(f"=== {pv_name} ===")
print(f"net={result.network_read} write={result.network_write} "
f"cache={result.cache_read} ops={result.read_ops} "
f"hits={result.cache_hits} "
f"precision={result.precision.value}")
result = await ws.execute(f"sed -n 1p {prov_target}")
out = await result.stdout_str()
print("=== sed_stream_1p ===")
print(out, end="" if out.endswith("\n") else "\n")
result = await ws.execute(f"sed -i s/x/y/ {prov_target}")
err = (await result.stderr_str()).strip()
print("=== sed_i_readonly ===")
print(f"exit={result.exit_code}")
if err:
print(err)
def _client() -> AsyncQdrantClient:
if QDRANT_URL:
return AsyncQdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
return AsyncQdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
async def seed(client: AsyncQdrantClient) -> None:
await client.delete_collection(COLLECTION)
await client.create_collection(COLLECTION,
vectors_config=models.VectorParams(
size=EMBED_DIM,
distance=models.Distance.COSINE))
await client.upsert(COLLECTION,
points=[
models.PointStruct(id=i,
vector=[0.1] * EMBED_DIM,
payload={
"label": label,
"kind": kind,
"name": name
})
for i, label, kind, name in ROWS
])
for field in ("label", "kind"):
await client.create_payload_index(
COLLECTION,
field_name=field,
field_schema=models.PayloadSchemaType.KEYWORD)
await asyncio.sleep(2)
async def main() -> None:
client = _client()
try:
await seed(client)
config = QdrantConfig(
url=QDRANT_URL,
api_key=QDRANT_API_KEY,
host=QDRANT_HOST,
port=QDRANT_PORT,
collection=COLLECTION,
group_by=["label", "kind"],
id_field="id",
text_field="name",
)
ws = Workspace({MOUNT: QdrantResource(config)}, mode=MountMode.READ)
await run_cases(ws)
finally:
await client.delete_collection(COLLECTION)
if __name__ == "__main__":
asyncio.run(main())
integ/qdrant.ts
-198
View File
@@ -1,198 +0,0 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { QdrantClient } from "@qdrant/js-client-rest";
import { QdrantResource, MountMode, Workspace } from "@struktoai/mirage-node";
const QDRANT_URL = process.env.QDRANT_URL;
const QDRANT_API_KEY = process.env.QDRANT_API_KEY;
const QDRANT_HOST = process.env.QDRANT_HOST ?? "localhost";
const QDRANT_PORT = Number(process.env.QDRANT_PORT ?? "6333");
const EMBED_DIM = 8;
const COLLECTION = "mirage_integ";
const MOUNT = "/db/";
const DEC = new TextDecoder();
const ROWS: [number, string, string, string][] = [
[1, "cat", "big", "a big orange cat"],
[2, "cat", "small", "a small grey cat"],
[3, "dog", "big", "a big brown dog"],
[4, "dog", "small", "a small white dog"],
];
const CASES: [string, string][] = [
["ls_root", "ls {root}"],
["ls_group", "ls {root}cat"],
["ls_leaf", "ls {root}cat/big"],
["find_txt", "find {root} -name '*.txt'"],
["find_json", "find {root} -name '*.json'"],
["cat_txt", "cat {root}cat/big/1.txt"],
["cat_json", "cat {root}cat/big/1.json"],
["wc_c_txt", "wc -c {root}cat/big/1.txt"],
["grep_text", "grep orange {root}cat/big/1.txt"],
["grep_json_field", "grep label {root}cat/big/1.json"],
["grep_i", "grep -i ORANGE {root}cat/big/1.txt"],
["grep_n", "grep -n cat {root}cat/big/1.json"],
["grep_c", "grep -c cat {root}cat/big/1.json"],
["grep_o", "grep -o cat {root}cat/big/1.json"],
["grep_w", "grep -w big {root}cat/big/1.json"],
["grep_F_literal", "grep -F 'orange cat' {root}cat/big/1.txt"],
["grep_E_alt", 'grep -E "orange|brown" {root}cat/big/1.txt'],
["grep_v", "grep -v zebra {root}cat/big/1.txt"],
["grep_multi", "grep small {root}cat/small/2.json {root}dog/small/4.json"],
["grep_r_group", "grep -r orange {root}cat"],
["grep_rl", "grep -rl cat {root}"],
["pipe_grep_stdin", "cat {root}cat/big/1.json | grep orange"],
["rg_basic", "rg orange {root}cat/big/1.txt"],
// du has no native op -> exercises the stat/readdir walk fallback,
// which must match the Python du builder byte for byte.
["du_leaf", "du {root}cat/big"],
["du_group", "du {root}cat"],
["du_root", "du {root}"],
["du_c_multi", "du -c {root}cat {root}dog"],
// symlink into the mount (namespace state; works on a read-only backend)
["sym_ln", "ln -s {root}cat/big/1.json {root}meta_link"],
["sym_readlink", "readlink {root}meta_link"],
["sym_cat", "cat {root}meta_link"],
["sym_grep", "grep label {root}meta_link"],
["sym_ls", "ls -F {root} | grep meta_link"],
["sym_rm", "rm {root}meta_link && ls {root}"],
];
const EXIT_CODE_CASES: [string, string][] = [
["grep_q_match", "grep -q cat {root}cat/big/1.txt"],
["grep_q_no_match", "grep -q zebra {root}cat/big/1.txt"],
["grep_no_match", "grep zebra {root}cat/big/1.txt"],
];
const NOT_FOUND_PROGS: [string, string][] = [
["nf_cat", "cat"],
["nf_head", "head"],
["nf_tail", "tail"],
["nf_wc", "wc"],
["nf_stat", "stat"],
["nf_grep", "grep x"],
];
async function runCases(ws: Workspace): Promise<void> {
for (const [name, tmpl] of CASES) {
const result = await ws.execute(tmpl.replaceAll("{root}", MOUNT));
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(out.endsWith("\n") ? out : out + "\n");
}
for (const [name, tmpl] of EXIT_CODE_CASES) {
const result = await ws.execute(tmpl.replaceAll("{root}", MOUNT));
const out = DEC.decode(result.stdout);
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(`exit=${result.exitCode}\n`);
if (out) process.stdout.write(out.endsWith("\n") ? out : out + "\n");
}
const target = `${MOUNT.replace(/\/+$/, "")}/cat/big/__nf_missing__.json`;
for (const [name, prog] of NOT_FOUND_PROGS) {
const result = await ws.execute(`${prog} ${target}`);
const err = DEC.decode(result.stderr).trim();
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(`exit=${result.exitCode}\n`);
if (err) process.stdout.write(err + "\n");
}
const provTarget = `${MOUNT}cat/big/1.txt`;
const provProbes: ReadonlyArray<readonly [string, string]> = [
["prov_probe_cat", `cat ${provTarget}`],
["prov_probe_grep", `grep x ${provTarget}`],
["prov_probe_ls", `ls ${MOUNT}cat/big`],
];
for (const [name, cmd] of provProbes) {
const result = await ws.execute(cmd, { provision: true });
process.stdout.write(`=== ${name} ===\n`);
process.stdout.write(
`net=${result.networkRead} write=${result.networkWrite} ` +
`cache=${result.cacheRead} ops=${String(result.readOps)} ` +
`hits=${String(result.cacheHits)} precision=${result.precision}\n`,
);
}
let result = await ws.execute(`sed -n 1p ${provTarget}`);
const out = new TextDecoder().decode(result.stdout);
process.stdout.write(`=== sed_stream_1p ===\n`);
process.stdout.write(out.endsWith("\n") ? out : `${out}\n`);
result = await ws.execute(`sed -i s/x/y/ ${provTarget}`);
const err = new TextDecoder().decode(result.stderr).trim();
process.stdout.write(`=== sed_i_readonly ===\n`);
process.stdout.write(`exit=${String(result.exitCode)}\n`);
if (err) process.stdout.write(`${err}\n`);
}
function client(): QdrantClient {
if (QDRANT_URL !== undefined && QDRANT_URL !== "") {
return new QdrantClient({ url: QDRANT_URL, apiKey: QDRANT_API_KEY });
}
return new QdrantClient({ host: QDRANT_HOST, port: QDRANT_PORT });
}
async function seed(c: QdrantClient): Promise<void> {
await c.deleteCollection(COLLECTION).catch(() => {});
await c.createCollection(COLLECTION, {
vectors: { size: EMBED_DIM, distance: "Cosine" },
});
await c.upsert(COLLECTION, {
points: ROWS.map(([id, label, kind, name]) => ({
id,
vector: Array(EMBED_DIM).fill(0.1),
payload: { label, kind, name },
})),
});
for (const field of ["label", "kind"]) {
await c.createPayloadIndex(COLLECTION, {
field_name: field,
field_schema: "keyword",
});
}
await new Promise((r) => setTimeout(r, 2000));
}
async function main(): Promise<void> {
const c = client();
try {
await seed(c);
const ws = new Workspace(
{
[MOUNT]: new QdrantResource({
url: QDRANT_URL,
apiKey: QDRANT_API_KEY,
host: QDRANT_HOST,
port: QDRANT_PORT,
collection: COLLECTION,
groupBy: ["label", "kind"],
idField: "id",
textField: "name",
}),
},
{ mode: MountMode.READ },
);
try {
await runCases(ws);
} finally {
await ws.close();
}
} finally {
await c.deleteCollection(COLLECTION).catch(() => {});
}
}
main().catch((err: unknown) => {
process.stderr.write(String(err) + "\n");
process.exit(1);
});
integ/resources/chroma/ops.json
+615
View File
@@ -0,0 +1,615 @@
{
"cases": [
{
"id": "chroma_ls",
"seq": 630000,
"targets": [
"chroma"
],
"command": "ls /knowledge/",
"expect": {
"exit": 0,
"stdout": "CHANGELOG.md\nguides\npolicies\n",
"stderr": ""
}
},
{
"id": "chroma_ls_guides",
"seq": 630001,
"targets": [
"chroma"
],
"command": "ls /knowledge/guides/",
"expect": {
"exit": 0,
"stdout": "auth.md\nquickstart.md\n",
"stderr": ""
}
},
{
"id": "chroma_find_md",
"seq": 630002,
"targets": [
"chroma"
],
"command": "find /knowledge/ -name '*.md'",
"expect": {
"exit": 0,
"stdout": "/knowledge/CHANGELOG.md\n/knowledge/guides/auth.md\n/knowledge/guides/quickstart.md\n/knowledge/policies/archived.md\n/knowledge/policies/privacy.md\n/knowledge/policies/refunds.md\n",
"stderr": ""
}
},
{
"id": "chroma_find_type_f",
"seq": 630003,
"targets": [
"chroma"
],
"command": "find /knowledge/ -type f | sort",
"expect": {
"exit": 0,
"stdout": "/knowledge/CHANGELOG.md\n/knowledge/guides/auth.md\n/knowledge/guides/quickstart.md\n/knowledge/policies/archived.md\n/knowledge/policies/privacy.md\n/knowledge/policies/refunds.md\n",
"stderr": ""
}
},
{
"id": "chroma_find_root_maxdepth0",
"seq": 630004,
"targets": [
"chroma"
],
"command": "find /knowledge/ -maxdepth 0",
"expect": {
"exit": 0,
"stdout": "/knowledge\n",
"stderr": ""
}
},
{
"id": "chroma_find_root_name",
"seq": 630005,
"targets": [
"chroma"
],
"command": "find /knowledge/ -name knowledge",
"expect": {
"exit": 0,
"stdout": "/knowledge\n",
"stderr": ""
}
},
{
"id": "chroma_find_size_plus_100c",
"seq": 630006,
"targets": [
"chroma"
],
"command": "find /knowledge/ -type f -size +100c | sort",
"expect": {
"exit": 0,
"stdout": "/knowledge/guides/auth.md\n/knowledge/guides/quickstart.md\n/knowledge/policies/privacy.md\n/knowledge/policies/refunds.md\n",
"stderr": ""
}
},
{
"id": "chroma_find_size_archived_kept",
"seq": 630007,
"targets": [
"chroma"
],
"command": "find /knowledge/policies/ -name 'archived*' -size -1k",
"expect": {
"exit": 0,
"stdout": "/knowledge/policies/archived.md\n",
"stderr": ""
}
},
{
"id": "chroma_grep_cold_single",
"seq": 630008,
"targets": [
"chroma"
],
"command": "grep bearer /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\n",
"stderr": ""
}
},
{
"id": "chroma_grep_warm_single",
"seq": 630009,
"targets": [
"chroma"
],
"command": "grep bearer /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\n",
"stderr": ""
}
},
{
"id": "chroma_cat_auth",
"seq": 630010,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\nRequests are rate limited to 100 calls per minute per token.\nIf you exceed the limit you receive HTTP 429 and must back off.",
"stderr": ""
}
},
{
"id": "chroma_cat_quickstart",
"seq": 630011,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "Welcome to Acme. This quickstart gets you running fast.\nInstall the CLI with npm i -g acme then run acme login.\nSet your token in the ACME_TOKEN environment variable.",
"stderr": ""
}
},
{
"id": "chroma_head_1",
"seq": 630012,
"targets": [
"chroma"
],
"command": "head -n 1 /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "Welcome to Acme. This quickstart gets you running fast.\n",
"stderr": ""
}
},
{
"id": "chroma_tail_1",
"seq": 630013,
"targets": [
"chroma"
],
"command": "tail -n 1 /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "Set your token in the ACME_TOKEN environment variable.",
"stderr": ""
}
},
{
"id": "chroma_grep_429",
"seq": 630014,
"targets": [
"chroma"
],
"command": "grep 429 /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "If you exceed the limit you receive HTTP 429 and must back off.\n",
"stderr": ""
}
},
{
"id": "chroma_grep_e_multi",
"seq": 630015,
"targets": [
"chroma"
],
"command": "grep -e bearer -e 429 /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\nIf you exceed the limit you receive HTTP 429 and must back off.\n",
"stderr": ""
}
},
{
"id": "chroma_grep_c_rate",
"seq": 630016,
"targets": [
"chroma"
],
"command": "grep -c rate /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "1\n",
"stderr": ""
}
},
{
"id": "chroma_grep_r_refund",
"seq": 630017,
"targets": [
"chroma"
],
"command": "grep -r refund /knowledge/policies/",
"expect": {
"exit": 0,
"stdout": "/knowledge/policies/refunds.md:Email support to start a refund with your order id.\n/knowledge/policies/refunds.md:Approved refunds are processed within five business days.\n",
"stderr": ""
}
},
{
"id": "chroma_grep_cold_count",
"seq": 630018,
"targets": [
"chroma"
],
"command": "grep -c sell /knowledge/policies/privacy.md",
"expect": {
"exit": 0,
"stdout": "1\n",
"stderr": ""
}
},
{
"id": "chroma_grep_warm_count",
"seq": 630019,
"targets": [
"chroma"
],
"command": "grep -c sell /knowledge/policies/privacy.md",
"expect": {
"exit": 0,
"stdout": "1\n",
"stderr": ""
}
},
{
"id": "chroma_grep_rl_encrypted",
"seq": 630020,
"targets": [
"chroma"
],
"command": "grep -rl encrypted /knowledge/",
"expect": {
"exit": 0,
"stdout": "/knowledge/policies/privacy.md\n",
"stderr": ""
}
},
{
"id": "chroma_grep_v_bearer",
"seq": 630021,
"targets": [
"chroma"
],
"command": "grep -v bearer /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Requests are rate limited to 100 calls per minute per token.\nIf you exceed the limit you receive HTTP 429 and must back off.\n",
"stderr": ""
}
},
{
"id": "chroma_grep_rE_alternation",
"seq": 630022,
"targets": [
"chroma"
],
"command": "grep -rE \"rate limited|refund\" /knowledge/",
"expect": {
"exit": 0,
"stdout": "/knowledge/CHANGELOG.md:v2.0 added rate limit headers and refund automation.\n/knowledge/guides/auth.md:Requests are rate limited to 100 calls per minute per token.\n/knowledge/policies/refunds.md:Email support to start a refund with your order id.\n/knowledge/policies/refunds.md:Approved refunds are processed within five business days.\n",
"stderr": ""
}
},
{
"id": "chroma_wc_l_auth",
"seq": 630023,
"targets": [
"chroma"
],
"command": "wc -l /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "2 /knowledge/guides/auth.md\n",
"stderr": ""
}
},
{
"id": "chroma_sort_auth",
"seq": 630024,
"targets": [
"chroma"
],
"command": "sort /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\nIf you exceed the limit you receive HTTP 429 and must back off.\nRequests are rate limited to 100 calls per minute per token.\n",
"stderr": ""
}
},
{
"id": "chroma_uniq_auth",
"seq": 630025,
"targets": [
"chroma"
],
"command": "uniq /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\nRequests are rate limited to 100 calls per minute per token.\nIf you exceed the limit you receive HTTP 429 and must back off.\n",
"stderr": ""
}
},
{
"id": "chroma_uniq_w0_auth",
"seq": 630026,
"targets": [
"chroma"
],
"command": "uniq -w 0 /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\n",
"stderr": ""
}
},
{
"id": "chroma_stat_name_auth",
"seq": 630027,
"targets": [
"chroma"
],
"command": "stat -c \"%n\" /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "/knowledge/guides/auth.md\n",
"stderr": ""
}
},
{
"id": "chroma_cut_d_f1",
"seq": 630028,
"targets": [
"chroma"
],
"command": "cut -d ' ' -f 1 /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "Welcome\nInstall\nSet\n",
"stderr": ""
}
},
{
"id": "chroma_awk_first_word",
"seq": 630029,
"targets": [
"chroma"
],
"command": "awk '{{print $1}}' /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "chroma_sed_upper_acme",
"seq": 630030,
"targets": [
"chroma"
],
"command": "sed s/Acme/ACME/ /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "Welcome to ACME. This quickstart gets you running fast.\nInstall the CLI with npm i -g acme then run acme login.\nSet your token in the ACME_TOKEN environment variable.",
"stderr": ""
}
},
{
"id": "chroma_rg_l_token",
"seq": 630031,
"targets": [
"chroma"
],
"command": "rg -l token /knowledge/",
"expect": {
"exit": 0,
"stdout": "/knowledge/guides/auth.md\n/knowledge/guides/quickstart.md\n",
"stderr": ""
}
},
{
"id": "chroma_pipe_cat_wc",
"seq": 630032,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/auth.md | wc -l",
"expect": {
"exit": 0,
"stdout": "2\n",
"stderr": ""
}
},
{
"id": "chroma_pipe_sort_uniq_wc",
"seq": 630033,
"targets": [
"chroma"
],
"command": "cat /knowledge/policies/refunds.md | sort | uniq | wc -l",
"expect": {
"exit": 0,
"stdout": "3\n",
"stderr": ""
}
},
{
"id": "chroma_poison_concat",
"seq": 630034,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/quickstart.md /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Welcome to Acme. This quickstart gets you running fast.\nInstall the CLI with npm i -g acme then run acme login.\nSet your token in the ACME_TOKEN environment variable.Authentication uses bearer tokens via the Authorization header.\nRequests are rate limited to 100 calls per minute per token.\nIf you exceed the limit you receive HTTP 429 and must back off.",
"stderr": ""
}
},
{
"id": "chroma_poison_first_intact",
"seq": 630035,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/quickstart.md",
"expect": {
"exit": 0,
"stdout": "Welcome to Acme. This quickstart gets you running fast.\nInstall the CLI with npm i -g acme then run acme login.\nSet your token in the ACME_TOKEN environment variable.",
"stderr": ""
}
},
{
"id": "chroma_poison_second_intact",
"seq": 630036,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\nRequests are rate limited to 100 calls per minute per token.\nIf you exceed the limit you receive HTTP 429 and must back off.",
"stderr": ""
}
},
{
"id": "chroma_pipe_concat_head",
"seq": 630037,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/quickstart.md /knowledge/guides/auth.md | head -n 1",
"expect": {
"exit": 0,
"stdout": "Welcome to Acme. This quickstart gets you running fast.\n",
"stderr": ""
}
},
{
"id": "chroma_du_guides",
"seq": 630038,
"targets": [
"chroma"
],
"command": "du /knowledge/guides",
"expect": {
"exit": 0,
"stdout": "370\t/knowledge/guides\n",
"stderr": ""
}
},
{
"id": "chroma_du_root",
"seq": 630039,
"targets": [
"chroma"
],
"command": "du /knowledge/",
"expect": {
"exit": 0,
"stdout": "370\t/knowledge/guides\n270\t/knowledge/policies\n730\t/knowledge\n",
"stderr": ""
}
},
{
"id": "chroma_du_c_multi",
"seq": 630040,
"targets": [
"chroma"
],
"command": "du -c /knowledge/guides /knowledge/policies",
"expect": {
"exit": 0,
"stdout": "370\t/knowledge/guides\n270\t/knowledge/policies\n640\ttotal\n",
"stderr": ""
}
},
{
"id": "chroma_sym_ln",
"seq": 630041,
"targets": [
"chroma"
],
"command": "ln -s /knowledge/guides/auth.md /knowledge/meta_link",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "chroma_sym_readlink",
"seq": 630042,
"targets": [
"chroma"
],
"command": "readlink /knowledge/meta_link",
"expect": {
"exit": 0,
"stdout": "/knowledge/guides/auth.md\n",
"stderr": ""
}
},
{
"id": "chroma_sym_cat",
"seq": 630043,
"targets": [
"chroma"
],
"command": "cat /knowledge/meta_link",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\nRequests are rate limited to 100 calls per minute per token.\nIf you exceed the limit you receive HTTP 429 and must back off.",
"stderr": ""
}
},
{
"id": "chroma_sym_wc",
"seq": 630044,
"targets": [
"chroma"
],
"command": "wc -l /knowledge/meta_link",
"expect": {
"exit": 0,
"stdout": "2 /knowledge/meta_link\n",
"stderr": ""
}
},
{
"id": "chroma_sym_ls",
"seq": 630045,
"targets": [
"chroma"
],
"command": "ls -F /knowledge/ | grep meta_link",
"expect": {
"exit": 0,
"stdout": "meta_link@\n",
"stderr": ""
}
},
{
"id": "chroma_sym_rm",
"seq": 630046,
"targets": [
"chroma"
],
"command": "rm /knowledge/meta_link && ls /knowledge/",
"expect": {
"exit": 0,
"stdout": "CHANGELOG.md\nguides\npolicies\n",
"stderr": ""
}
}
]
}
integ/resources/chroma/probe.json
+98
View File
@@ -0,0 +1,98 @@
{
"cases": [
{
"id": "chroma_cat_missing",
"seq": 630047,
"targets": [
"chroma"
],
"command": "cat /knowledge/__nf_missing__.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "cat: /knowledge/__nf_missing__.txt: No such file or directory\n"
}
},
{
"id": "chroma_write_rejected",
"seq": 630048,
"targets": [
"chroma"
],
"command": "echo hi > /knowledge/guides/auth.md",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "/knowledge/guides/auth.md: Operation not supported\n"
}
},
{
"id": "chroma_sed_stream_1p",
"seq": 630049,
"targets": [
"chroma"
],
"command": "sed -n 1p /knowledge/guides/auth.md",
"expect": {
"exit": 0,
"stdout": "Authentication uses bearer tokens via the Authorization header.\n",
"stderr": ""
}
},
{
"id": "chroma_sed_i_readonly",
"seq": 630050,
"targets": [
"chroma"
],
"command": "sed -i s/x/y/ /knowledge/guides/auth.md",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "sed: -i not supported on this backend: Permission denied\n"
}
},
{
"id": "chroma_prov_cat",
"seq": 630051,
"targets": [
"chroma"
],
"command": "cat /knowledge/guides/auth.md",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=190 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
},
{
"id": "chroma_prov_grep",
"seq": 630052,
"targets": [
"chroma"
],
"command": "grep x /knowledge/guides/auth.md",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=190 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
},
{
"id": "chroma_prov_ls",
"seq": 630053,
"targets": [
"chroma"
],
"command": "ls /knowledge/guides",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
}
]
}
integ/resources/lancedb/ops.json
+433
View File
@@ -0,0 +1,433 @@
{
"cases": [
{
"id": "lancedb_ls_root",
"seq": 650000,
"targets": [
"lancedb"
],
"command": "ls /db/",
"expect": {
"exit": 0,
"stdout": "animals\n",
"stderr": ""
}
},
{
"id": "lancedb_ls_table",
"seq": 650001,
"targets": [
"lancedb"
],
"command": "ls /db/animals",
"expect": {
"exit": 0,
"stdout": "cat\ndog\n",
"stderr": ""
}
},
{
"id": "lancedb_ls_group",
"seq": 650002,
"targets": [
"lancedb"
],
"command": "ls /db/animals/cat",
"expect": {
"exit": 0,
"stdout": "big\nsmall\n",
"stderr": ""
}
},
{
"id": "lancedb_find_md",
"seq": 650003,
"targets": [
"lancedb"
],
"command": "find /db/animals -name '*.md'",
"expect": {
"exit": 0,
"stdout": "/db/animals/cat/big/1.md\n/db/animals/cat/small/2.md\n/db/animals/dog/big/3.md\n/db/animals/dog/small/4.md\n",
"stderr": ""
}
},
{
"id": "lancedb_cat_card",
"seq": 650004,
"targets": [
"lancedb"
],
"command": "cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\n\nid: 1\nlabel: cat\nkind: big\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_wc_c_card",
"seq": 650005,
"targets": [
"lancedb"
],
"command": "wc -c /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "70 /db/animals/cat/big/1.md\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_cold_single",
"seq": 650006,
"targets": [
"lancedb"
],
"command": "grep orange /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_warm_single",
"seq": 650007,
"targets": [
"lancedb"
],
"command": "grep orange /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_i",
"seq": 650008,
"targets": [
"lancedb"
],
"command": "grep -i ORANGE /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_n",
"seq": 650009,
"targets": [
"lancedb"
],
"command": "grep -n label /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "4:label: cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_v",
"seq": 650010,
"targets": [
"lancedb"
],
"command": "grep -v cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "\nid: 1\nkind: big\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_c",
"seq": 650011,
"targets": [
"lancedb"
],
"command": "grep -c cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "3\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_o",
"seq": 650012,
"targets": [
"lancedb"
],
"command": "grep -o cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "cat\ncat\ncat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_w",
"seq": 650013,
"targets": [
"lancedb"
],
"command": "grep -w cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nlabel: cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_F_literal",
"seq": 650014,
"targets": [
"lancedb"
],
"command": "grep -F \"id: 1\" /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "id: 1\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_m1",
"seq": 650015,
"targets": [
"lancedb"
],
"command": "grep -m 1 cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_A1",
"seq": 650016,
"targets": [
"lancedb"
],
"command": "grep -A 1 id /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "id: 1\nlabel: cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_B1",
"seq": 650017,
"targets": [
"lancedb"
],
"command": "grep -B 1 label /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "id: 1\nlabel: cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_C1",
"seq": 650018,
"targets": [
"lancedb"
],
"command": "grep -C 1 label /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "id: 1\nlabel: cat\nkind: big\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_multi",
"seq": 650019,
"targets": [
"lancedb"
],
"command": "grep small /db/animals/cat/small/2.md /db/animals/dog/small/4.md",
"expect": {
"exit": 0,
"stdout": "/db/animals/cat/small/2.md:# a small grey cat\n/db/animals/cat/small/2.md:kind: small\n/db/animals/cat/small/2.md:name: a small grey cat\n/db/animals/dog/small/4.md:# a small white dog\n/db/animals/dog/small/4.md:kind: small\n/db/animals/dog/small/4.md:name: a small white dog\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_r_table",
"seq": 650020,
"targets": [
"lancedb"
],
"command": "grep -r orange /db/animals",
"expect": {
"exit": 0,
"stdout": "/db/animals/cat/big/1.md:# a big orange cat\n/db/animals/cat/big/1.md:name: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_r_multipath",
"seq": 650021,
"targets": [
"lancedb"
],
"command": "grep -r small /db/animals/cat /db/animals/dog",
"expect": {
"exit": 0,
"stdout": "/db/animals/cat/small/2.md:# a small grey cat\n/db/animals/cat/small/2.md:kind: small\n/db/animals/cat/small/2.md:name: a small grey cat\n/db/animals/dog/small/4.md:# a small white dog\n/db/animals/dog/small/4.md:kind: small\n/db/animals/dog/small/4.md:name: a small white dog\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_rl",
"seq": 650022,
"targets": [
"lancedb"
],
"command": "grep -rl cat /db/animals",
"expect": {
"exit": 0,
"stdout": "/db/animals/cat/big/1.md\n/db/animals/cat/small/2.md\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_E_alt",
"seq": 650023,
"targets": [
"lancedb"
],
"command": "grep -E \"orange|brown\" /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_pipe_grep_stdin",
"seq": 650024,
"targets": [
"lancedb"
],
"command": "cat /db/animals/cat/big/1.md | grep orange",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_rg_basic",
"seq": 650025,
"targets": [
"lancedb"
],
"command": "rg orange /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\nname: a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_du_file",
"seq": 650026,
"targets": [
"lancedb"
],
"command": "du /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "70\t/db/animals/cat/big/1.md\n",
"stderr": ""
}
},
{
"id": "lancedb_du_group",
"seq": 650027,
"targets": [
"lancedb"
],
"command": "du /db/animals/cat",
"expect": {
"exit": 0,
"stdout": "70\t/db/animals/cat/big\n72\t/db/animals/cat/small\n142\t/db/animals/cat\n",
"stderr": ""
}
},
{
"id": "lancedb_du_table",
"seq": 650028,
"targets": [
"lancedb"
],
"command": "du /db/animals",
"expect": {
"exit": 0,
"stdout": "70\t/db/animals/cat/big\n72\t/db/animals/cat/small\n142\t/db/animals/cat\n68\t/db/animals/dog/big\n74\t/db/animals/dog/small\n142\t/db/animals/dog\n284\t/db/animals\n",
"stderr": ""
}
},
{
"id": "lancedb_du_c_multi",
"seq": 650029,
"targets": [
"lancedb"
],
"command": "du -c /db/animals/cat /db/animals/dog",
"expect": {
"exit": 0,
"stdout": "70\t/db/animals/cat/big\n72\t/db/animals/cat/small\n142\t/db/animals/cat\n68\t/db/animals/dog/big\n74\t/db/animals/dog/small\n142\t/db/animals/dog\n284\ttotal\n",
"stderr": ""
}
},
{
"id": "lancedb_grep_q_match",
"seq": 650030,
"targets": [
"lancedb"
],
"command": "grep -q cat /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "lancedb_grep_q_no_match",
"seq": 650031,
"targets": [
"lancedb"
],
"command": "grep -q zebra /db/animals/cat/big/1.md",
"expect": {
"exit": 1,
"stdout": "",
"stderr": ""
}
},
{
"id": "lancedb_grep_no_match",
"seq": 650032,
"targets": [
"lancedb"
],
"command": "grep zebra /db/animals/cat/big/1.md",
"expect": {
"exit": 1,
"stdout": "",
"stderr": ""
}
}
]
}
integ/resources/lancedb/probe.json
+98
View File
@@ -0,0 +1,98 @@
{
"cases": [
{
"id": "lancedb_cat_missing",
"seq": 650033,
"targets": [
"lancedb"
],
"command": "cat /db/__nf_missing__.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "cat: /db/__nf_missing__.txt: No such file or directory\n"
}
},
{
"id": "lancedb_write_rejected",
"seq": 650034,
"targets": [
"lancedb"
],
"command": "echo hi > /db/animals/cat/big/1.md",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "/db/animals/cat/big/1.md: Operation not supported\n"
}
},
{
"id": "lancedb_sed_stream_1p",
"seq": 650035,
"targets": [
"lancedb"
],
"command": "sed -n 1p /db/animals/cat/big/1.md",
"expect": {
"exit": 0,
"stdout": "# a big orange cat\n",
"stderr": ""
}
},
{
"id": "lancedb_sed_i_readonly",
"seq": 650036,
"targets": [
"lancedb"
],
"command": "sed -i s/x/y/ /db/animals/cat/big/1.md",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "sed: -i not supported on this backend: Permission denied\n"
}
},
{
"id": "lancedb_prov_cat",
"seq": 650037,
"targets": [
"lancedb"
],
"command": "cat /db/animals/cat/big/1.md",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=70 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
},
{
"id": "lancedb_prov_grep",
"seq": 650038,
"targets": [
"lancedb"
],
"command": "grep x /db/animals/cat/big/1.md",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=70 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
},
{
"id": "lancedb_prov_ls",
"seq": 650039,
"targets": [
"lancedb"
],
"command": "ls /db/animals/cat/big",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/cat.json
+69
View File
@@ -0,0 +1,69 @@
{
"cases": [
{
"id": "mdb_cat_database_json",
"seq": 620010,
"targets": [
"mongodb"
],
"command": "cat /mongodb/mirage_integ/database.json",
"expect": {
"exit": 0,
"stdout": "{\"database\": \"mirage_integ\", \"collections\": [{\"name\": \"authors\", \"document_count\": 3}, {\"name\": \"books\", \"document_count\": 5}, {\"name\": \"system.views\", \"document_count\": 1}], \"views\": [{\"name\": \"recent_books\"}]}\n",
"stderr": ""
}
},
{
"id": "mdb_cat_docs",
"seq": 620012,
"targets": [
"mongodb"
],
"command": "cat /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\n{\"_id\": 3, \"title\": \"gamma\", \"author\": \"cara\", \"year\": 2022, \"rating\": 5}\n{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
},
{
"id": "mdb_cat_view_docs",
"seq": 620013,
"targets": [
"mongodb"
],
"command": "cat /mongodb/mirage_integ/views/recent_books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 3, \"title\": \"gamma\", \"author\": \"cara\", \"year\": 2022, \"rating\": 5}\n{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
},
{
"id": "mdb_head_2",
"seq": 620014,
"targets": [
"mongodb"
],
"command": "head -n 2 /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\n",
"stderr": ""
}
},
{
"id": "mdb_tail_2",
"seq": 620015,
"targets": [
"mongodb"
],
"command": "tail -n 2 /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/find.json
+30
View File
@@ -0,0 +1,30 @@
{
"cases": [
{
"id": "mdb_find_docs",
"seq": 620040,
"targets": [
"mongodb"
],
"command": "find /mongodb/mirage_integ/ -name documents.jsonl",
"expect": {
"exit": 0,
"stdout": "/mongodb/mirage_integ/collections/authors/documents.jsonl\n/mongodb/mirage_integ/collections/books/documents.jsonl\n/mongodb/mirage_integ/collections/system.views/documents.jsonl\n/mongodb/mirage_integ/views/recent_books/documents.jsonl\n",
"stderr": ""
}
},
{
"id": "mdb_find_schema",
"seq": 620041,
"targets": [
"mongodb"
],
"command": "find /mongodb/mirage_integ/ -name schema.json",
"expect": {
"exit": 0,
"stdout": "/mongodb/mirage_integ/collections/authors/schema.json\n/mongodb/mirage_integ/collections/books/schema.json\n/mongodb/mirage_integ/collections/system.views/schema.json\n/mongodb/mirage_integ/views/recent_books/schema.json\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/grep.json
+108
View File
@@ -0,0 +1,108 @@
{
"cases": [
{
"id": "mdb_grep_ada",
"seq": 620050,
"targets": [
"mongodb"
],
"command": "grep ada /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n",
"stderr": ""
}
},
{
"id": "mdb_grep_c_title",
"seq": 620051,
"targets": [
"mongodb"
],
"command": "grep -c title /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "5\n",
"stderr": ""
}
},
{
"id": "mdb_grep_v_ada",
"seq": 620052,
"targets": [
"mongodb"
],
"command": "grep -v ada /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\n{\"_id\": 3, \"title\": \"gamma\", \"author\": \"cara\", \"year\": 2022, \"rating\": 5}\n{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
},
{
"id": "mdb_grep_e_multi",
"seq": 620053,
"targets": [
"mongodb"
],
"command": "grep -n -e ada -e ben /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "1:{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n2:{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\n4:{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n5:{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
},
{
"id": "mdb_grep_r_e_multi",
"seq": 620054,
"targets": [
"mongodb"
],
"command": "grep -r -e alpha -e beta /mongodb/mirage_integ/collections/books",
"expect": {
"exit": 0,
"stdout": "/mongodb/mirage_integ/collections/books/documents.jsonl:{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n/mongodb/mirage_integ/collections/books/documents.jsonl:{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\n",
"stderr": ""
}
},
{
"id": "mdb_grep_db_scope",
"seq": 620055,
"targets": [
"mongodb"
],
"command": "grep ada /mongodb/mirage_integ/",
"expect": {
"exit": 0,
"stdout": "mirage_integ/collections/authors/documents.jsonl:{\"_id\": 1, \"name\": \"ada\", \"books\": 2}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n",
"stderr": ""
}
},
{
"id": "mdb_grep_root_scope",
"seq": 620056,
"targets": [
"mongodb"
],
"command": "grep ada /mongodb/",
"expect": {
"exit": 0,
"stdout": "mirage_integ/collections/authors/documents.jsonl:{\"_id\": 1, \"name\": \"ada\", \"books\": 2}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n",
"stderr": ""
}
},
{
"id": "mdb_pipe_grep_c",
"seq": 620057,
"targets": [
"mongodb"
],
"command": "cat /mongodb/mirage_integ/collections/books/documents.jsonl | grep -c fiction",
"expect": {
"exit": 0,
"stdout": "2\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/jq.json
+30
View File
@@ -0,0 +1,30 @@
{
"cases": [
{
"id": "mdb_jq_titles",
"seq": 620070,
"targets": [
"mongodb"
],
"command": "jq '.title' /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "\"alpha\"\n\"beta\"\n\"gamma\"\n\"delta\"\n\"epsilon\"\n",
"stderr": ""
}
},
{
"id": "mdb_jq_schema_fields",
"seq": 620071,
"targets": [
"mongodb"
],
"command": "cat /mongodb/mirage_integ/collections/books/schema.json | jq -c '.fields'",
"expect": {
"exit": 0,
"stdout": "[{\"path\":\"author\",\"presence\":1,\"types\":{\"string\":1}},{\"path\":\"rating\",\"presence\":1,\"types\":{\"double\":0.6,\"int\":0.4}},{\"path\":\"tags\",\"presence\":0.6,\"types\":{\"array<string>\":0.6}},{\"path\":\"title\",\"presence\":1,\"types\":{\"string\":1}},{\"path\":\"year\",\"presence\":1,\"types\":{\"int\":1}}]\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/ls.json
+82
View File
@@ -0,0 +1,82 @@
{
"cases": [
{
"id": "mdb_ls_root",
"seq": 620000,
"targets": [
"mongodb"
],
"command": "ls /mongodb/",
"expect": {
"exit": 0,
"stdout": "mirage_integ\n",
"stderr": ""
}
},
{
"id": "mdb_ls_db",
"seq": 620001,
"targets": [
"mongodb"
],
"command": "ls /mongodb/mirage_integ/",
"expect": {
"exit": 0,
"stdout": "collections\ndatabase.json\nviews\n",
"stderr": ""
}
},
{
"id": "mdb_ls_collections",
"seq": 620002,
"targets": [
"mongodb"
],
"command": "ls /mongodb/mirage_integ/collections/",
"expect": {
"exit": 0,
"stdout": "authors\nbooks\nsystem.views\n",
"stderr": ""
}
},
{
"id": "mdb_ls_views",
"seq": 620003,
"targets": [
"mongodb"
],
"command": "ls /mongodb/mirage_integ/views/",
"expect": {
"exit": 0,
"stdout": "recent_books\n",
"stderr": ""
}
},
{
"id": "mdb_ls_entity",
"seq": 620004,
"targets": [
"mongodb"
],
"command": "ls /mongodb/mirage_integ/collections/books/",
"expect": {
"exit": 0,
"stdout": "documents.jsonl\nschema.json\n",
"stderr": ""
}
},
{
"id": "mdb_tree",
"seq": 620005,
"targets": [
"mongodb"
],
"command": "tree -L 3 /mongodb/mirage_integ/",
"expect": {
"exit": 0,
"stdout": "/mongodb/mirage_integ\n|-- collections\n| |-- authors\n| | |-- documents.jsonl\n| | `-- schema.json\n| |-- books\n| | |-- documents.jsonl\n| | `-- schema.json\n| `-- system.views\n| |-- documents.jsonl\n| `-- schema.json\n|-- database.json\n`-- views\n `-- recent_books\n |-- documents.jsonl\n `-- schema.json\n\n7 directories, 9 files\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/probe.json
+98
View File
@@ -0,0 +1,98 @@
{
"cases": [
{
"id": "mdb_cat_missing",
"seq": 620090,
"targets": [
"mongodb"
],
"command": "cat /mongodb/__nf_missing__.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "cat: /mongodb/__nf_missing__.txt: No such file or directory\n"
}
},
{
"id": "mdb_write_rejected",
"seq": 620091,
"targets": [
"mongodb"
],
"command": "echo hi > /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "/mongodb/mirage_integ/collections/books/documents.jsonl: Operation not supported\n"
}
},
{
"id": "mdb_sed_stream_1p",
"seq": 620092,
"targets": [
"mongodb"
],
"command": "sed -n 1p /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n",
"stderr": ""
}
},
{
"id": "mdb_sed_i_readonly",
"seq": 620093,
"targets": [
"mongodb"
],
"command": "sed -i s/x/y/ /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "sed: -i not supported on this backend: Permission denied\n"
}
},
{
"id": "mdb_prov_cat",
"seq": 620094,
"targets": [
"mongodb"
],
"command": "cat /mongodb/mirage_integ/collections/books/documents.jsonl",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=unknown\n",
"stderr": ""
}
},
{
"id": "mdb_prov_grep",
"seq": 620095,
"targets": [
"mongodb"
],
"command": "grep x /mongodb/mirage_integ/collections/books/documents.jsonl",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=unknown\n",
"stderr": ""
}
},
{
"id": "mdb_prov_ls",
"seq": 620096,
"targets": [
"mongodb"
],
"command": "ls /mongodb/mirage_integ/collections/books",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/rg.json
+69
View File
@@ -0,0 +1,69 @@
{
"cases": [
{
"id": "mdb_rg_ada",
"seq": 620060,
"targets": [
"mongodb"
],
"command": "rg ada /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\n{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n",
"stderr": ""
}
},
{
"id": "mdb_rg_v_ada",
"seq": 620061,
"targets": [
"mongodb"
],
"command": "rg -v ada /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\n{\"_id\": 3, \"title\": \"gamma\", \"author\": \"cara\", \"year\": 2022, \"rating\": 5}\n{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
},
{
"id": "mdb_rg_e_multi",
"seq": 620062,
"targets": [
"mongodb"
],
"command": "rg -e gamma -e cara /mongodb/mirage_integ/collections/books",
"expect": {
"exit": 0,
"stdout": "/mongodb/mirage_integ/collections/books/documents.jsonl:{\"_id\": 3, \"title\": \"gamma\", \"author\": \"cara\", \"year\": 2022, \"rating\": 5}\n",
"stderr": ""
}
},
{
"id": "mdb_rg_db_scope",
"seq": 620063,
"targets": [
"mongodb"
],
"command": "rg ben /mongodb/mirage_integ/",
"expect": {
"exit": 0,
"stdout": "mirage_integ/collections/authors/documents.jsonl:{\"_id\": 2, \"name\": \"ben\", \"books\": 2}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 2, \"title\": \"beta\", \"author\": \"ben\", \"year\": 2021, \"tags\": [\"fiction\"], \"rating\": 3.2}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 5, \"title\": \"epsilon\", \"author\": \"ben\", \"year\": 2024, \"rating\": 2.5}\n",
"stderr": ""
}
},
{
"id": "mdb_rg_root_scope",
"seq": 620064,
"targets": [
"mongodb"
],
"command": "rg ada /mongodb/",
"expect": {
"exit": 0,
"stdout": "mirage_integ/collections/authors/documents.jsonl:{\"_id\": 1, \"name\": \"ada\", \"books\": 2}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 1, \"title\": \"alpha\", \"author\": \"ada\", \"year\": 2020, \"tags\": [\"fiction\", \"classic\"], \"rating\": 4.5}\nmirage_integ/collections/books/documents.jsonl:{\"_id\": 4, \"title\": \"delta\", \"author\": \"ada\", \"year\": 2023, \"tags\": [\"history\"], \"rating\": 4}\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/stat.json
+17
View File
@@ -0,0 +1,17 @@
{
"cases": [
{
"id": "mdb_stat_size_docs",
"seq": 620030,
"targets": [
"mongodb"
],
"command": "stat -c '%s %n' /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "0 /mongodb/mirage_integ/collections/books/documents.jsonl\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/sym.json
+69
View File
@@ -0,0 +1,69 @@
{
"cases": [
{
"id": "mdb_sym_ln",
"seq": 620100,
"targets": [
"mongodb"
],
"command": "ln -s /mongodb/mirage_integ/database.json /mongodb/meta_link",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "mdb_sym_readlink",
"seq": 620101,
"targets": [
"mongodb"
],
"command": "readlink /mongodb/meta_link",
"expect": {
"exit": 0,
"stdout": "/mongodb/mirage_integ/database.json\n",
"stderr": ""
}
},
{
"id": "mdb_sym_cat",
"seq": 620102,
"targets": [
"mongodb"
],
"command": "cat /mongodb/meta_link",
"expect": {
"exit": 0,
"stdout": "{\"database\": \"mirage_integ\", \"collections\": [{\"name\": \"authors\", \"document_count\": 3}, {\"name\": \"books\", \"document_count\": 5}, {\"name\": \"system.views\", \"document_count\": 1}], \"views\": [{\"name\": \"recent_books\"}]}\n",
"stderr": ""
}
},
{
"id": "mdb_sym_ls",
"seq": 620103,
"targets": [
"mongodb"
],
"command": "ls -F /mongodb | grep meta_link",
"expect": {
"exit": 0,
"stdout": "meta_link@\n",
"stderr": ""
}
},
{
"id": "mdb_sym_rm",
"seq": 620104,
"targets": [
"mongodb"
],
"command": "rm /mongodb/meta_link && ls /mongodb",
"expect": {
"exit": 0,
"stdout": "mirage_integ\n",
"stderr": ""
}
}
]
}
integ/resources/mongodb/wc.json
+56
View File
@@ -0,0 +1,56 @@
{
"cases": [
{
"id": "mdb_wc_l_books",
"seq": 620020,
"targets": [
"mongodb"
],
"command": "wc -l /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "5 /mongodb/mirage_integ/collections/books/documents.jsonl\n",
"stderr": ""
}
},
{
"id": "mdb_wc_default_books",
"seq": 620021,
"targets": [
"mongodb"
],
"command": "wc /mongodb/mirage_integ/collections/books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": " 5 57 447 /mongodb/mirage_integ/collections/books/documents.jsonl\n",
"stderr": ""
}
},
{
"id": "mdb_wc_l_authors",
"seq": 620022,
"targets": [
"mongodb"
],
"command": "wc -l /mongodb/mirage_integ/collections/authors/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "3 /mongodb/mirage_integ/collections/authors/documents.jsonl\n",
"stderr": ""
}
},
{
"id": "mdb_wc_l_view",
"seq": 620023,
"targets": [
"mongodb"
],
"command": "wc -l /mongodb/mirage_integ/views/recent_books/documents.jsonl",
"expect": {
"exit": 0,
"stdout": "3 /mongodb/mirage_integ/views/recent_books/documents.jsonl\n",
"stderr": ""
}
}
]
}
integ/resources/notion/ops.json
+394
View File
@@ -0,0 +1,394 @@
{
"cases": [
{
"id": "notion_ls_root",
"seq": 660000,
"targets": [
"notion"
],
"command": "ls /notion/",
"expect": {
"exit": 0,
"stdout": "databases\npages\n",
"stderr": ""
}
},
{
"id": "notion_ls_pages",
"seq": 660001,
"targets": [
"notion"
],
"command": "ls /notion/pages/",
"expect": {
"exit": 0,
"stdout": "Notes__bbbb2222-3333-4444-5555-666677778888\nProject_Roadmap__aaaa1111-2222-3333-4444-555566667777\n",
"stderr": ""
}
},
{
"id": "notion_ls_l_pages",
"seq": 660002,
"targets": [
"notion"
],
"command": "ls -l /notion/pages/",
"expect": {
"exit": 0,
"stdout": "drwxr-xr-x 1 user user 0 Jan 2 00:00 Notes__bbbb2222-3333-4444-5555-666677778888\ndrwxr-xr-x 1 user user 0 Jan 2 00:00 Project_Roadmap__aaaa1111-2222-3333-4444-555566667777\n",
"stderr": ""
}
},
{
"id": "notion_ls_page_a",
"seq": 660003,
"targets": [
"notion"
],
"command": "ls /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/",
"expect": {
"exit": 0,
"stdout": "Q1_Goals__cccc1111-2222-3333-4444-555566667777\npage.json\n",
"stderr": ""
}
},
{
"id": "notion_stat_dir_a",
"seq": 660004,
"targets": [
"notion"
],
"command": "stat -c '%n %y' /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777",
"expect": {
"exit": 0,
"stdout": "/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777 2026-01-02T00:00:00.000Z\n",
"stderr": ""
}
},
{
"id": "notion_cat_page_a",
"seq": 660005,
"targets": [
"notion"
],
"command": "cat /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "{\n \"page_id\": \"aaaa1111-2222-3333-4444-555566667777\",\n \"title\": \"Project Roadmap\",\n \"url\": \"https://notion.example/aaaa1111222233334444555566667777\",\n \"created_time\": \"2026-01-01T00:00:00.000Z\",\n \"last_edited_time\": \"2026-01-02T00:00:00.000Z\",\n \"parent_type\": \"workspace\",\n \"parent_id\": \"\",\n \"archived\": false,\n \"created_by\": \"user-1\",\n \"last_edited_by\": \"user-2\",\n \"markdown\": \"# Roadmap\\n\\nShip the **beta** soon\\n\\n- phase one\\n\\n - phase one detail\\n\\n```python\\nprint(1)\\n```\\n\",\n \"blocks\": [\n {\n \"object\": \"block\",\n \"id\": \"b-a1\",\n \"type\": \"heading_1\",\n \"has_children\": false,\n \"heading_1\": {\n \"rich_text\": [\n {\n \"type\": \"text\",\n \"plain_text\": \"Roadmap\",\n \"annotations\": {},\n \"text\": {\n \"content\": \"Roadmap\"\n }\n }\n ]\n }\n },\n {\n \"object\": \"block\",\n \"id\": \"b-a2\",\n \"type\": \"paragraph\",\n \"has_children\": false,\n \"paragraph\": {\n \"rich_text\": [\n {\n \"type\": \"text\",\n \"plain_text\": \"Ship the \",\n \"annotations\": {},\n \"text\": {\n \"content\": \"Ship the \"\n }\n },\n {\n \"type\": \"text\",\n \"plain_text\": \"beta\",\n \"annotations\": {\n \"bold\": true\n },\n \"text\": {\n \"content\": \"beta\"\n }\n },\n {\n \"type\": \"text\",\n \"plain_text\": \" soon\",\n \"annotations\": {},\n \"text\": {\n \"content\": \" soon\"\n }\n }\n ]\n }\n },\n {\n \"object\": \"block\",\n \"id\": \"dddd2222-3333-4444-5555-666677778888\",\n \"type\": \"bulleted_list_item\",\n \"has_children\": true,\n \"bulleted_list_item\": {\n \"rich_text\": [\n {\n \"type\": \"text\",\n \"plain_text\": \"phase one\",\n \"annotations\": {},\n \"text\": {\n \"content\": \"phase one\"\n }\n }\n ]\n },\n \"children\": [\n {\n \"object\": \"block\",\n \"id\": \"b-d1\",\n \"type\": \"bulleted_list_item\",\n \"has_children\": false,\n \"bulleted_list_item\": {\n \"rich_text\": [\n {\n \"type\": \"text\",\n \"plain_text\": \"phase one detail\",\n \"annotations\": {},\n \"text\": {\n \"content\": \"phase one detail\"\n }\n }\n ]\n }\n }\n ]\n },\n {\n \"object\": \"block\",\n \"id\": \"b-a4\",\n \"type\": \"code\",\n \"has_children\": false,\n \"code\": {\n \"rich_text\": [\n {\n \"type\": \"text\",\n \"plain_text\": \"print(1)\",\n \"annotations\": {},\n \"text\": {\n \"content\": \"print(1)\"\n }\n }\n ],\n \"language\": \"python\"\n }\n }\n ]\n}",
"stderr": ""
}
},
{
"id": "notion_cat_child",
"seq": 660006,
"targets": [
"notion"
],
"command": "cat /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "{\n \"page_id\": \"cccc1111-2222-3333-4444-555566667777\",\n \"title\": \"Q1 Goals\",\n \"url\": \"https://notion.example/cccc1111222233334444555566667777\",\n \"created_time\": \"2026-01-01T00:00:00.000Z\",\n \"last_edited_time\": \"2026-01-02T00:00:00.000Z\",\n \"parent_type\": \"page_id\",\n \"parent_id\": \"aaaa1111-2222-3333-4444-555566667777\",\n \"archived\": false,\n \"created_by\": \"user-1\",\n \"last_edited_by\": \"user-2\",\n \"markdown\": \"Q1 contents\\n\",\n \"blocks\": [\n {\n \"object\": \"block\",\n \"id\": \"b-c1\",\n \"type\": \"paragraph\",\n \"has_children\": false,\n \"paragraph\": {\n \"rich_text\": [\n {\n \"type\": \"text\",\n \"plain_text\": \"Q1 contents\",\n \"annotations\": {},\n \"text\": {\n \"content\": \"Q1 contents\"\n }\n }\n ]\n }\n }\n ]\n}",
"stderr": ""
}
},
{
"id": "notion_jq_title",
"seq": 660007,
"targets": [
"notion"
],
"command": "jq \".title\" /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "\"Project Roadmap\"\n",
"stderr": ""
}
},
{
"id": "notion_jq_markdown",
"seq": 660008,
"targets": [
"notion"
],
"command": "jq \".markdown\" /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json",
"expect": {
"exit": 0,
"stdout": "\"alpha beta gamma\\n\\n- [x] done item\\n\"\n",
"stderr": ""
}
},
{
"id": "notion_head_4",
"seq": 660009,
"targets": [
"notion"
],
"command": "head -n 4 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "{\n \"page_id\": \"aaaa1111-2222-3333-4444-555566667777\",\n \"title\": \"Project Roadmap\",\n \"url\": \"https://notion.example/aaaa1111222233334444555566667777\",\n",
"stderr": ""
}
},
{
"id": "notion_wc_l_two",
"seq": 660010,
"targets": [
"notion"
],
"command": "wc -l /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json",
"expect": {
"exit": 0,
"stdout": "125 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json\n 51 /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json\n176 total\n",
"stderr": ""
}
},
{
"id": "notion_stat_page_json",
"seq": 660011,
"targets": [
"notion"
],
"command": "stat /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "name=page.json size=2979 modified=None type=json\n",
"stderr": ""
}
},
{
"id": "notion_find_json",
"seq": 660012,
"targets": [
"notion"
],
"command": "find /notion/pages/ -name page.json",
"expect": {
"exit": 0,
"stdout": "/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json\n/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777/page.json\n/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json\n",
"stderr": ""
}
},
{
"id": "notion_find_root_maxdepth0",
"seq": 660013,
"targets": [
"notion"
],
"command": "find /notion -maxdepth 0",
"expect": {
"exit": 0,
"stdout": "/notion\n",
"stderr": ""
}
},
{
"id": "notion_find_root_name",
"seq": 660014,
"targets": [
"notion"
],
"command": "find /notion -name notion",
"expect": {
"exit": 0,
"stdout": "/notion\n",
"stderr": ""
}
},
{
"id": "notion_pipe_grep",
"seq": 660015,
"targets": [
"notion"
],
"command": "cat /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json | grep -c alpha",
"expect": {
"exit": 0,
"stdout": "3\n",
"stderr": ""
}
},
{
"id": "notion_grep_file",
"seq": 660016,
"targets": [
"notion"
],
"command": "grep -n alpha /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json",
"expect": {
"exit": 0,
"stdout": "12: \"markdown\": \"alpha beta gamma\\n\\n- [x] done item\\n\",\n23: \"plain_text\": \"alpha beta gamma\",\n26: \"content\": \"alpha beta gamma\"\n",
"stderr": ""
}
},
{
"id": "notion_grep_multi",
"seq": 660017,
"targets": [
"notion"
],
"command": "grep -c alpha /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json",
"expect": {
"exit": 0,
"stdout": "/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json:0\n/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json:3\n",
"stderr": ""
}
},
{
"id": "notion_grep_recursive",
"seq": 660018,
"targets": [
"notion"
],
"command": "grep -rl alpha /notion/pages/",
"expect": {
"exit": 0,
"stdout": "/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json\n",
"stderr": ""
}
},
{
"id": "notion_realpath_dotdot",
"seq": 660019,
"targets": [
"notion"
],
"command": "realpath -e /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777/../page.json",
"expect": {
"exit": 0,
"stdout": "/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json\n",
"stderr": ""
}
},
{
"id": "notion_ls_databases",
"seq": 660020,
"targets": [
"notion"
],
"command": "ls /notion/databases/",
"expect": {
"exit": 0,
"stdout": "Tasks__eeee1111-2222-3333-4444-555566667777\n",
"stderr": ""
}
},
{
"id": "notion_ls_database_dir",
"seq": 660021,
"targets": [
"notion"
],
"command": "ls /notion/databases/Tasks__eeee1111-2222-3333-4444-555566667777/",
"expect": {
"exit": 0,
"stdout": "Ship_beta__ffff2222-3333-4444-5555-666677778888\nWrite_spec__ffff1111-2222-3333-4444-555566667777\ndatabase.json\n",
"stderr": ""
}
},
{
"id": "notion_cat_database_json",
"seq": 660022,
"targets": [
"notion"
],
"command": "cat /notion/databases/Tasks__eeee1111-2222-3333-4444-555566667777/database.json",
"expect": {
"exit": 0,
"stdout": "{\n \"database_id\": \"eeee1111-2222-3333-4444-555566667777\",\n \"title\": \"Tasks\",\n \"url\": \"https://notion.example/eeee1111222233334444555566667777\",\n \"created_time\": \"2026-01-01T00:00:00.000Z\",\n \"last_edited_time\": \"2026-01-02T00:00:00.000Z\",\n \"parent\": {\n \"type\": \"workspace\",\n \"workspace\": true\n },\n \"archived\": false,\n \"is_inline\": false,\n \"properties\": {\n \"Name\": {\n \"id\": \"title\",\n \"name\": \"Name\",\n \"type\": \"title\",\n \"title\": {}\n },\n \"Priority\": {\n \"id\": \"pri\",\n \"name\": \"Priority\",\n \"type\": \"number\",\n \"number\": {\n \"format\": \"number\"\n }\n }\n }\n}",
"stderr": ""
}
},
{
"id": "notion_jq_db_props",
"seq": 660023,
"targets": [
"notion"
],
"command": "jq \".properties | keys\" /notion/databases/Tasks__eeee1111-2222-3333-4444-555566667777/database.json",
"expect": {
"exit": 0,
"stdout": "[\n \"Name\",\n \"Priority\"\n]\n",
"stderr": ""
}
},
{
"id": "notion_cat_row",
"seq": 660024,
"targets": [
"notion"
],
"command": "cat /notion/databases/Tasks__eeee1111-2222-3333-4444-555566667777/Write_spec__ffff1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "{\n \"page_id\": \"ffff1111-2222-3333-4444-555566667777\",\n \"title\": \"Write spec\",\n \"url\": \"https://notion.example/ffff1111222233334444555566667777\",\n \"created_time\": \"2026-01-01T00:00:00.000Z\",\n \"last_edited_time\": \"2026-01-02T00:00:00.000Z\",\n \"parent_type\": \"database_id\",\n \"parent_id\": \"eeee1111-2222-3333-4444-555566667777\",\n \"archived\": false,\n \"created_by\": \"user-1\",\n \"last_edited_by\": \"user-2\",\n \"markdown\": \"\",\n \"blocks\": []\n}",
"stderr": ""
}
},
{
"id": "notion_du_pages",
"seq": 660025,
"targets": [
"notion"
],
"command": "du /notion/pages/",
"expect": {
"exit": 0,
"stdout": "1211\t/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888\n826\t/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777\n3805\t/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777\n5016\t/notion/pages\n",
"stderr": ""
}
},
{
"id": "notion_du_page_a",
"seq": 660026,
"targets": [
"notion"
],
"command": "du /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/",
"expect": {
"exit": 0,
"stdout": "826\t/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777\n3805\t/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777\n",
"stderr": ""
}
},
{
"id": "notion_grep_c_match_exit",
"seq": 660027,
"targets": [
"notion"
],
"command": "grep -c alpha /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json",
"expect": {
"exit": 0,
"stdout": "3\n",
"stderr": ""
}
},
{
"id": "notion_grep_c_no_match_exit",
"seq": 660028,
"targets": [
"notion"
],
"command": "grep -c zzz /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json",
"expect": {
"exit": 1,
"stdout": "0\n",
"stderr": ""
}
},
{
"id": "notion_grep_rc_no_match_exit",
"seq": 660029,
"targets": [
"notion"
],
"command": "grep -rc zzz /notion/pages/",
"expect": {
"exit": 1,
"stdout": "/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json:0\n/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777/page.json:0\n/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json:0\n",
"stderr": ""
}
}
]
}
integ/resources/notion/probe.json
+98
View File
@@ -0,0 +1,98 @@
{
"cases": [
{
"id": "notion_cat_missing",
"seq": 660030,
"targets": [
"notion"
],
"command": "cat /notion/__nf_missing__.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "cat: /notion/__nf_missing__.txt: No such file or directory\n"
}
},
{
"id": "notion_write_rejected",
"seq": 660031,
"targets": [
"notion"
],
"command": "echo hi > /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json: Operation not supported\n"
}
},
{
"id": "notion_sed_stream_1p",
"seq": 660032,
"targets": [
"notion"
],
"command": "sed -n 1p /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 0,
"stdout": "{\n",
"stderr": ""
}
},
{
"id": "notion_sed_i_readonly",
"seq": 660033,
"targets": [
"notion"
],
"command": "sed -i s/x/y/ /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "sed: -i not supported on this backend: Permission denied\n"
}
},
{
"id": "notion_prov_cat",
"seq": 660034,
"targets": [
"notion"
],
"command": "cat /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=1 precision=unknown\n",
"stderr": ""
}
},
{
"id": "notion_prov_grep",
"seq": 660035,
"targets": [
"notion"
],
"command": "grep x /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=1 precision=unknown\n",
"stderr": ""
}
},
{
"id": "notion_prov_ls",
"seq": 660036,
"targets": [
"notion"
],
"command": "ls /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/cat.json
+82
View File
@@ -0,0 +1,82 @@
{
"cases": [
{
"id": "pg_cat_schema_json",
"seq": 610010,
"targets": [
"postgres"
],
"command": "cat /pg/public/tables/books/schema.json",
"expect": {
"exit": 0,
"stdout": "{\n \"schema\": \"public\",\n \"name\": \"books\",\n \"kind\": \"table\",\n \"columns\": [\n {\n \"name\": \"id\",\n \"type\": \"integer\",\n \"nullable\": false,\n \"primary_key\": true\n },\n {\n \"name\": \"title\",\n \"type\": \"text\",\n \"nullable\": true\n },\n {\n \"name\": \"author\",\n \"type\": \"text\",\n \"nullable\": true\n },\n {\n \"name\": \"year\",\n \"type\": \"integer\",\n \"nullable\": true\n },\n {\n \"name\": \"rating\",\n \"type\": \"double precision\",\n \"nullable\": true\n }\n ],\n \"primary_key\": [\n \"id\"\n ],\n \"foreign_keys\": [],\n \"indexes\": [\n {\n \"name\": \"books_pkey\",\n \"columns\": [\n \"id\"\n ],\n \"unique\": true\n }\n ],\n \"row_count_estimate\": 5,\n \"size_bytes_estimate\": 32768\n}",
"stderr": ""
}
},
{
"id": "pg_cat_semantic_json",
"seq": 610011,
"targets": [
"postgres"
],
"command": "cat /pg/public/tables/books/semantic.json",
"expect": {
"exit": 0,
"stdout": "{\n \"name\": \"books\",\n \"schema\": \"public\",\n \"kind\": \"table\",\n \"primary_key\": [\n \"id\"\n ],\n \"dimensions\": [\n {\n \"name\": \"id\",\n \"expr\": \"id\",\n \"data_type\": \"integer\"\n },\n {\n \"name\": \"title\",\n \"expr\": \"title\",\n \"data_type\": \"text\"\n },\n {\n \"name\": \"author\",\n \"expr\": \"author\",\n \"data_type\": \"text\",\n \"sample_values\": [\n \"ada\",\n \"ben\"\n ]\n }\n ],\n \"facts\": [\n {\n \"name\": \"year\",\n \"expr\": \"year\",\n \"data_type\": \"integer\"\n },\n {\n \"name\": \"rating\",\n \"expr\": \"rating\",\n \"data_type\": \"double precision\"\n }\n ]\n}",
"stderr": ""
}
},
{
"id": "pg_cat_rows",
"seq": 610012,
"targets": [
"postgres"
],
"command": "cat /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\n{\"id\":2,\"title\":\"beta\",\"author\":\"ben\",\"year\":2021,\"rating\":3.2}\n{\"id\":3,\"title\":\"gamma\",\"author\":\"cara\",\"year\":2022,\"rating\":5}\n{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
},
{
"id": "pg_cat_view_rows",
"seq": 610013,
"targets": [
"postgres"
],
"command": "cat /pg/public/views/recent_books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":3,\"title\":\"gamma\",\"author\":\"cara\",\"year\":2022,\"rating\":5}\n{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
},
{
"id": "pg_head_2",
"seq": 610014,
"targets": [
"postgres"
],
"command": "head -n 2 /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\n{\"id\":2,\"title\":\"beta\",\"author\":\"ben\",\"year\":2021,\"rating\":3.2}\n",
"stderr": ""
}
},
{
"id": "pg_tail_2",
"seq": 610015,
"targets": [
"postgres"
],
"command": "tail -n 2 /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/find.json
+56
View File
@@ -0,0 +1,56 @@
{
"cases": [
{
"id": "pg_find_rows",
"seq": 610040,
"targets": [
"postgres"
],
"command": "find /pg/public/ -name rows.jsonl",
"expect": {
"exit": 0,
"stdout": "/pg/public/tables/authors/rows.jsonl\n/pg/public/tables/books/rows.jsonl\n/pg/public/views/recent_books/rows.jsonl\n",
"stderr": ""
}
},
{
"id": "pg_find_schema",
"seq": 610041,
"targets": [
"postgres"
],
"command": "find /pg/public/ -name schema.json",
"expect": {
"exit": 0,
"stdout": "/pg/public/tables/authors/schema.json\n/pg/public/tables/books/schema.json\n/pg/public/views/recent_books/schema.json\n",
"stderr": ""
}
},
{
"id": "pg_find_size_plus_rows",
"seq": 610042,
"targets": [
"postgres"
],
"command": "find /pg/public/tables/ -name rows.jsonl -size +1c",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "pg_find_size_under_rows",
"seq": 610043,
"targets": [
"postgres"
],
"command": "find /pg/public/tables/books/ -name rows.jsonl -size -1k",
"expect": {
"exit": 0,
"stdout": "/pg/public/tables/books/rows.jsonl\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/grep.json
+95
View File
@@ -0,0 +1,95 @@
{
"cases": [
{
"id": "pg_grep_ada",
"seq": 610050,
"targets": [
"postgres"
],
"command": "grep ada /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "public/tables/books/rows.jsonl:{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\npublic/tables/books/rows.jsonl:{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n",
"stderr": ""
}
},
{
"id": "pg_grep_c_title",
"seq": 610051,
"targets": [
"postgres"
],
"command": "grep -c title /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "5\n",
"stderr": ""
}
},
{
"id": "pg_grep_v_ada",
"seq": 610052,
"targets": [
"postgres"
],
"command": "grep -v ada /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":2,\"title\":\"beta\",\"author\":\"ben\",\"year\":2021,\"rating\":3.2}\n{\"id\":3,\"title\":\"gamma\",\"author\":\"cara\",\"year\":2022,\"rating\":5}\n{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
},
{
"id": "pg_grep_e_multi",
"seq": 610053,
"targets": [
"postgres"
],
"command": "grep -n -e ada -e ben /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "1:{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\n2:{\"id\":2,\"title\":\"beta\",\"author\":\"ben\",\"year\":2021,\"rating\":3.2}\n4:{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n5:{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
},
{
"id": "pg_grep_schema_scope",
"seq": 610054,
"targets": [
"postgres"
],
"command": "grep ada /pg/public/tables/",
"expect": {
"exit": 0,
"stdout": "public/tables/authors/rows.jsonl:{\"id\":1,\"name\":\"ada\",\"books\":2}\npublic/tables/books/rows.jsonl:{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\npublic/tables/books/rows.jsonl:{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\npublic/tables/books/semantic.json: \"ada\",\n",
"stderr": ""
}
},
{
"id": "pg_grep_semantic_scope",
"seq": 610055,
"targets": [
"postgres"
],
"command": "grep dimensions /pg/public/tables/books/",
"expect": {
"exit": 0,
"stdout": "public/tables/books/semantic.json: \"dimensions\": [\n",
"stderr": ""
}
},
{
"id": "pg_pipe_grep_c",
"seq": 610056,
"targets": [
"postgres"
],
"command": "cat /pg/public/tables/books/rows.jsonl | grep -c ada",
"expect": {
"exit": 0,
"stdout": "2\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/jq.json
+17
View File
@@ -0,0 +1,17 @@
{
"cases": [
{
"id": "pg_jq_titles",
"seq": 610070,
"targets": [
"postgres"
],
"command": "jq '.title' /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "\"alpha\"\n\"beta\"\n\"gamma\"\n\"delta\"\n\"epsilon\"\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/ls.json
+82
View File
@@ -0,0 +1,82 @@
{
"cases": [
{
"id": "pg_ls_root",
"seq": 610000,
"targets": [
"postgres"
],
"command": "ls /pg/",
"expect": {
"exit": 0,
"stdout": "database.json\npublic\n",
"stderr": ""
}
},
{
"id": "pg_ls_schema",
"seq": 610001,
"targets": [
"postgres"
],
"command": "ls /pg/public/",
"expect": {
"exit": 0,
"stdout": "tables\nviews\n",
"stderr": ""
}
},
{
"id": "pg_ls_tables",
"seq": 610002,
"targets": [
"postgres"
],
"command": "ls /pg/public/tables/",
"expect": {
"exit": 0,
"stdout": "authors\nbooks\n",
"stderr": ""
}
},
{
"id": "pg_ls_views",
"seq": 610003,
"targets": [
"postgres"
],
"command": "ls /pg/public/views/",
"expect": {
"exit": 0,
"stdout": "recent_books\n",
"stderr": ""
}
},
{
"id": "pg_ls_entity",
"seq": 610004,
"targets": [
"postgres"
],
"command": "ls /pg/public/tables/books/",
"expect": {
"exit": 0,
"stdout": "rows.jsonl\nschema.json\nsemantic.json\n",
"stderr": ""
}
},
{
"id": "pg_tree",
"seq": 610005,
"targets": [
"postgres"
],
"command": "tree -L 3 /pg/public/",
"expect": {
"exit": 0,
"stdout": "/pg/public\n|-- tables\n| |-- authors\n| | |-- rows.jsonl\n| | |-- schema.json\n| | `-- semantic.json\n| `-- books\n| |-- rows.jsonl\n| |-- schema.json\n| `-- semantic.json\n`-- views\n `-- recent_books\n |-- rows.jsonl\n |-- schema.json\n `-- semantic.json\n\n6 directories, 9 files\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/probe.json
+98
View File
@@ -0,0 +1,98 @@
{
"cases": [
{
"id": "pg_cat_missing",
"seq": 610090,
"targets": [
"postgres"
],
"command": "cat /pg/__nf_missing__.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "cat: /pg/__nf_missing__.txt: No such file or directory\n"
}
},
{
"id": "pg_write_rejected",
"seq": 610091,
"targets": [
"postgres"
],
"command": "echo hi > /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "/pg/public/tables/books/rows.jsonl: Operation not supported\n"
}
},
{
"id": "pg_sed_stream_1p",
"seq": 610092,
"targets": [
"postgres"
],
"command": "sed -n 1p /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\n",
"stderr": ""
}
},
{
"id": "pg_sed_i_readonly",
"seq": 610093,
"targets": [
"postgres"
],
"command": "sed -i s/x/y/ /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "sed: -i not supported on this backend: Permission denied\n"
}
},
{
"id": "pg_prov_cat",
"seq": 610094,
"targets": [
"postgres"
],
"command": "cat /pg/public/tables/books/rows.jsonl",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=unknown\n",
"stderr": ""
}
},
{
"id": "pg_prov_grep",
"seq": 610095,
"targets": [
"postgres"
],
"command": "grep x /pg/public/tables/books/rows.jsonl",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=unknown\n",
"stderr": ""
}
},
{
"id": "pg_prov_ls",
"seq": 610096,
"targets": [
"postgres"
],
"command": "ls /pg/public/tables/books",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/rg.json
+82
View File
@@ -0,0 +1,82 @@
{
"cases": [
{
"id": "pg_rg_ada",
"seq": 610060,
"targets": [
"postgres"
],
"command": "rg ada /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "public/tables/books/rows.jsonl:{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\npublic/tables/books/rows.jsonl:{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n",
"stderr": ""
}
},
{
"id": "pg_rg_c_title",
"seq": 610061,
"targets": [
"postgres"
],
"command": "rg -c title /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "5\n",
"stderr": ""
}
},
{
"id": "pg_rg_v_ada",
"seq": 610062,
"targets": [
"postgres"
],
"command": "rg -v ada /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "{\"id\":2,\"title\":\"beta\",\"author\":\"ben\",\"year\":2021,\"rating\":3.2}\n{\"id\":3,\"title\":\"gamma\",\"author\":\"cara\",\"year\":2022,\"rating\":5}\n{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
},
{
"id": "pg_rg_e_multi",
"seq": 610063,
"targets": [
"postgres"
],
"command": "rg -n -e ada -e ben /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "1:{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\n2:{\"id\":2,\"title\":\"beta\",\"author\":\"ben\",\"year\":2021,\"rating\":3.2}\n4:{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\n5:{\"id\":5,\"title\":\"epsilon\",\"author\":\"ben\",\"year\":2024,\"rating\":2.5}\n",
"stderr": ""
}
},
{
"id": "pg_rg_schema_scope",
"seq": 610064,
"targets": [
"postgres"
],
"command": "rg ada /pg/public/tables/",
"expect": {
"exit": 0,
"stdout": "public/tables/authors/rows.jsonl:{\"id\":1,\"name\":\"ada\",\"books\":2}\npublic/tables/books/rows.jsonl:{\"id\":1,\"title\":\"alpha\",\"author\":\"ada\",\"year\":2020,\"rating\":4.5}\npublic/tables/books/rows.jsonl:{\"id\":4,\"title\":\"delta\",\"author\":\"ada\",\"year\":2023,\"rating\":4}\npublic/tables/books/semantic.json: \"ada\",\n",
"stderr": ""
}
},
{
"id": "pg_rg_semantic_scope",
"seq": 610065,
"targets": [
"postgres"
],
"command": "rg dimensions /pg/public/tables/books/",
"expect": {
"exit": 0,
"stdout": "public/tables/books/semantic.json: \"dimensions\": [\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/stat.json
+17
View File
@@ -0,0 +1,17 @@
{
"cases": [
{
"id": "pg_stat_size_rows",
"seq": 610030,
"targets": [
"postgres"
],
"command": "stat -c '%s %n' /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "0 /pg/public/tables/books/rows.jsonl\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/sym.json
+69
View File
@@ -0,0 +1,69 @@
{
"cases": [
{
"id": "pg_sym_ln",
"seq": 610100,
"targets": [
"postgres"
],
"command": "ln -s /pg/public/tables/books/schema.json /pg/meta_link",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "pg_sym_readlink",
"seq": 610101,
"targets": [
"postgres"
],
"command": "readlink /pg/meta_link",
"expect": {
"exit": 0,
"stdout": "/pg/public/tables/books/schema.json\n",
"stderr": ""
}
},
{
"id": "pg_sym_cat",
"seq": 610102,
"targets": [
"postgres"
],
"command": "cat /pg/meta_link",
"expect": {
"exit": 0,
"stdout": "{\n \"schema\": \"public\",\n \"name\": \"books\",\n \"kind\": \"table\",\n \"columns\": [\n {\n \"name\": \"id\",\n \"type\": \"integer\",\n \"nullable\": false,\n \"primary_key\": true\n },\n {\n \"name\": \"title\",\n \"type\": \"text\",\n \"nullable\": true\n },\n {\n \"name\": \"author\",\n \"type\": \"text\",\n \"nullable\": true\n },\n {\n \"name\": \"year\",\n \"type\": \"integer\",\n \"nullable\": true\n },\n {\n \"name\": \"rating\",\n \"type\": \"double precision\",\n \"nullable\": true\n }\n ],\n \"primary_key\": [\n \"id\"\n ],\n \"foreign_keys\": [],\n \"indexes\": [\n {\n \"name\": \"books_pkey\",\n \"columns\": [\n \"id\"\n ],\n \"unique\": true\n }\n ],\n \"row_count_estimate\": 5,\n \"size_bytes_estimate\": 32768\n}",
"stderr": ""
}
},
{
"id": "pg_sym_ls",
"seq": 610103,
"targets": [
"postgres"
],
"command": "ls -F /pg | grep meta_link",
"expect": {
"exit": 0,
"stdout": "meta_link@\n",
"stderr": ""
}
},
{
"id": "pg_sym_rm",
"seq": 610104,
"targets": [
"postgres"
],
"command": "rm /pg/meta_link && ls /pg",
"expect": {
"exit": 0,
"stdout": "database.json\npublic\n",
"stderr": ""
}
}
]
}
integ/resources/postgres/wc.json
+56
View File
@@ -0,0 +1,56 @@
{
"cases": [
{
"id": "pg_wc_l_books",
"seq": 610020,
"targets": [
"postgres"
],
"command": "wc -l /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "5 /pg/public/tables/books/rows.jsonl\n",
"stderr": ""
}
},
{
"id": "pg_wc_default_books",
"seq": 610021,
"targets": [
"postgres"
],
"command": "wc /pg/public/tables/books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": " 5 5 323 /pg/public/tables/books/rows.jsonl\n",
"stderr": ""
}
},
{
"id": "pg_wc_l_authors",
"seq": 610022,
"targets": [
"postgres"
],
"command": "wc -l /pg/public/tables/authors/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "3 /pg/public/tables/authors/rows.jsonl\n",
"stderr": ""
}
},
{
"id": "pg_wc_l_view",
"seq": 610023,
"targets": [
"postgres"
],
"command": "wc -l /pg/public/views/recent_books/rows.jsonl",
"expect": {
"exit": 0,
"stdout": "3 /pg/public/views/recent_books/rows.jsonl\n",
"stderr": ""
}
}
]
}
integ/resources/qdrant/ops.json
+472
View File
@@ -0,0 +1,472 @@
{
"cases": [
{
"id": "qdrant_ls_root",
"seq": 640000,
"targets": [
"qdrant"
],
"command": "ls /db/",
"expect": {
"exit": 0,
"stdout": "cat\ndog\n",
"stderr": ""
}
},
{
"id": "qdrant_ls_group",
"seq": 640001,
"targets": [
"qdrant"
],
"command": "ls /db/cat",
"expect": {
"exit": 0,
"stdout": "big\nsmall\n",
"stderr": ""
}
},
{
"id": "qdrant_ls_leaf",
"seq": 640002,
"targets": [
"qdrant"
],
"command": "ls /db/cat/big",
"expect": {
"exit": 0,
"stdout": "1.json\n1.txt\n",
"stderr": ""
}
},
{
"id": "qdrant_find_txt",
"seq": 640003,
"targets": [
"qdrant"
],
"command": "find /db/ -name '*.txt'",
"expect": {
"exit": 0,
"stdout": "/db/cat/big/1.txt\n/db/cat/small/2.txt\n/db/dog/big/3.txt\n/db/dog/small/4.txt\n",
"stderr": ""
}
},
{
"id": "qdrant_find_json",
"seq": 640004,
"targets": [
"qdrant"
],
"command": "find /db/ -name '*.json'",
"expect": {
"exit": 0,
"stdout": "/db/cat/big/1.json\n/db/cat/small/2.json\n/db/dog/big/3.json\n/db/dog/small/4.json\n",
"stderr": ""
}
},
{
"id": "qdrant_cat_txt",
"seq": 640005,
"targets": [
"qdrant"
],
"command": "cat /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_cat_json",
"seq": 640006,
"targets": [
"qdrant"
],
"command": "cat /db/cat/big/1.json",
"expect": {
"exit": 0,
"stdout": "{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_wc_c_txt",
"seq": 640007,
"targets": [
"qdrant"
],
"command": "wc -c /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "17 /db/cat/big/1.txt\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_text",
"seq": 640008,
"targets": [
"qdrant"
],
"command": "grep orange /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_json_field",
"seq": 640009,
"targets": [
"qdrant"
],
"command": "grep label /db/cat/big/1.json",
"expect": {
"exit": 0,
"stdout": "{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_i",
"seq": 640010,
"targets": [
"qdrant"
],
"command": "grep -i ORANGE /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_n",
"seq": 640011,
"targets": [
"qdrant"
],
"command": "grep -n cat /db/cat/big/1.json",
"expect": {
"exit": 0,
"stdout": "1:{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_c",
"seq": 640012,
"targets": [
"qdrant"
],
"command": "grep -c cat /db/cat/big/1.json",
"expect": {
"exit": 0,
"stdout": "1\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_o",
"seq": 640013,
"targets": [
"qdrant"
],
"command": "grep -o cat /db/cat/big/1.json",
"expect": {
"exit": 0,
"stdout": "cat\ncat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_w",
"seq": 640014,
"targets": [
"qdrant"
],
"command": "grep -w big /db/cat/big/1.json",
"expect": {
"exit": 0,
"stdout": "{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_F_literal",
"seq": 640015,
"targets": [
"qdrant"
],
"command": "grep -F 'orange cat' /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_E_alt",
"seq": 640016,
"targets": [
"qdrant"
],
"command": "grep -E \"orange|brown\" /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_v",
"seq": 640017,
"targets": [
"qdrant"
],
"command": "grep -v zebra /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_multi",
"seq": 640018,
"targets": [
"qdrant"
],
"command": "grep small /db/cat/small/2.json /db/dog/small/4.json",
"expect": {
"exit": 0,
"stdout": "/db/cat/small/2.json:{\"label\":\"cat\",\"kind\":\"small\",\"name\":\"a small grey cat\",\"id\":2}\n/db/dog/small/4.json:{\"label\":\"dog\",\"kind\":\"small\",\"name\":\"a small white dog\",\"id\":4}\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_r_group",
"seq": 640019,
"targets": [
"qdrant"
],
"command": "grep -r orange /db/cat",
"expect": {
"exit": 0,
"stdout": "/db/cat/big/1.json:{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n/db/cat/big/1.txt:a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_rl",
"seq": 640020,
"targets": [
"qdrant"
],
"command": "grep -rl cat /db/",
"expect": {
"exit": 0,
"stdout": "/db/cat/big/1.json\n/db/cat/big/1.txt\n/db/cat/small/2.json\n/db/cat/small/2.txt\n",
"stderr": ""
}
},
{
"id": "qdrant_pipe_grep_stdin",
"seq": 640021,
"targets": [
"qdrant"
],
"command": "cat /db/cat/big/1.json | grep orange",
"expect": {
"exit": 0,
"stdout": "{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_rg_basic",
"seq": 640022,
"targets": [
"qdrant"
],
"command": "rg orange /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_du_leaf",
"seq": 640023,
"targets": [
"qdrant"
],
"command": "du /db/cat/big",
"expect": {
"exit": 0,
"stdout": "79\t/db/cat/big\n",
"stderr": ""
}
},
{
"id": "qdrant_du_group",
"seq": 640024,
"targets": [
"qdrant"
],
"command": "du /db/cat",
"expect": {
"exit": 0,
"stdout": "79\t/db/cat/big\n81\t/db/cat/small\n160\t/db/cat\n",
"stderr": ""
}
},
{
"id": "qdrant_du_root",
"seq": 640025,
"targets": [
"qdrant"
],
"command": "du /db/",
"expect": {
"exit": 0,
"stdout": "79\t/db/cat/big\n81\t/db/cat/small\n160\t/db/cat\n77\t/db/dog/big\n83\t/db/dog/small\n160\t/db/dog\n320\t/db\n",
"stderr": ""
}
},
{
"id": "qdrant_du_c_multi",
"seq": 640026,
"targets": [
"qdrant"
],
"command": "du -c /db/cat /db/dog",
"expect": {
"exit": 0,
"stdout": "79\t/db/cat/big\n81\t/db/cat/small\n160\t/db/cat\n77\t/db/dog/big\n83\t/db/dog/small\n160\t/db/dog\n320\ttotal\n",
"stderr": ""
}
},
{
"id": "qdrant_sym_ln",
"seq": 640027,
"targets": [
"qdrant"
],
"command": "ln -s /db/cat/big/1.json /db/meta_link",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "qdrant_sym_readlink",
"seq": 640028,
"targets": [
"qdrant"
],
"command": "readlink /db/meta_link",
"expect": {
"exit": 0,
"stdout": "/db/cat/big/1.json\n",
"stderr": ""
}
},
{
"id": "qdrant_sym_cat",
"seq": 640029,
"targets": [
"qdrant"
],
"command": "cat /db/meta_link",
"expect": {
"exit": 0,
"stdout": "{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_sym_grep",
"seq": 640030,
"targets": [
"qdrant"
],
"command": "grep label /db/meta_link",
"expect": {
"exit": 0,
"stdout": "{\"label\":\"cat\",\"kind\":\"big\",\"name\":\"a big orange cat\",\"id\":1}\n",
"stderr": ""
}
},
{
"id": "qdrant_sym_ls",
"seq": 640031,
"targets": [
"qdrant"
],
"command": "ls -F /db/ | grep meta_link",
"expect": {
"exit": 0,
"stdout": "meta_link@\n",
"stderr": ""
}
},
{
"id": "qdrant_sym_rm",
"seq": 640032,
"targets": [
"qdrant"
],
"command": "rm /db/meta_link && ls /db/",
"expect": {
"exit": 0,
"stdout": "cat\ndog\n",
"stderr": ""
}
},
{
"id": "qdrant_grep_q_match",
"seq": 640033,
"targets": [
"qdrant"
],
"command": "grep -q cat /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "",
"stderr": ""
}
},
{
"id": "qdrant_grep_q_no_match",
"seq": 640034,
"targets": [
"qdrant"
],
"command": "grep -q zebra /db/cat/big/1.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": ""
}
},
{
"id": "qdrant_grep_no_match",
"seq": 640035,
"targets": [
"qdrant"
],
"command": "grep zebra /db/cat/big/1.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": ""
}
}
]
}
integ/resources/qdrant/probe.json
+98
View File
@@ -0,0 +1,98 @@
{
"cases": [
{
"id": "qdrant_cat_missing",
"seq": 640036,
"targets": [
"qdrant"
],
"command": "cat /db/__nf_missing__.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "cat: /db/__nf_missing__.txt: Is a directory\n"
}
},
{
"id": "qdrant_write_rejected",
"seq": 640037,
"targets": [
"qdrant"
],
"command": "echo hi > /db/cat/big/1.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "/db/cat/big/1.txt: Operation not supported\n"
}
},
{
"id": "qdrant_sed_stream_1p",
"seq": 640038,
"targets": [
"qdrant"
],
"command": "sed -n 1p /db/cat/big/1.txt",
"expect": {
"exit": 0,
"stdout": "a big orange cat\n",
"stderr": ""
}
},
{
"id": "qdrant_sed_i_readonly",
"seq": 640039,
"targets": [
"qdrant"
],
"command": "sed -i s/x/y/ /db/cat/big/1.txt",
"expect": {
"exit": 1,
"stdout": "",
"stderr": "sed: -i not supported on this backend: Permission denied\n"
}
},
{
"id": "qdrant_prov_cat",
"seq": 640040,
"targets": [
"qdrant"
],
"command": "cat /db/cat/big/1.txt",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=17 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
},
{
"id": "qdrant_prov_grep",
"seq": 640041,
"targets": [
"qdrant"
],
"command": "grep x /db/cat/big/1.txt",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=17 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
},
{
"id": "qdrant_prov_ls",
"seq": 640042,
"targets": [
"qdrant"
],
"command": "ls /db/cat/big",
"provision": true,
"expect": {
"exit": 0,
"stdout": "net=0 write=0 cache=0 ops=1 hits=0 precision=exact\n",
"stderr": ""
}
}
]
}
integ/runners/python/adapters.py
+434 -1
View File
@@ -15,6 +15,7 @@
import asyncio
import base64
import functools
import gzip
import imaplib
import importlib.util
import json
@@ -32,9 +33,13 @@ from pathlib import Path
from types import ModuleType
import aiohttp
import asyncpg
import boto3
import chromadb
import lancedb
from moto.server import ThreadedMotoServer
from pymongo import AsyncMongoClient
from qdrant_client import AsyncQdrantClient, models
from mirage import MountMode, Workspace
from mirage.accessor.onedrive import OneDriveConfig
@@ -46,6 +51,7 @@ from mirage.resource.aliyun import AliyunConfig, AliyunResource
from mirage.resource.backblaze import BackblazeConfig, BackblazeResource
from mirage.resource.box import BoxConfig, BoxResource
from mirage.resource.ceph import CephConfig, CephResource
from mirage.resource.chroma import ChromaConfig, ChromaResource
from mirage.resource.databricks_volume import (DatabricksVolumeConfig,
DatabricksVolumeResource)
from mirage.resource.dify import DifyConfig, DifyResource
@@ -70,13 +76,18 @@ from mirage.resource.gslides.config import GSlidesConfig
from mirage.resource.gslides.gslides import GSlidesResource
from mirage.resource.hf_buckets import HfBucketsConfig, HfBucketsResource
from mirage.resource.jaeger import JaegerConfig, JaegerResource
from mirage.resource.lancedb import LanceDBConfig, LanceDBResource
from mirage.resource.langfuse import LangfuseConfig, LangfuseResource
from mirage.resource.linear import LinearConfig, LinearResource
from mirage.resource.mem0 import Mem0Config, Mem0Resource
from mirage.resource.minio import MinIOConfig, MinIOResource
from mirage.resource.mongodb import MongoDBConfig, MongoDBResource
from mirage.resource.nextcloud import NextcloudConfig, NextcloudResource
from mirage.resource.notion import NotionConfig, NotionResource
from mirage.resource.oci import OCIConfig, OCIResource
from mirage.resource.onedrive.onedrive import OneDriveResource
from mirage.resource.postgres import PostgresConfig, PostgresResource
from mirage.resource.qdrant import QdrantConfig, QdrantResource
from mirage.resource.qingstor import QingStorConfig, QingStorResource
from mirage.resource.r2 import R2Config, R2Resource
from mirage.resource.ram import RAMResource
@@ -318,6 +329,11 @@ def _load_ssh_server() -> ModuleType:
Path(__file__).resolve().parents[2] / "server" / "ssh_server.py")
def _load_notion_server() -> ModuleType:
return _load_module(
Path(__file__).resolve().parents[2] / "server" / "notion_server.py")
def _load_box_server() -> ModuleType:
return _load_module(
Path(__file__).resolve().parents[2] / "server" / "box_server.py")
@@ -1073,8 +1089,361 @@ class SharePointService:
await self.runner.cleanup()
class NotionService:
def __init__(self, server, port: int) -> None:
self.server = server
self.port = port
@classmethod
async def create(cls) -> "NotionService":
module = _load_notion_server()
server, port = module.start_server()
return cls(server, port)
def resource(self, mount: dict) -> NotionResource:
return NotionResource(config=NotionConfig(
api_key="integ-test", base_url=f"http://127.0.0.1:{self.port}/v1"))
async def teardown(self) -> None:
self.server.shutdown()
LANCEDB_ROWS = [
{
"id": 1,
"label": "cat",
"kind": "big",
"name": "a big orange cat"
},
{
"id": 2,
"label": "cat",
"kind": "small",
"name": "a small grey cat"
},
{
"id": 3,
"label": "dog",
"kind": "big",
"name": "a big brown dog"
},
{
"id": 4,
"label": "dog",
"kind": "small",
"name": "a small white dog"
},
]
class LanceDBService:
def __init__(self, uri: str) -> None:
self.uri = uri
@classmethod
async def create(cls) -> "LanceDBService":
uri = tempfile.mkdtemp(prefix="mirage-integ-lancedb-")
db = lancedb.connect(uri)
db.create_table("animals", data=LANCEDB_ROWS)
return cls(uri)
def resource(self, mount: dict) -> LanceDBResource:
return LanceDBResource(
LanceDBConfig(uri=self.uri,
group_by=["label", "kind"],
id_column="id",
title_column="name",
text_column="name"))
async def teardown(self) -> None:
shutil.rmtree(self.uri, ignore_errors=True)
QDRANT_EMBED_DIM = 8
QDRANT_ROWS = [
(1, "cat", "big", "a big orange cat"),
(2, "cat", "small", "a small grey cat"),
(3, "dog", "big", "a big brown dog"),
(4, "dog", "small", "a small white dog"),
]
class QdrantService:
def __init__(self, host: str, port: int, collection: str) -> None:
self.host = host
self.port = port
self.collection = collection
@classmethod
async def create(cls) -> "QdrantService":
host = os.environ.get("QDRANT_HOST", "localhost")
port = int(os.environ.get("QDRANT_PORT", "6333"))
collection = f"mirage-integ-{uuid.uuid4().hex[:8]}"
client = AsyncQdrantClient(host=host, port=port)
try:
await client.create_collection(
collection,
vectors_config=models.VectorParams(
size=QDRANT_EMBED_DIM, distance=models.Distance.COSINE))
await client.upsert(collection,
points=[
models.PointStruct(id=i,
vector=[0.1] *
QDRANT_EMBED_DIM,
payload={
"label": label,
"kind": kind,
"name": name
})
for i, label, kind, name in QDRANT_ROWS
])
for field in ("label", "kind"):
await client.create_payload_index(
collection,
field_name=field,
field_schema=models.PayloadSchemaType.KEYWORD)
await asyncio.sleep(2)
finally:
await client.close()
return cls(host, port, collection)
def resource(self, mount: dict) -> QdrantResource:
return QdrantResource(
QdrantConfig(host=self.host,
port=self.port,
collection=self.collection,
group_by=["label", "kind"],
id_field="id",
text_field="name"))
async def teardown(self) -> None:
client = AsyncQdrantClient(host=self.host, port=self.port)
try:
await client.delete_collection(self.collection)
finally:
await client.close()
CHROMA_EMBED_DIM = 8
def _chroma_embedding(position: int) -> list[float]:
vector = [0.0] * CHROMA_EMBED_DIM
vector[position % CHROMA_EMBED_DIM] = 1.0
return vector
class ChromaService:
def __init__(self, host: str, port: int, collection_name: str) -> None:
self.host = host
self.port = port
self.collection_name = collection_name
@classmethod
async def create(cls) -> "ChromaService":
host = os.environ.get("CHROMA_HOST", "localhost")
port = int(os.environ.get("CHROMA_PORT", "8000"))
collection_name = f"mirage-integ-{uuid.uuid4().hex[:8]}"
seed_path = (Path(__file__).resolve().parents[2] / "server" /
"chroma_seed.json")
seed = json.loads(seed_path.read_text())
encoded = base64.b64encode(
gzip.compress(json.dumps(seed["path_tree"]).encode())).decode()
ids = ["__path_tree__"]
documents = [encoded]
metadatas: list[dict] = [{"kind": "path_tree"}]
embeddings = [_chroma_embedding(0)]
position = 1
for chunks in seed["chunks"].values():
for chunk in chunks:
slug = chunk["metadata"]["page_slug"]
index = chunk["metadata"]["chunk_index"]
ids.append(f"{slug}#{index}")
documents.append(chunk["document"])
metadatas.append(chunk["metadata"])
embeddings.append(_chroma_embedding(position))
position += 1
client = await chromadb.AsyncHttpClient(host=host, port=port)
collection = await client.create_collection(collection_name)
await collection.add(ids=ids,
documents=documents,
metadatas=metadatas,
embeddings=embeddings)
return cls(host, port, collection_name)
def resource(self, mount: dict) -> ChromaResource:
return ChromaResource(
config=ChromaConfig(host=self.host,
port=self.port,
collection_name=self.collection_name))
async def teardown(self) -> None:
client = await chromadb.AsyncHttpClient(host=self.host, port=self.port)
await client.delete_collection(self.collection_name)
MONGODB_DB = "mirage_integ"
MONGODB_BOOKS = [
{
"_id": 1,
"title": "alpha",
"author": "ada",
"year": 2020,
"tags": ["fiction", "classic"],
"rating": 4.5,
},
{
"_id": 2,
"title": "beta",
"author": "ben",
"year": 2021,
"tags": ["fiction"],
"rating": 3.2,
},
{
"_id": 3,
"title": "gamma",
"author": "cara",
"year": 2022,
"rating": 5.0,
},
{
"_id": 4,
"title": "delta",
"author": "ada",
"year": 2023,
"tags": ["history"],
"rating": 4.0,
},
{
"_id": 5,
"title": "epsilon",
"author": "ben",
"year": 2024,
"rating": 2.5,
},
]
MONGODB_AUTHORS = [
{
"_id": 1,
"name": "ada",
"books": 2
},
{
"_id": 2,
"name": "ben",
"books": 2
},
{
"_id": 3,
"name": "cara",
"books": 1
},
]
class MongoDBService:
def __init__(self, uri: str) -> None:
self.uri = uri
@classmethod
async def create(cls) -> "MongoDBService":
uri = os.environ["MONGODB_URI"]
client: AsyncMongoClient = AsyncMongoClient(uri)
try:
await client.drop_database(MONGODB_DB)
db = client[MONGODB_DB]
await db["books"].insert_many([dict(d) for d in MONGODB_BOOKS])
await db["authors"].insert_many([dict(d) for d in MONGODB_AUTHORS])
await db.create_collection(
"recent_books",
viewOn="books",
pipeline=[{
"$match": {
"year": {
"$gte": 2022
}
}
}],
)
finally:
await client.close()
return cls(uri)
def resource(self, mount: dict) -> MongoDBResource:
return MongoDBResource(
config=MongoDBConfig(uri=self.uri, databases=[MONGODB_DB]))
async def teardown(self) -> None:
return None
POSTGRES_BOOKS = [
(1, "alpha", "ada", 2020, 4.5),
(2, "beta", "ben", 2021, 3.2),
(3, "gamma", "cara", 2022, 5.0),
(4, "delta", "ada", 2023, 4.0),
(5, "epsilon", "ben", 2024, 2.5),
]
POSTGRES_AUTHORS = [
(1, "ada", 2),
(2, "ben", 2),
(3, "cara", 1),
]
class PostgresService:
def __init__(self, dsn: str) -> None:
self.dsn = dsn
@classmethod
async def create(cls) -> "PostgresService":
dsn = os.environ["POSTGRES_DSN"]
conn = await asyncpg.connect(dsn)
try:
await conn.execute("DROP VIEW IF EXISTS recent_books")
await conn.execute("DROP TABLE IF EXISTS books")
await conn.execute("DROP TABLE IF EXISTS authors")
await conn.execute(
"CREATE TABLE books (id int PRIMARY KEY, title text, "
"author text, year int, rating double precision)")
await conn.execute("CREATE TABLE authors (id int PRIMARY KEY, "
"name text, books int)")
await conn.executemany(
"INSERT INTO books (id, title, author, year, rating) "
"VALUES ($1, $2, $3, $4, $5)", POSTGRES_BOOKS)
await conn.executemany(
"INSERT INTO authors (id, name, books) VALUES ($1, $2, $3)",
POSTGRES_AUTHORS)
await conn.execute("CREATE VIEW recent_books AS SELECT * FROM "
"books WHERE year >= 2022")
await conn.execute("ANALYZE books")
await conn.execute("ANALYZE authors")
finally:
await conn.close()
return cls(dsn)
def resource(self, mount: dict) -> PostgresResource:
return PostgresResource(PostgresConfig(dsn=self.dsn,
max_read_rows=200))
async def teardown(self) -> None:
return None
Service = (S3Service | OneDriveService | SharePointService | Mem0Service
| SSHService
| SSHService | PostgresService | MongoDBService | ChromaService
| QdrantService | LanceDBService | NotionService
| NextcloudService | GwsService | HfService | BoxService
| DropboxService | GridFSService | SlackService | TrelloService
| LinearService | DifyService | DatabricksVolumeService
@@ -1148,6 +1517,52 @@ def build_mem0(
return service.resource(mount), _noop
def build_postgres(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, PostgresService)
resource = service.resource(mount)
return resource, resource.accessor.close
def build_mongodb(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, MongoDBService)
resource = service.resource(mount)
return resource, resource.accessor.close
def build_chroma(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, ChromaService)
return service.resource(mount), _noop
def build_qdrant(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, QdrantService)
resource = service.resource(mount)
return resource, resource.accessor.close
def build_lancedb(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, LanceDBService)
resource = service.resource(mount)
return resource, resource.accessor.close
def build_notion(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
assert isinstance(service, NotionService)
return service.resource(mount), _noop
def build_hf(
mount: dict, run_id: str, service: Service | None
) -> tuple[object, Callable[[], Awaitable[None]]]:
@@ -1298,6 +1713,12 @@ BUILDERS = {
"onedrive": build_onedrive,
"sharepoint": build_sharepoint,
"mem0": build_mem0,
"postgres": build_postgres,
"mongodb": build_mongodb,
"chroma": build_chroma,
"qdrant": build_qdrant,
"lancedb": build_lancedb,
"notion": build_notion,
"ssh": build_ssh,
"nextcloud": build_nextcloud,
"gdrive": build_gdrive,
@@ -1332,6 +1753,18 @@ async def make_service(target: dict, run_id: str) -> "Service | None":
return await SharePointService.create()
if target.get("service") == "mem0":
return await Mem0Service.create()
if target.get("service") == "postgres":
return await PostgresService.create()
if target.get("service") == "mongodb":
return await MongoDBService.create()
if target.get("service") == "chroma":
return await ChromaService.create()
if target.get("service") == "qdrant":
return await QdrantService.create()
if target.get("service") == "lancedb":
return await LanceDBService.create()
if target.get("service") == "notion":
return await NotionService.create()
if target.get("service") == "ssh":
return await SSHService.create(run_id, target)
if target.get("service") == "nextcloud":
integ/runners/python/main.py
+26
View File
@@ -141,6 +141,32 @@ async def main() -> None:
and not os.environ.get("SLACK_URL")):
print(f"skip [{target_id}]: SLACK_URL not set", file=sys.stderr)
continue
if (target.get("service") == "postgres"
and not os.environ.get("POSTGRES_DSN")):
print(f"skip [{target_id}]: POSTGRES_DSN not set", file=sys.stderr)
continue
if (target.get("service") == "mongodb"
and not os.environ.get("MONGODB_URI")):
print(f"skip [{target_id}]: MONGODB_URI not set", file=sys.stderr)
continue
if (target.get("service") == "chroma"
and not os.environ.get("CHROMA_HOST")):
print(f"skip [{target_id}]: CHROMA_HOST not set", file=sys.stderr)
continue
if (target.get("service") == "qdrant"
and not os.environ.get("QDRANT_HOST")):
print(f"skip [{target_id}]: QDRANT_HOST not set", file=sys.stderr)
continue
if (target.get("service") == "lancedb"
and not os.environ.get("LANCEDB_ENABLED")):
print(f"skip [{target_id}]: LANCEDB_ENABLED not set",
file=sys.stderr)
continue
if (target.get("service") == "notion"
and not os.environ.get("NOTION_ENABLED")):
print(f"skip [{target_id}]: NOTION_ENABLED not set",
file=sys.stderr)
continue
if (target.get("service") == "jaeger"
and not os.environ.get("JAEGER_URL")):
print(f"skip [{target_id}]: JAEGER_URL not set", file=sys.stderr)
integ/runners/typescript/adapters.ts
+296
View File
@@ -15,6 +15,7 @@
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join, relative, sep } from 'node:path'
import { gzipSync } from 'node:zlib'
import {
CreateBucketCommand,
DeleteBucketCommand,
@@ -28,6 +29,7 @@ import {
BackblazeResource,
BoxResource,
CephResource,
ChromaResource,
DatabricksVolumeResource,
DifyResource,
DigitalOceanResource,
@@ -44,15 +46,20 @@ import {
GSlidesResource,
HfBucketsResource,
JaegerResource,
LanceDBResource,
LangfuseResource,
LinearResource,
MinIOResource,
Mem0Resource,
MongoDBResource,
NotionResource,
ConsistencyPolicy,
MountMode,
NextcloudResource,
OCIResource,
OneDriveResource,
PostgresResource,
QdrantResource,
QingStorResource,
R2Resource,
RAMResource,
@@ -69,9 +76,15 @@ import {
WasabiResource,
Workspace,
} from '@struktoai/mirage-node'
import * as lancedb from '@lancedb/lancedb'
import { QdrantClient } from '@qdrant/js-client-rest'
import { ChromaClient } from 'chromadb'
import { ImapFlow } from 'imapflow'
import { Double, MongoClient } from 'mongodb'
import pg from 'pg'
import { installFakeNavigator, makeMockRoot } from '../../../typescript/packages/browser/src/test-utils.ts'
import { startFakeDropbox, type FakeDropbox } from '../../server/dropbox.ts'
import { startMockServer as startNotionMock } from '../../server/notion_server.ts'
import { integRoot, walkFiles } from './harness.ts'
import type { ExecWorkspace, Mount, Target } from './harness.ts'
import { startPythonServer } from './server_process.ts'
@@ -649,6 +662,283 @@ async function openGraphConsistency(
return { ws: ws as unknown as ExecWorkspace, mutate, cleanup }
}
async function openNotion(target: Target): Promise<Open> {
const { server, port } = await startNotionMock()
const mounts: Record<string, NotionResource | [NotionResource, MountMode]> = {}
for (const mount of target.mounts) {
const resource = new NotionResource({
apiKey: 'integ-test',
baseUrl: `http://127.0.0.1:${String(port)}/v1`,
})
mounts[mount.path] = mount.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
const cleanup = async (): Promise<void> => {
await ws.close()
server.close()
}
return { ws: ws as unknown as ExecWorkspace, cleanup }
}
const LANCEDB_ROWS: ReadonlyArray<Record<string, unknown>> = [
{ id: 1, label: 'cat', kind: 'big', name: 'a big orange cat' },
{ id: 2, label: 'cat', kind: 'small', name: 'a small grey cat' },
{ id: 3, label: 'dog', kind: 'big', name: 'a big brown dog' },
{ id: 4, label: 'dog', kind: 'small', name: 'a small white dog' },
]
async function openLancedb(target: Target): Promise<Open> {
const uri = mkdtempSync(join(tmpdir(), 'mirage-integ-lancedb-'))
const db = await lancedb.connect(uri)
await db.createTable('animals', LANCEDB_ROWS as Record<string, unknown>[])
const mounts: Record<string, LanceDBResource | [LanceDBResource, MountMode]> = {}
for (const mount of target.mounts) {
const resource = new LanceDBResource({
uri,
groupBy: ['label', 'kind'],
idColumn: 'id',
titleColumn: 'name',
textColumn: 'name',
})
mounts[mount.path] = mount.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
const cleanup = async (): Promise<void> => {
await ws.close()
rmSync(uri, { recursive: true, force: true })
}
return { ws: ws as unknown as ExecWorkspace, cleanup }
}
const QDRANT_EMBED_DIM = 8
const QDRANT_ROWS: ReadonlyArray<readonly [number, string, string, string]> = [
[1, 'cat', 'big', 'a big orange cat'],
[2, 'cat', 'small', 'a small grey cat'],
[3, 'dog', 'big', 'a big brown dog'],
[4, 'dog', 'small', 'a small white dog'],
]
async function openQdrant(target: Target): Promise<Open> {
const host = process.env.QDRANT_HOST ?? 'localhost'
const port = Number.parseInt(process.env.QDRANT_PORT ?? '6333', 10)
const collection = `mirage-integ-${runId()}`
const client = new QdrantClient({ host, port })
await client.createCollection(collection, {
vectors: { size: QDRANT_EMBED_DIM, distance: 'Cosine' },
})
await client.upsert(collection, {
points: QDRANT_ROWS.map(([id, label, kind, name]) => ({
id,
vector: Array<number>(QDRANT_EMBED_DIM).fill(0.1),
payload: { label, kind, name },
})),
})
for (const field of ['label', 'kind']) {
await client.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' })
}
await new Promise((r) => setTimeout(r, 2000))
const mounts: Record<string, QdrantResource | [QdrantResource, MountMode]> = {}
for (const mount of target.mounts) {
const resource = new QdrantResource({
host,
port,
collection,
groupBy: ['label', 'kind'],
idField: 'id',
textField: 'name',
})
mounts[mount.path] = mount.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
const cleanup = async (): Promise<void> => {
await ws.close()
await new QdrantClient({ host, port }).deleteCollection(collection)
}
return { ws: ws as unknown as ExecWorkspace, cleanup }
}
const CHROMA_EMBED_DIM = 8
interface ChromaChunk {
document: string
metadata: { page_slug: string; chunk_index: number }
}
interface ChromaSeed {
path_tree: Record<string, unknown>
chunks: Record<string, ChromaChunk[]>
}
function chromaEmbedding(position: number): number[] {
const vector = new Array<number>(CHROMA_EMBED_DIM).fill(0)
vector[position % CHROMA_EMBED_DIM] = 1
return vector
}
async function seedChroma(host: string, port: number, collectionName: string): Promise<void> {
const seed = JSON.parse(
readFileSync(join(integRoot(), 'server', 'chroma_seed.json'), 'utf8'),
) as ChromaSeed
const encoded = gzipSync(Buffer.from(JSON.stringify(seed.path_tree))).toString('base64')
const ids = ['__path_tree__']
const documents = [encoded]
const metadatas: Record<string, string | number>[] = [{ kind: 'path_tree' }]
const embeddings = [chromaEmbedding(0)]
let position = 1
for (const chunks of Object.values(seed.chunks)) {
for (const chunk of chunks) {
ids.push(`${chunk.metadata.page_slug}#${String(chunk.metadata.chunk_index)}`)
documents.push(chunk.document)
metadatas.push(chunk.metadata)
embeddings.push(chromaEmbedding(position))
position += 1
}
}
const client = new ChromaClient({ host, port })
const collection = await client.createCollection({ name: collectionName, embeddingFunction: null })
await collection.add({ ids, documents, metadatas, embeddings })
}
async function openChroma(target: Target): Promise<Open> {
const host = process.env.CHROMA_HOST ?? 'localhost'
const port = Number.parseInt(process.env.CHROMA_PORT ?? '8000', 10)
const collectionName = `mirage-integ-${runId()}`
await seedChroma(host, port, collectionName)
const mounts: Record<string, ChromaResource | [ChromaResource, MountMode]> = {}
for (const mount of target.mounts) {
const resource = new ChromaResource({ host, port, collectionName })
mounts[mount.path] = mount.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
const cleanup = async (): Promise<void> => {
await ws.close()
await new ChromaClient({ host, port }).deleteCollection({ name: collectionName })
}
return { ws: ws as unknown as ExecWorkspace, cleanup }
}
const MONGODB_DB = 'mirage_integ'
const MONGODB_BOOKS: ReadonlyArray<Record<string, unknown>> = [
{ _id: 1, title: 'alpha', author: 'ada', year: 2020, tags: ['fiction', 'classic'], rating: 4.5 },
{ _id: 2, title: 'beta', author: 'ben', year: 2021, tags: ['fiction'], rating: 3.2 },
{ _id: 3, title: 'gamma', author: 'cara', year: 2022, rating: 5.0 },
{ _id: 4, title: 'delta', author: 'ada', year: 2023, tags: ['history'], rating: 4.0 },
{ _id: 5, title: 'epsilon', author: 'ben', year: 2024, rating: 2.5 },
]
const MONGODB_AUTHORS: ReadonlyArray<Record<string, unknown>> = [
{ _id: 1, name: 'ada', books: 2 },
{ _id: 2, name: 'ben', books: 2 },
{ _id: 3, name: 'cara', books: 1 },
]
async function seedMongodb(uri: string): Promise<void> {
const client = new MongoClient(uri)
await client.connect()
try {
const db = client.db(MONGODB_DB)
await db.dropDatabase()
// Python seeds floats (BSON double); insert Double so the inferred schema
// and rendered documents match byte-for-byte across languages.
await db
.collection('books')
.insertMany(MONGODB_BOOKS.map((d) => ({ ...d, rating: new Double(d.rating as number) })))
await db.collection('authors').insertMany(MONGODB_AUTHORS.map((d) => ({ ...d })))
await db.createCollection('recent_books', {
viewOn: 'books',
pipeline: [{ $match: { year: { $gte: 2022 } } }],
})
} finally {
await client.close()
}
}
async function openMongodb(target: Target): Promise<Open> {
const uri = process.env.MONGODB_URI
if (uri === undefined) throw new Error('mongodb target requires MONGODB_URI')
await seedMongodb(uri)
const resources: MongoDBResource[] = []
const mounts: Record<string, MongoDBResource | [MongoDBResource, MountMode]> = {}
for (const mount of target.mounts) {
const resource = new MongoDBResource({ uri, databases: [MONGODB_DB] })
resources.push(resource)
mounts[mount.path] = mount.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
const cleanup = async (): Promise<void> => {
await ws.close()
for (const resource of resources) await resource.close()
}
return { ws: ws as unknown as ExecWorkspace, cleanup }
}
const POSTGRES_BOOKS: ReadonlyArray<readonly [number, string, string, number, number]> = [
[1, 'alpha', 'ada', 2020, 4.5],
[2, 'beta', 'ben', 2021, 3.2],
[3, 'gamma', 'cara', 2022, 5.0],
[4, 'delta', 'ada', 2023, 4.0],
[5, 'epsilon', 'ben', 2024, 2.5],
]
const POSTGRES_AUTHORS: ReadonlyArray<readonly [number, string, number]> = [
[1, 'ada', 2],
[2, 'ben', 2],
[3, 'cara', 1],
]
async function seedPostgres(dsn: string): Promise<void> {
const client = new pg.Client({ connectionString: dsn })
await client.connect()
try {
await client.query('DROP VIEW IF EXISTS recent_books')
await client.query('DROP TABLE IF EXISTS books')
await client.query('DROP TABLE IF EXISTS authors')
await client.query(
'CREATE TABLE books (id int PRIMARY KEY, title text, author text, year int, rating double precision)',
)
await client.query('CREATE TABLE authors (id int PRIMARY KEY, name text, books int)')
for (const [id, title, author, year, rating] of POSTGRES_BOOKS) {
await client.query(
'INSERT INTO books (id, title, author, year, rating) VALUES ($1, $2, $3, $4, $5)',
[id, title, author, year, rating],
)
}
for (const [id, name, books] of POSTGRES_AUTHORS) {
await client.query('INSERT INTO authors (id, name, books) VALUES ($1, $2, $3)', [
id,
name,
books,
])
}
await client.query('CREATE VIEW recent_books AS SELECT * FROM books WHERE year >= 2022')
await client.query('ANALYZE books')
await client.query('ANALYZE authors')
} finally {
await client.end()
}
}
async function openPostgres(target: Target): Promise<Open> {
const dsn = process.env.POSTGRES_DSN
if (dsn === undefined) throw new Error('postgres target requires POSTGRES_DSN')
await seedPostgres(dsn)
const resources: PostgresResource[] = []
const mounts: Record<string, PostgresResource | [PostgresResource, MountMode]> = {}
for (const mount of target.mounts) {
const resource = new PostgresResource({ dsn, maxReadRows: 200 })
resources.push(resource)
mounts[mount.path] = mount.mode === 'read' ? [resource, MountMode.READ] : resource
}
const ws = new Workspace(mounts, { mode: MountMode.WRITE })
const cleanup = async (): Promise<void> => {
await ws.close()
for (const resource of resources) await resource.close()
}
return { ws: ws as unknown as ExecWorkspace, cleanup }
}
async function openMem0(target: Target): Promise<Open> {
const server = await startPythonServer('mem0_server.py')
const mounts: Record<string, Mem0Resource> = {}
@@ -1074,6 +1364,12 @@ export const ADAPTERS: Record<string, (target: Target) => Promise<Open>> = {
onedrive: openOneDrive,
sharepoint: openSharePoint,
mem0: openMem0,
postgres: openPostgres,
mongodb: openMongodb,
chroma: openChroma,
qdrant: openQdrant,
lancedb: openLancedb,
notion: openNotion,
github: openGitHub,
slack: openSlack,
trello: openTrello,
integ/runners/typescript/main.ts
+24
View File
@@ -184,6 +184,30 @@ async function main(): Promise<void> {
process.stderr.write(`skip [${id}]: LINEAR_ENDPOINT not set\n`)
continue
}
if (target.service === 'postgres' && !process.env.POSTGRES_DSN) {
process.stderr.write(`skip [${id}]: POSTGRES_DSN not set\n`)
continue
}
if (target.service === 'mongodb' && !process.env.MONGODB_URI) {
process.stderr.write(`skip [${id}]: MONGODB_URI not set\n`)
continue
}
if (target.service === 'chroma' && !process.env.CHROMA_HOST) {
process.stderr.write(`skip [${id}]: CHROMA_HOST not set\n`)
continue
}
if (target.service === 'qdrant' && !process.env.QDRANT_HOST) {
process.stderr.write(`skip [${id}]: QDRANT_HOST not set\n`)
continue
}
if (target.service === 'lancedb' && !process.env.LANCEDB_ENABLED) {
process.stderr.write(`skip [${id}]: LANCEDB_ENABLED not set\n`)
continue
}
if (target.service === 'notion' && !process.env.NOTION_ENABLED) {
process.stderr.write(`skip [${id}]: NOTION_ENABLED not set\n`)
continue
}
if (target.service === 'jaeger' && !process.env.JAEGER_URL) {
process.stderr.write(`skip [${id}]: JAEGER_URL not set\n`)
continue
integ/server/chroma_seed.json
+135
View File
@@ -0,0 +1,135 @@
{
"path_tree": {
"guides/quickstart.md": {
"size": 180,
"created_at": "2026-01-01T00:00:00Z",
"updated_at": "2026-02-01T00:00:00Z"
},
"guides/auth.md": {
"size": 190,
"created_at": "2026-01-15T00:00:00Z",
"updated_at": "2026-02-15T00:00:00Z"
},
"policies/refunds.md": {
"size": 150,
"created_at": "2026-02-01T00:00:00Z",
"updated_at": "2026-03-01T00:00:00Z"
},
"policies/privacy.md": {
"size": 120,
"created_at": "2026-02-10T00:00:00Z",
"updated_at": "2026-03-10T00:00:00Z"
},
"CHANGELOG.md": {
"size": 90
},
"policies/archived.md": {
"created_at": "2026-03-01T00:00:00Z"
}
},
"chunks": {
"policies/archived.md": [
{
"document": "Archived policy retained for records.",
"metadata": {
"page_slug": "policies/archived.md",
"chunk_index": 0
}
}
],
"guides/quickstart.md": [
{
"document": "Welcome to Acme. This quickstart gets you running fast.",
"metadata": {
"page_slug": "guides/quickstart.md",
"chunk_index": 0
}
},
{
"document": "Install the CLI with npm i -g acme then run acme login.",
"metadata": {
"page_slug": "guides/quickstart.md",
"chunk_index": 1
}
},
{
"document": "Set your token in the ACME_TOKEN environment variable.",
"metadata": {
"page_slug": "guides/quickstart.md",
"chunk_index": 2
}
}
],
"guides/auth.md": [
{
"document": "Authentication uses bearer tokens via the Authorization header.",
"metadata": {
"page_slug": "guides/auth.md",
"chunk_index": 0
}
},
{
"document": "Requests are rate limited to 100 calls per minute per token.",
"metadata": {
"page_slug": "guides/auth.md",
"chunk_index": 1
}
},
{
"document": "If you exceed the limit you receive HTTP 429 and must back off.",
"metadata": {
"page_slug": "guides/auth.md",
"chunk_index": 2
}
}
],
"policies/refunds.md": [
{
"document": "Refunds are available within 30 days of purchase.",
"metadata": {
"page_slug": "policies/refunds.md",
"chunk_index": 0
}
},
{
"document": "Email support to start a refund with your order id.",
"metadata": {
"page_slug": "policies/refunds.md",
"chunk_index": 1
}
},
{
"document": "Approved refunds are processed within five business days.",
"metadata": {
"page_slug": "policies/refunds.md",
"chunk_index": 2
}
}
],
"policies/privacy.md": [
{
"document": "Customer data is stored encrypted at rest and in transit.",
"metadata": {
"page_slug": "policies/privacy.md",
"chunk_index": 0
}
},
{
"document": "We never sell personal information to third parties.",
"metadata": {
"page_slug": "policies/privacy.md",
"chunk_index": 1
}
}
],
"CHANGELOG.md": [
{
"document": "v2.0 added rate limit headers and refund automation.",
"metadata": {
"page_slug": "CHANGELOG.md",
"chunk_index": 0
}
}
]
}
}
integ/notion.py → integ/server/notion_server.py
+2 -79
View File
@@ -12,16 +12,10 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import asyncio
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from cases import run_not_found, run_provision_probe, run_sed_readonly_probe
from mirage import MountMode, Workspace
from mirage.resource.notion import NotionConfig, NotionResource
MOUNT = "/notion"
PAGE_A = "aaaa1111-2222-3333-4444-555566667777"
PAGE_B = "bbbb2222-3333-4444-5555-666677778888"
@@ -290,79 +284,8 @@ class NotionMockHandler(BaseHTTPRequestHandler):
self.end_headers()
CASES: list[tuple[str, str]] = [
("ls_root", f"ls {MOUNT}/"),
("ls_pages", f"ls {MOUNT}/pages/"),
("ls_l_pages", f"ls -l {MOUNT}/pages/"),
("ls_page_a", f"ls {DIR_A}/"),
("stat_dir_a", f"stat -c '%n %y' {DIR_A}"),
("cat_page_a", f"cat {DIR_A}/page.json"),
("cat_child", f"cat {DIR_C}/page.json"),
("jq_title", f'jq ".title" {DIR_A}/page.json'),
("jq_markdown", f'jq ".markdown" {DIR_B}/page.json'),
("head_4", f"head -n 4 {DIR_A}/page.json"),
("wc_l_two", f"wc -l {DIR_A}/page.json {DIR_B}/page.json"),
("stat_page_json", f"stat {DIR_A}/page.json"),
("find_json", f"find {MOUNT}/pages/ -name page.json"),
("find_root_maxdepth0", f"find {MOUNT} -maxdepth 0"),
("find_root_name", f"find {MOUNT} -name notion"),
("pipe_grep", f"cat {DIR_B}/page.json | grep -c alpha"),
("grep_file", f"grep -n alpha {DIR_B}/page.json"),
("grep_multi", f"grep -c alpha {DIR_A}/page.json {DIR_B}/page.json"),
("grep_recursive", f"grep -rl alpha {MOUNT}/pages/"),
("realpath_dotdot", f"realpath -e {DIR_C}/../page.json"),
("ls_databases", f"ls {MOUNT}/databases/"),
("ls_database_dir", f"ls {DB_DIR}/"),
("cat_database_json", f"cat {DB_DIR}/database.json"),
("jq_db_props", f'jq ".properties | keys" {DB_DIR}/database.json'),
("cat_row", f"cat {ROW_1_DIR}/page.json"),
("du_pages", f"du {MOUNT}/pages/"),
("du_page_a", f"du {DIR_A}/"),
]
EXIT_CODE_CASES: list[tuple[str, str]] = [
("grep_c_match_exit", f"grep -c alpha {DIR_B}/page.json"),
("grep_c_no_match_exit", f"grep -c zzz {DIR_B}/page.json"),
("grep_rc_no_match_exit", f"grep -rc zzz {MOUNT}/pages/"),
]
async def _run(ws: Workspace, name: str, cmd: str) -> None:
result = await ws.execute(cmd)
out = await result.stdout_str()
print(f"=== {name} ===")
print(out, end="" if out.endswith("\n") else "\n")
async def _run_exit(ws: Workspace, name: str, cmd: str) -> None:
result = await ws.execute(cmd)
out = await result.stdout_str()
print(f"=== {name} ===")
print(f"exit={result.exit_code}")
if out:
print(out, end="" if out.endswith("\n") else "\n")
async def main() -> None:
def start_server() -> tuple[ThreadingHTTPServer, int]:
server = ThreadingHTTPServer(("127.0.0.1", 0), NotionMockHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
port = server.server_address[1]
try:
config = NotionConfig(api_key="integ-test",
base_url=f"http://127.0.0.1:{port}/v1")
resource = NotionResource(config=config)
ws = Workspace({MOUNT: resource}, mode=MountMode.READ)
for name, cmd in CASES:
await _run(ws, name, cmd)
for name, cmd in EXIT_CODE_CASES:
await _run_exit(ws, name, cmd)
await run_not_found(ws, MOUNT)
await run_provision_probe(ws, f"{DIR_A}/page.json")
await run_sed_readonly_probe(ws, f"{DIR_A}/page.json")
finally:
server.shutdown()
if __name__ == "__main__":
asyncio.run(main())
return server, server.server_address[1]
integ/notion.ts → integ/server/notion_server.ts
+4 -102
View File
@@ -19,10 +19,6 @@ import {
CallToolRequestSchema,
ListToolsRequestSchema,
} from "@modelcontextprotocol/sdk/types.js";
import { NotionResource as BrowserNotionResource } from "@struktoai/mirage-browser";
import { MemoryOAuthClientProvider } from "@struktoai/mirage-core";
import { MountMode, NotionResource, Workspace } from "@struktoai/mirage-node";
import { runNotFound, runProvisionProbe, runSedReadonlyProbe } from "./cases.ts";
const MOUNT = "/notion";
const PAGE_A = "aaaa1111-2222-3333-4444-555566667777";
@@ -38,8 +34,6 @@ const DIR_C = `${DIR_A}/Q1_Goals__${PAGE_C}`;
const DB_DIR = `${MOUNT}/databases/Tasks__${DB_TASKS}`;
const ROW_1_DIR = `${DB_DIR}/Write_spec__${ROW_1}`;
const DEC = new TextDecoder();
type Json = Record<string, unknown>;
function user(uid: string): Json {
@@ -180,7 +174,7 @@ function searchResults(args: Json): Json[] {
: Object.values(PAGES);
}
function startMockServer(): Promise<{ server: Server; port: number }> {
export function startMockServer(): Promise<{ server: Server; port: number }> {
const server = createServer((req, res) => {
const path = (req.url ?? "").split("?")[0] ?? "";
const parts = path.split("/").filter((part) => part !== "");
@@ -335,7 +329,7 @@ function buildMcpServer(): McpServer {
return server;
}
function startMockMcpServer(): Promise<{ server: Server; port: number }> {
export function startMockMcpServer(): Promise<{ server: Server; port: number }> {
const server = createServer((req, res) => {
void (async () => {
const mcp = buildMcpServer();
@@ -363,7 +357,7 @@ function startMockMcpServer(): Promise<{ server: Server; port: number }> {
});
}
const CASES: ReadonlyArray<readonly [string, string]> = [
export const CASES: ReadonlyArray<readonly [string, string]> = [
["ls_root", `ls ${MOUNT}/`],
["ls_pages", `ls ${MOUNT}/pages/`],
["ls_l_pages", `ls -l ${MOUNT}/pages/`],
@@ -393,100 +387,8 @@ const CASES: ReadonlyArray<readonly [string, string]> = [
["du_page_a", `du ${DIR_A}/`],
];
const EXIT_CODE_CASES: ReadonlyArray<readonly [string, string]> = [
export const EXIT_CODE_CASES: ReadonlyArray<readonly [string, string]> = [
["grep_c_match_exit", `grep -c alpha ${DIR_B}/page.json`],
["grep_c_no_match_exit", `grep -c zzz ${DIR_B}/page.json`],
["grep_rc_no_match_exit", `grep -rc zzz ${MOUNT}/pages/`],
];
async function runCase(
ws: Workspace,
name: string,
cmd: string,
): Promise<string> {
const result = await ws.execute(cmd);
const out = DEC.decode(result.stdout);
return `=== ${name} ===\n` + (out.endsWith("\n") ? out : out + "\n");
}
async function runExitCase(
ws: Workspace,
name: string,
cmd: string,
): Promise<string> {
const result = await ws.execute(cmd);
const out = DEC.decode(result.stdout);
let rendered = `=== ${name} ===\nexit=${String(result.exitCode)}\n`;
if (out !== "") rendered += out.endsWith("\n") ? out : out + "\n";
return rendered;
}
async function main(): Promise<void> {
const { server, port } = await startMockServer();
const { server: mcpServer, port: mcpPort } = await startMockMcpServer();
const restResource = new NotionResource({
apiKey: "integ-test",
baseUrl: `http://127.0.0.1:${String(port)}/v1`,
});
const restWs = new Workspace(
{ [MOUNT]: restResource },
{ mode: MountMode.READ },
);
const authProvider = new MemoryOAuthClientProvider({
clientMetadata: { redirect_uris: ["http://127.0.0.1/cb"] },
redirect: () => {},
});
const mcpResource = new BrowserNotionResource({
authProvider,
serverUrl: `http://127.0.0.1:${String(mcpPort)}/mcp`,
});
const mcpWs = new Workspace(
{ [MOUNT]: mcpResource },
{ mode: MountMode.READ },
);
try {
const allCases: ReadonlyArray<
readonly [
string,
string,
(ws: Workspace, name: string, cmd: string) => Promise<string>,
]
> = [
...CASES.map(([name, cmd]) => [name, cmd, runCase] as const),
...EXIT_CODE_CASES.map(
([name, cmd]) => [name, cmd, runExitCase] as const,
),
];
let mismatches = 0;
for (const [name, cmd, run] of allCases) {
const restOut = await run(restWs, name, cmd);
process.stdout.write(restOut);
const mcpOut = await run(mcpWs, name, cmd);
if (mcpOut !== restOut) {
mismatches += 1;
process.stderr.write(
`MCP/REST MISMATCH in ${name}:\n--- rest ---\n${restOut}--- mcp ---\n${mcpOut}`,
);
}
}
if (mismatches > 0) {
process.exitCode = 1;
} else {
const n = String(allCases.length);
process.stderr.write(`mcp parity: ${n}/${n} cases byte-identical\n`);
}
await runNotFound(restWs, MOUNT);
await runProvisionProbe(restWs, `${DIR_A}/page.json`);
await runSedReadonlyProbe(restWs, `${DIR_A}/page.json`);
} finally {
await restWs.close();
await mcpWs.close();
server.close();
mcpServer.close();
}
}
main().catch((err: unknown) => {
console.error(err);
process.exit(1);
});
integ/targets.json
+96
View File
@@ -722,6 +722,102 @@
}
]
},
{
"id": "postgres",
"hosts": [
"python",
"typescript-node"
],
"service": "postgres",
"mounts": [
{
"path": "/pg",
"resource": "postgres",
"backend": "postgres",
"mode": "read"
}
]
},
{
"id": "mongodb",
"hosts": [
"python",
"typescript-node"
],
"service": "mongodb",
"mounts": [
{
"path": "/mongodb",
"resource": "mongodb",
"backend": "mongodb",
"mode": "read"
}
]
},
{
"id": "chroma",
"hosts": [
"python",
"typescript-node"
],
"service": "chroma",
"mounts": [
{
"path": "/knowledge",
"resource": "chroma",
"backend": "chroma",
"mode": "read"
}
]
},
{
"id": "qdrant",
"hosts": [
"python",
"typescript-node"
],
"service": "qdrant",
"mounts": [
{
"path": "/db",
"resource": "qdrant",
"backend": "qdrant",
"mode": "read"
}
]
},
{
"id": "lancedb",
"hosts": [
"python",
"typescript-node"
],
"service": "lancedb",
"mounts": [
{
"path": "/db",
"resource": "lancedb",
"backend": "lancedb",
"mode": "read"
}
]
},
{
"id": "notion",
"hosts": [
"python",
"typescript-node"
],
"service": "notion",
"mounts": [
{
"path": "/notion",
"resource": "notion",
"backend": "notion",
"mode": "read"
}
]
},
{
"id": "hf",
"hosts": [
integ/truth_chroma.txt
-176
View File
@@ -1,176 +0,0 @@
=== ls ===
CHANGELOG.md
guides
policies
=== ls_guides ===
auth.md
quickstart.md
=== find_md ===
/knowledge/CHANGELOG.md
/knowledge/guides/auth.md
/knowledge/guides/quickstart.md
/knowledge/policies/archived.md
/knowledge/policies/privacy.md
/knowledge/policies/refunds.md
=== find_type_f ===
/knowledge/CHANGELOG.md
/knowledge/guides/auth.md
/knowledge/guides/quickstart.md
/knowledge/policies/archived.md
/knowledge/policies/privacy.md
/knowledge/policies/refunds.md
=== find_root_maxdepth0 ===
/knowledge
=== find_root_name ===
/knowledge
=== find_size_plus_100c ===
/knowledge/guides/auth.md
/knowledge/guides/quickstart.md
/knowledge/policies/privacy.md
/knowledge/policies/refunds.md
=== find_size_archived_kept ===
/knowledge/policies/archived.md
=== grep_cold_single ===
Authentication uses bearer tokens via the Authorization header.
=== grep_warm_single ===
Authentication uses bearer tokens via the Authorization header.
=== cat_auth ===
Authentication uses bearer tokens via the Authorization header.
Requests are rate limited to 100 calls per minute per token.
If you exceed the limit you receive HTTP 429 and must back off.
=== cat_quickstart ===
Welcome to Acme. This quickstart gets you running fast.
Install the CLI with npm i -g acme then run acme login.
Set your token in the ACME_TOKEN environment variable.
=== head_1 ===
Welcome to Acme. This quickstart gets you running fast.
=== tail_1 ===
Set your token in the ACME_TOKEN environment variable.
=== grep_429 ===
If you exceed the limit you receive HTTP 429 and must back off.
=== grep_e_multi ===
Authentication uses bearer tokens via the Authorization header.
If you exceed the limit you receive HTTP 429 and must back off.
=== grep_c_rate ===
1
=== grep_r_refund ===
/knowledge/policies/refunds.md:Email support to start a refund with your order id.
/knowledge/policies/refunds.md:Approved refunds are processed within five business days.
=== grep_cold_count ===
1
=== grep_warm_count ===
1
=== grep_rl_encrypted ===
/knowledge/policies/privacy.md
=== grep_v_bearer ===
Requests are rate limited to 100 calls per minute per token.
If you exceed the limit you receive HTTP 429 and must back off.
=== grep_rE_alternation ===
/knowledge/CHANGELOG.md:v2.0 added rate limit headers and refund automation.
/knowledge/guides/auth.md:Requests are rate limited to 100 calls per minute per token.
/knowledge/policies/refunds.md:Email support to start a refund with your order id.
/knowledge/policies/refunds.md:Approved refunds are processed within five business days.
=== wc_l_auth ===
2 /knowledge/guides/auth.md
=== sort_auth ===
Authentication uses bearer tokens via the Authorization header.
If you exceed the limit you receive HTTP 429 and must back off.
Requests are rate limited to 100 calls per minute per token.
=== uniq_auth ===
Authentication uses bearer tokens via the Authorization header.
Requests are rate limited to 100 calls per minute per token.
If you exceed the limit you receive HTTP 429 and must back off.
=== uniq_w0_auth ===
Authentication uses bearer tokens via the Authorization header.
=== stat_name_auth ===
/knowledge/guides/auth.md
=== cut_d_f1 ===
Welcome
Install
Set
=== awk_first_word ===
Welcome
Install
Set
=== sed_upper_acme ===
Welcome to ACME. This quickstart gets you running fast.
Install the CLI with npm i -g acme then run acme login.
Set your token in the ACME_TOKEN environment variable.
=== rg_l_token ===
/knowledge/guides/auth.md
/knowledge/guides/quickstart.md
=== pipe_cat_wc ===
2
=== pipe_sort_uniq_wc ===
3
=== poison_concat ===
Welcome to Acme. This quickstart gets you running fast.
Install the CLI with npm i -g acme then run acme login.
Set your token in the ACME_TOKEN environment variable.Authentication uses bearer tokens via the Authorization header.
Requests are rate limited to 100 calls per minute per token.
If you exceed the limit you receive HTTP 429 and must back off.
=== poison_first_intact ===
Welcome to Acme. This quickstart gets you running fast.
Install the CLI with npm i -g acme then run acme login.
Set your token in the ACME_TOKEN environment variable.
=== poison_second_intact ===
Authentication uses bearer tokens via the Authorization header.
Requests are rate limited to 100 calls per minute per token.
If you exceed the limit you receive HTTP 429 and must back off.
=== pipe_concat_head ===
Welcome to Acme. This quickstart gets you running fast.
=== du_guides ===
370 /knowledge/guides
=== du_root ===
370 /knowledge/guides
270 /knowledge/policies
730 /knowledge
=== du_c_multi ===
370 /knowledge/guides
270 /knowledge/policies
640 total
=== sym_ln ===
=== sym_readlink ===
/knowledge/guides/auth.md
=== sym_cat ===
Authentication uses bearer tokens via the Authorization header.
Requests are rate limited to 100 calls per minute per token.
If you exceed the limit you receive HTTP 429 and must back off.
=== sym_wc ===
2 /knowledge/meta_link
=== sym_ls ===
meta_link@
=== sym_rm ===
CHANGELOG.md
guides
policies
=== nf_cat ===
exit=1
cat: /knowledge/__nf_missing__.txt: No such file or directory
=== nf_head ===
exit=1
head: /knowledge/__nf_missing__.txt: No such file or directory
=== nf_tail ===
exit=1
tail: /knowledge/__nf_missing__.txt: No such file or directory
=== nf_wc ===
exit=1
wc: /knowledge/__nf_missing__.txt: No such file or directory
=== nf_stat ===
exit=1
stat: /knowledge/__nf_missing__.txt: No such file or directory
=== nf_grep ===
exit=1
grep: /knowledge/__nf_missing__.txt: No such file or directory
=== prov_probe_cat ===
net=190 write=0 cache=0 ops=1 hits=0 precision=exact
=== prov_probe_grep ===
net=190 write=0 cache=0 ops=1 hits=0 precision=exact
=== prov_probe_ls ===
net=0 write=0 cache=0 ops=1 hits=0 precision=exact
=== sed_stream_1p ===
Authentication uses bearer tokens via the Authorization header.
=== sed_i_readonly ===
exit=1
sed: -i not supported on this backend: Permission denied
integ/truth_lancedb.txt
-147
View File
@@ -1,147 +0,0 @@
=== ls_root ===
animals
=== ls_table ===
cat
dog
=== ls_group ===
big
small
=== find_md ===
/db/animals/cat/big/1.md
/db/animals/cat/small/2.md
/db/animals/dog/big/3.md
/db/animals/dog/small/4.md
=== cat_card ===
# a big orange cat
id: 1
label: cat
kind: big
name: a big orange cat
=== wc_c_card ===
70 /db/animals/cat/big/1.md
=== grep_cold_single ===
# a big orange cat
name: a big orange cat
=== grep_warm_single ===
# a big orange cat
name: a big orange cat
=== grep_i ===
# a big orange cat
name: a big orange cat
=== grep_n ===
4:label: cat
=== grep_v ===
id: 1
kind: big
=== grep_c ===
3
=== grep_o ===
cat
cat
cat
=== grep_w ===
# a big orange cat
label: cat
name: a big orange cat
=== grep_F_literal ===
id: 1
=== grep_m1 ===
# a big orange cat
=== grep_A1 ===
id: 1
label: cat
=== grep_B1 ===
id: 1
label: cat
=== grep_C1 ===
id: 1
label: cat
kind: big
=== grep_multi ===
/db/animals/cat/small/2.md:# a small grey cat
/db/animals/cat/small/2.md:kind: small
/db/animals/cat/small/2.md:name: a small grey cat
/db/animals/dog/small/4.md:# a small white dog
/db/animals/dog/small/4.md:kind: small
/db/animals/dog/small/4.md:name: a small white dog
=== grep_r_table ===
/db/animals/cat/big/1.md:# a big orange cat
/db/animals/cat/big/1.md:name: a big orange cat
=== grep_r_multipath ===
/db/animals/cat/small/2.md:# a small grey cat
/db/animals/cat/small/2.md:kind: small
/db/animals/cat/small/2.md:name: a small grey cat
/db/animals/dog/small/4.md:# a small white dog
/db/animals/dog/small/4.md:kind: small
/db/animals/dog/small/4.md:name: a small white dog
=== grep_rl ===
/db/animals/cat/big/1.md
/db/animals/cat/small/2.md
=== grep_E_alt ===
# a big orange cat
name: a big orange cat
=== pipe_grep_stdin ===
# a big orange cat
name: a big orange cat
=== rg_basic ===
# a big orange cat
name: a big orange cat
=== du_file ===
70 /db/animals/cat/big/1.md
=== du_group ===
70 /db/animals/cat/big
72 /db/animals/cat/small
142 /db/animals/cat
=== du_table ===
70 /db/animals/cat/big
72 /db/animals/cat/small
142 /db/animals/cat
68 /db/animals/dog/big
74 /db/animals/dog/small
142 /db/animals/dog
284 /db/animals
=== du_c_multi ===
70 /db/animals/cat/big
72 /db/animals/cat/small
142 /db/animals/cat
68 /db/animals/dog/big
74 /db/animals/dog/small
142 /db/animals/dog
284 total
=== grep_q_match ===
exit=0
=== grep_q_no_match ===
exit=1
=== grep_no_match ===
exit=1
=== nf_cat ===
exit=1
cat: /db/__nf_missing__.txt: No such file or directory
=== nf_head ===
exit=1
head: /db/__nf_missing__.txt: No such file or directory
=== nf_tail ===
exit=1
tail: /db/__nf_missing__.txt: No such file or directory
=== nf_wc ===
exit=1
wc: /db/__nf_missing__.txt: No such file or directory
=== nf_stat ===
exit=1
stat: /db/__nf_missing__.txt: No such file or directory
=== nf_grep ===
exit=1
grep: /db/__nf_missing__.txt: No such file or directory
=== prov_probe_cat ===
net=70 write=0 cache=0 ops=1 hits=0 precision=exact
=== prov_probe_grep ===
net=70 write=0 cache=0 ops=1 hits=0 precision=exact
=== prov_probe_ls ===
net=0 write=0 cache=0 ops=1 hits=0 precision=exact
=== sed_stream_1p ===
# a big orange cat
=== sed_i_readonly ===
exit=1
sed: -i not supported on this backend: Permission denied
integ/truth_mongodb.txt
-119
View File
@@ -1,119 +0,0 @@
# Expected substrings for the MongoDB integ run (integ/mongodb.py).
# Matched as fixed substrings by integ/check_lines.sh, so volatile output
# (index $indexStats timestamps, system.views internal collection, stat sizes)
# is tolerated. Seed data is fixed (integer _ids) so content is deterministic.
# ---- directory structure (ls / tree) ----
mirage_integ
collections
views
database.json
documents.jsonl
schema.json
authors
books
recent_books
# ---- database.json ----
"database": "mirage_integ"
"name": "authors", "document_count": 3
"name": "books", "document_count": 5
"views": [{"name": "recent_books"}]
# ---- schema.json sampled fields (volatile index stats ignored) ----
"name": "books", "kind": "collection"
"path": "author"
"path": "rating"
"path": "tags"
"path": "title"
"path": "year"
"primary_key": "_id"
"sampled": 100
# ---- documents (cat / head / tail / view) ----
"title": "alpha", "author": "ada"
"title": "beta", "author": "ben"
"title": "gamma", "author": "cara"
"title": "delta", "author": "ada"
"title": "epsilon", "author": "ben"
{"_id": 1, "name": "ada", "books": 2}
# ---- wc (GNU layout: right-aligned space-separated counts) ----
5 /mongodb/mirage_integ/collections/books/documents.jsonl
# default mode pads to the byte-count width; byte total differs per
# language (python renders 4.0, JS renders 4), so match the stable prefix
5 57 4
# ---- stat ----
name=documents.jsonl
# ---- grep / rg at db + root scope (search push-down, path-prefixed) ----
mirage_integ/collections/authors/documents.jsonl:
mirage_integ/collections/books/documents.jsonl:
# ---- find ----
/mongodb/mirage_integ/collections/books/documents.jsonl
/mongodb/mirage_integ/collections/books/schema.json
/mongodb/mirage_integ/views/recent_books/documents.jsonl
# ---- safeguard demo (cat cap=2 lines) ----
=== safeguard_cat_truncates ===
"title": "alpha"
"title": "beta"
output truncated at safeguard limit (2 lines)
=== safeguard_cat_pipe_uncapped ===
5 /mongodb/mirage_integ/collections/books/documents.jsonl
# ---- grep -e multi-pattern (newline-joined list, no search push-down) ----
=== grep_e_multi ===
1:{"_id": 1, "title": "alpha", "author": "ada"
2:{"_id": 2, "title": "beta", "author": "ben"
4:{"_id": 4, "title": "delta", "author": "ada"
5:{"_id": 5, "title": "epsilon", "author": "ben"
=== grep_r_e_multi ===
/mongodb/mirage_integ/collections/books/documents.jsonl:{"_id": 1, "title": "alpha"
/mongodb/mirage_integ/collections/books/documents.jsonl:{"_id": 2, "title": "beta"
=== rg_e_multi ===
/mongodb/mirage_integ/collections/books/documents.jsonl:{"_id": 3, "title": "gamma"
=== nf_cat ===
exit=1
cat: /mongodb/__nf_missing__.txt: No such file or directory
=== nf_head ===
exit=1
head: /mongodb/__nf_missing__.txt: No such file or directory
=== nf_tail ===
exit=1
tail: /mongodb/__nf_missing__.txt: No such file or directory
=== nf_wc ===
exit=1
wc: /mongodb/__nf_missing__.txt: No such file or directory
=== nf_stat ===
exit=1
stat: /mongodb/__nf_missing__.txt: No such file or directory
=== nf_grep ===
exit=1
grep: /mongodb/__nf_missing__.txt: No such file or directory
# ---- tail -f change stream (live insert via PyMongo async watch) ----
=== tail_f_change_stream ===
"title": "live_insert"
# symlink to the database meta file (namespace links; read-only backend)
=== sym_ln ===
=== sym_readlink ===
/mongodb/mirage_integ/database.json
=== sym_cat ===
=== sym_ls ===
meta_link@
=== sym_rm ===
=== prov_probe_cat ===
net=0 write=0 cache=0 ops=1 hits=0 precision=unknown
=== prov_probe_grep ===
net=0 write=0 cache=0 ops=1 hits=0 precision=unknown
=== prov_probe_ls ===
net=0 write=0 cache=0 ops=1 hits=0 precision=exact
# ---- sed (streaming works on the read-only mount; -i is rejected) ----
=== sed_stream_1p ===
=== sed_i_readonly ===
sed: -i not supported on this backend: Permission denied
integ/truth_notion.txt
-315
View File
@@ -1,315 +0,0 @@
=== ls_root ===
databases
pages
=== ls_pages ===
Notes__bbbb2222-3333-4444-5555-666677778888
Project_Roadmap__aaaa1111-2222-3333-4444-555566667777
=== ls_l_pages ===
drwxr-xr-x 1 user user 0 Jan 2 00:00 Notes__bbbb2222-3333-4444-5555-666677778888
drwxr-xr-x 1 user user 0 Jan 2 00:00 Project_Roadmap__aaaa1111-2222-3333-4444-555566667777
=== ls_page_a ===
Q1_Goals__cccc1111-2222-3333-4444-555566667777
page.json
=== stat_dir_a ===
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777 2026-01-02T00:00:00.000Z
=== cat_page_a ===
{
"page_id": "aaaa1111-2222-3333-4444-555566667777",
"title": "Project Roadmap",
"url": "https://notion.example/aaaa1111222233334444555566667777",
"created_time": "2026-01-01T00:00:00.000Z",
"last_edited_time": "2026-01-02T00:00:00.000Z",
"parent_type": "workspace",
"parent_id": "",
"archived": false,
"created_by": "user-1",
"last_edited_by": "user-2",
"markdown": "# Roadmap\n\nShip the **beta** soon\n\n- phase one\n\n - phase one detail\n\n```python\nprint(1)\n```\n",
"blocks": [
{
"object": "block",
"id": "b-a1",
"type": "heading_1",
"has_children": false,
"heading_1": {
"rich_text": [
{
"type": "text",
"plain_text": "Roadmap",
"annotations": {},
"text": {
"content": "Roadmap"
}
}
]
}
},
{
"object": "block",
"id": "b-a2",
"type": "paragraph",
"has_children": false,
"paragraph": {
"rich_text": [
{
"type": "text",
"plain_text": "Ship the ",
"annotations": {},
"text": {
"content": "Ship the "
}
},
{
"type": "text",
"plain_text": "beta",
"annotations": {
"bold": true
},
"text": {
"content": "beta"
}
},
{
"type": "text",
"plain_text": " soon",
"annotations": {},
"text": {
"content": " soon"
}
}
]
}
},
{
"object": "block",
"id": "dddd2222-3333-4444-5555-666677778888",
"type": "bulleted_list_item",
"has_children": true,
"bulleted_list_item": {
"rich_text": [
{
"type": "text",
"plain_text": "phase one",
"annotations": {},
"text": {
"content": "phase one"
}
}
]
},
"children": [
{
"object": "block",
"id": "b-d1",
"type": "bulleted_list_item",
"has_children": false,
"bulleted_list_item": {
"rich_text": [
{
"type": "text",
"plain_text": "phase one detail",
"annotations": {},
"text": {
"content": "phase one detail"
}
}
]
}
}
]
},
{
"object": "block",
"id": "b-a4",
"type": "code",
"has_children": false,
"code": {
"rich_text": [
{
"type": "text",
"plain_text": "print(1)",
"annotations": {},
"text": {
"content": "print(1)"
}
}
],
"language": "python"
}
}
]
}
=== cat_child ===
{
"page_id": "cccc1111-2222-3333-4444-555566667777",
"title": "Q1 Goals",
"url": "https://notion.example/cccc1111222233334444555566667777",
"created_time": "2026-01-01T00:00:00.000Z",
"last_edited_time": "2026-01-02T00:00:00.000Z",
"parent_type": "page_id",
"parent_id": "aaaa1111-2222-3333-4444-555566667777",
"archived": false,
"created_by": "user-1",
"last_edited_by": "user-2",
"markdown": "Q1 contents\n",
"blocks": [
{
"object": "block",
"id": "b-c1",
"type": "paragraph",
"has_children": false,
"paragraph": {
"rich_text": [
{
"type": "text",
"plain_text": "Q1 contents",
"annotations": {},
"text": {
"content": "Q1 contents"
}
}
]
}
}
]
}
=== jq_title ===
"Project Roadmap"
=== jq_markdown ===
"alpha beta gamma\n\n- [x] done item\n"
=== head_4 ===
{
"page_id": "aaaa1111-2222-3333-4444-555566667777",
"title": "Project Roadmap",
"url": "https://notion.example/aaaa1111222233334444555566667777",
=== wc_l_two ===
125 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json
51 /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json
176 total
=== stat_page_json ===
name=page.json size=2979 modified=None type=json
=== find_json ===
/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777/page.json
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json
=== find_root_maxdepth0 ===
/notion
=== find_root_name ===
/notion
=== pipe_grep ===
3
=== grep_file ===
12: "markdown": "alpha beta gamma\n\n- [x] done item\n",
23: "plain_text": "alpha beta gamma",
26: "content": "alpha beta gamma"
=== grep_multi ===
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json:0
/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json:3
=== grep_recursive ===
/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json
=== realpath_dotdot ===
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json
=== ls_databases ===
Tasks__eeee1111-2222-3333-4444-555566667777
=== ls_database_dir ===
Ship_beta__ffff2222-3333-4444-5555-666677778888
Write_spec__ffff1111-2222-3333-4444-555566667777
database.json
=== cat_database_json ===
{
"database_id": "eeee1111-2222-3333-4444-555566667777",
"title": "Tasks",
"url": "https://notion.example/eeee1111222233334444555566667777",
"created_time": "2026-01-01T00:00:00.000Z",
"last_edited_time": "2026-01-02T00:00:00.000Z",
"parent": {
"type": "workspace",
"workspace": true
},
"archived": false,
"is_inline": false,
"properties": {
"Name": {
"id": "title",
"name": "Name",
"type": "title",
"title": {}
},
"Priority": {
"id": "pri",
"name": "Priority",
"type": "number",
"number": {
"format": "number"
}
}
}
}
=== jq_db_props ===
[
"Name",
"Priority"
]
=== cat_row ===
{
"page_id": "ffff1111-2222-3333-4444-555566667777",
"title": "Write spec",
"url": "https://notion.example/ffff1111222233334444555566667777",
"created_time": "2026-01-01T00:00:00.000Z",
"last_edited_time": "2026-01-02T00:00:00.000Z",
"parent_type": "database_id",
"parent_id": "eeee1111-2222-3333-4444-555566667777",
"archived": false,
"created_by": "user-1",
"last_edited_by": "user-2",
"markdown": "",
"blocks": []
}
=== du_pages ===
1211 /notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888
826 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777
3805 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777
5016 /notion/pages
=== du_page_a ===
826 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777
3805 /notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777
=== grep_c_match_exit ===
exit=0
3
=== grep_c_no_match_exit ===
exit=1
0
=== grep_rc_no_match_exit ===
exit=1
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/page.json:0
/notion/pages/Project_Roadmap__aaaa1111-2222-3333-4444-555566667777/Q1_Goals__cccc1111-2222-3333-4444-555566667777/page.json:0
/notion/pages/Notes__bbbb2222-3333-4444-5555-666677778888/page.json:0
=== nf_cat ===
exit=1
cat: /notion/__nf_missing__.txt: No such file or directory
=== nf_head ===
exit=1
head: /notion/__nf_missing__.txt: No such file or directory
=== nf_tail ===
exit=1
tail: /notion/__nf_missing__.txt: No such file or directory
=== nf_wc ===
exit=1
wc: /notion/__nf_missing__.txt: No such file or directory
=== nf_stat ===
exit=1
stat: /notion/__nf_missing__.txt: No such file or directory
=== nf_grep ===
exit=1
grep: /notion/__nf_missing__.txt: No such file or directory
=== prov_probe_cat ===
net=0 write=0 cache=0 ops=1 hits=1 precision=unknown
=== prov_probe_grep ===
net=0 write=0 cache=0 ops=1 hits=1 precision=unknown
=== prov_probe_ls ===
net=0 write=0 cache=0 ops=1 hits=0 precision=exact
=== sed_stream_1p ===
{
=== sed_i_readonly ===
exit=1
sed: -i not supported on this backend: Permission denied
integ/truth_postgres.txt
-128
View File
@@ -1,128 +0,0 @@
# Expected substrings for the Postgres integ run (integ/postgres.py and
# integ/postgres.ts). Matched as fixed substrings by integ/check_lines.sh, so
# volatile output (row/size estimates, stat sizes) is tolerated. Seed data is
# fixed (integer ids) so content is deterministic. Whole-number rating values
# render as N.0 in Python (orjson) but N in TS (JS has no float type), so the
# row lines below assert everything up to (not including) the rating column.
# ---- directory structure (ls / tree) ----
database.json
public
tables
views
rows.jsonl
schema.json
authors
books
recent_books
# ---- schema.json (per-table synthetic schema, pretty-printed) ----
"schema": "public"
"name": "books"
"kind": "table"
"name": "id"
"name": "title"
"name": "author"
"name": "year"
"name": "rating"
"type": "integer"
"type": "text"
"type": "double precision"
"primary_key": [
"name": "books_pkey"
"unique": true
# ---- rows (cat / head / tail / view); rating column omitted, see header ----
{"id":1,"title":"alpha","author":"ada","year":2020
{"id":2,"title":"beta","author":"ben","year":2021
{"id":3,"title":"gamma","author":"cara","year":2022
{"id":4,"title":"delta","author":"ada","year":2023
{"id":5,"title":"epsilon","author":"ben","year":2024
# ---- wc (warm via cache -> count<TAB>path; cold pushdown also labeled) ----
5 /pg/public/tables/books/rows.jsonl
# default mode pads to the byte-count width; byte total differs per
# language (float rendering), so match the stable prefix
5 5 32
3 /pg/public/tables/authors/rows.jsonl
3 /pg/public/views/recent_books/rows.jsonl
# ---- stat ----
name=rows.jsonl
# ---- grep / rg at table + schema scope (ILIKE push-down, path-prefixed) ----
public/tables/books/rows.jsonl:{"id":1,"title":"alpha","author":"ada"
public/tables/books/rows.jsonl:{"id":4,"title":"delta","author":"ada"
# ---- find ----
/pg/public/tables/books/rows.jsonl
/pg/public/tables/books/schema.json
/pg/public/views/recent_books/rows.jsonl
# ---- safeguard demo (cat cap=2 lines) ----
=== safeguard_cat_truncates ===
{"id":1,"title":"alpha","author":"ada","year":2020
{"id":2,"title":"beta","author":"ben","year":2021
output truncated at safeguard limit (2 lines)
=== safeguard_cat_pipe_uncapped ===
5 /pg/public/tables/books/rows.jsonl
# ---- grep -e multi-pattern (newline-joined list, no ILIKE push-down) ----
=== grep_e_multi ===
1:{"id":1,"title":"alpha","author":"ada","year":2020
2:{"id":2,"title":"beta","author":"ben","year":2021
4:{"id":4,"title":"delta","author":"ada","year":2023
5:{"id":5,"title":"epsilon","author":"ben","year":2024
=== rg_e_multi ===
1:{"id":1,"title":"alpha","author":"ada","year":2020
2:{"id":2,"title":"beta","author":"ben","year":2021
4:{"id":4,"title":"delta","author":"ada","year":2023
5:{"id":5,"title":"epsilon","author":"ben","year":2024
=== nf_cat ===
exit=1
cat: /pg/__nf_missing__.txt: No such file or directory
=== nf_head ===
exit=1
head: /pg/__nf_missing__.txt: No such file or directory
=== nf_tail ===
exit=1
tail: /pg/__nf_missing__.txt: No such file or directory
=== nf_wc ===
exit=1
wc: /pg/__nf_missing__.txt: No such file or directory
=== nf_stat ===
exit=1
stat: /pg/__nf_missing__.txt: No such file or directory
=== nf_grep ===
exit=1
grep: /pg/__nf_missing__.txt: No such file or directory
# symlink into the mount (namespace links; read-only backend)
=== sym_ln ===
=== sym_readlink ===
/pg/public/tables/books/schema.json
=== sym_cat ===
=== sym_ls ===
meta_link@
=== sym_rm ===
# ---- sizeless rows.jsonl (storage size lives in extra.size_bytes) ----
=== stat_size_rows ===
0 /pg/public/tables/books/rows.jsonl
=== find_size_plus_rows ===
=== find_size_under_rows ===
/pg/public/tables/books/rows.jsonl
# rows.jsonl is sizeless (storage size lives in extra.size_bytes), so read
# provisions are honestly unknown instead of quoting the 49x-off table size.
=== prov_probe_cat ===
net=0 write=0 cache=0 ops=1 hits=0 precision=unknown
=== prov_probe_grep ===
net=0 write=0 cache=0 ops=1 hits=0 precision=unknown
=== prov_probe_ls ===
net=0 write=0 cache=0 ops=1 hits=0 precision=exact
# ---- sed (streaming works on the read-only mount; -i is rejected) ----
=== sed_stream_1p ===
{"id":1,"title":"alpha","author":"ada","year":2020,
=== sed_i_readonly ===
sed: -i not supported on this backend: Permission denied
integ/truth_qdrant.txt
-131
View File
@@ -1,131 +0,0 @@
=== ls_root ===
cat
dog
=== ls_group ===
big
small
=== ls_leaf ===
1.json
1.txt
=== find_txt ===
/db/cat/big/1.txt
/db/cat/small/2.txt
/db/dog/big/3.txt
/db/dog/small/4.txt
=== find_json ===
/db/cat/big/1.json
/db/cat/small/2.json
/db/dog/big/3.json
/db/dog/small/4.json
=== cat_txt ===
a big orange cat
=== cat_json ===
{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== wc_c_txt ===
17 /db/cat/big/1.txt
=== grep_text ===
a big orange cat
=== grep_json_field ===
{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== grep_i ===
a big orange cat
=== grep_n ===
1:{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== grep_c ===
1
=== grep_o ===
cat
cat
=== grep_w ===
{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== grep_F_literal ===
a big orange cat
=== grep_E_alt ===
a big orange cat
=== grep_v ===
a big orange cat
=== grep_multi ===
/db/cat/small/2.json:{"label":"cat","kind":"small","name":"a small grey cat","id":2}
/db/dog/small/4.json:{"label":"dog","kind":"small","name":"a small white dog","id":4}
=== grep_r_group ===
/db/cat/big/1.json:{"label":"cat","kind":"big","name":"a big orange cat","id":1}
/db/cat/big/1.txt:a big orange cat
=== grep_rl ===
/db/cat/big/1.json
/db/cat/big/1.txt
/db/cat/small/2.json
/db/cat/small/2.txt
=== pipe_grep_stdin ===
{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== rg_basic ===
a big orange cat
=== du_leaf ===
79 /db/cat/big
=== du_group ===
79 /db/cat/big
81 /db/cat/small
160 /db/cat
=== du_root ===
79 /db/cat/big
81 /db/cat/small
160 /db/cat
77 /db/dog/big
83 /db/dog/small
160 /db/dog
320 /db
=== du_c_multi ===
79 /db/cat/big
81 /db/cat/small
160 /db/cat
77 /db/dog/big
83 /db/dog/small
160 /db/dog
320 total
=== sym_ln ===
=== sym_readlink ===
/db/cat/big/1.json
=== sym_cat ===
{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== sym_grep ===
{"label":"cat","kind":"big","name":"a big orange cat","id":1}
=== sym_ls ===
meta_link@
=== sym_rm ===
cat
dog
=== grep_q_match ===
exit=0
=== grep_q_no_match ===
exit=1
=== grep_no_match ===
exit=1
=== nf_cat ===
exit=1
cat: /db/cat/big/__nf_missing__.json: No such file or directory
=== nf_head ===
exit=1
head: /db/cat/big/__nf_missing__.json: No such file or directory
=== nf_tail ===
exit=1
tail: /db/cat/big/__nf_missing__.json: No such file or directory
=== nf_wc ===
exit=1
wc: /db/cat/big/__nf_missing__.json: No such file or directory
=== nf_stat ===
exit=1
stat: /db/cat/big/__nf_missing__.json: No such file or directory
=== nf_grep ===
exit=1
grep: /db/cat/big/__nf_missing__.json: No such file or directory
=== prov_probe_cat ===
net=17 write=0 cache=0 ops=1 hits=0 precision=exact
=== prov_probe_grep ===
net=17 write=0 cache=0 ops=1 hits=0 precision=exact
=== prov_probe_ls ===
net=0 write=0 cache=0 ops=1 hits=0 precision=exact
=== sed_stream_1p ===
a big orange cat
=== sed_i_readonly ===
exit=1
sed: -i not supported on this backend: Permission denied
python/mirage/commands/builtin/grep_helper.py
+51 -1
View File
@@ -13,7 +13,8 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import re
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
from collections.abc import (AsyncIterator, Awaitable, Callable, Mapping,
Sequence)
from mirage.commands.builtin.constants import PatternType
from mirage.commands.builtin.grep_context import grep_context_lines
@@ -161,6 +162,55 @@ def search_query(pattern: str, fixed_string: bool) -> str | None:
return extract_required_literal(pattern)
_PUSHDOWN_SHAPING_BOOL = ("v", "n", "c", "args_l", "w", "o", "q", "H", "h",
"args_I")
_PUSHDOWN_SHAPING_INT = ("m", "A", "B", "C")
_PUSHDOWN_FILTER_STR = ("type", "glob")
def has_search_shaping_flags(flags: Mapping[str, object] | None) -> bool:
"""True when a flag alters the match set or output shape of grep/rg.
A search push-down prints each matching record as one whole line, so it
cannot honor -v/-n/-c/-l/-w/-o/-m/-A/-B/-C/-q/-H/-h, rg's -I (no filename),
nor rg's file-filtering --glob/--type; when any is present the wrapper must
defer to the generic scan, which applies exact semantics. Reads through a
spec-less FlagView so the shared key set works for both the grep and rg
specs (rg simply never sets the grep-only keys).
Args:
flags (Mapping[str, object] | None): raw flag kwargs.
"""
fl = FlagView(flags)
if any(fl.as_bool(k) for k in _PUSHDOWN_SHAPING_BOOL):
return True
if any(fl.as_int(k) is not None for k in _PUSHDOWN_SHAPING_INT):
return True
return any(fl.as_str(k) is not None for k in _PUSHDOWN_FILTER_STR)
def search_pushdown_ok(flags: Mapping[str, object] | None,
pattern: str) -> bool:
"""True when a literal-substring push-down faithfully reproduces grep/rg.
For the LIKE/ILIKE substring push-down (postgres/mysql), faithful means a
literal pattern with no shaping flags; a real regex is treated literally
by LIKE and so must take the generic scan, and a newline-joined pattern
list (-F with multiple -e) is a set of independent alternatives that LIKE
cannot express. Backends that push a real regex down (mongodb) gate on
has_search_shaping_flags alone instead.
Args:
flags (Mapping[str, object] | None): raw flag kwargs.
pattern (str): the resolved search pattern.
"""
if "\n" in pattern:
return False
fl = FlagView(flags)
return (is_literal_pattern(pattern, fl.as_bool("F"))
and not has_search_shaping_flags(flags))
def pattern_arg(texts: Sequence[str], flags: FlagView) -> str | None:
"""Resolve the pattern-list argument from -e values or the positional.
python/mirage/commands/builtin/mongodb/grep.py
+7 -2
View File
@@ -18,10 +18,12 @@ from mirage.accessor.mongodb import MongoDBAccessor
from mirage.cache.index import IndexCacheStore
from mirage.commands.builtin.generic.grep import grep as generic_grep
from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.grep_helper import pattern_arg
from mirage.commands.builtin.grep_helper import (has_search_shaping_flags,
pattern_arg)
from mirage.commands.builtin.mongodb._provision import search_provision
from mirage.commands.builtin.mongodb.io import resolve_glob
from mirage.commands.builtin.utils.output import format_records
from mirage.commands.builtin.utils.paths import has_unresolved_glob
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagView
@@ -61,7 +63,10 @@ async def grep(
config = accessor.config
limit = config.default_search_limit
if paths and pattern is not None and "\n" not in pattern:
# The $regex push-down prints each matching document as a whole line, so
# output/match-shaping flags must defer to the generic scan below.
if (paths and not has_unresolved_glob(paths) and pattern is not None
and "\n" not in pattern and not has_search_shaping_flags(flags)):
scope = detect_scope(paths[0])
if isinstance(scope, SEARCHABLE_SCOPE_TYPES):
python/mirage/commands/builtin/mongodb/rg.py
+7 -2
View File
@@ -16,9 +16,11 @@ from mirage.accessor.mongodb import MongoDBAccessor
from mirage.cache.index import IndexCacheStore
from mirage.commands.builtin.generic.rg import rg as generic_rg
from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.grep_helper import pattern_arg
from mirage.commands.builtin.grep_helper import (has_search_shaping_flags,
pattern_arg)
from mirage.commands.builtin.mongodb.io import resolve_glob
from mirage.commands.builtin.utils.output import format_records
from mirage.commands.builtin.utils.paths import has_unresolved_glob
from mirage.commands.errors import UsageError
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
@@ -57,7 +59,10 @@ async def rg(
config = accessor.config
limit = config.default_search_limit
if paths and "\n" not in pattern_str:
# The $regex push-down prints each matching document as a whole line, so
# output/match-shaping flags must defer to the generic scan below.
if (paths and not has_unresolved_glob(paths) and "\n" not in pattern_str
and not has_search_shaping_flags(flags)):
scope = detect_scope(paths[0])
if isinstance(scope, SEARCHABLE_SCOPE_TYPES):
python/mirage/commands/builtin/postgres/grep.py
+62 -13
View File
@@ -16,10 +16,11 @@ from mirage.accessor.postgres import PostgresAccessor
from mirage.cache.index import IndexCacheStore
from mirage.commands.builtin.generic.grep import grep as generic_grep
from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.grep_helper import pattern_arg
from mirage.commands.builtin.grep_helper import pattern_arg, search_pushdown_ok
from mirage.commands.builtin.postgres._provision import search_provision
from mirage.commands.builtin.postgres.io import resolve_glob
from mirage.commands.builtin.utils.output import format_records
from mirage.commands.builtin.utils.paths import has_unresolved_glob
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
from mirage.commands.spec.types import FlagView
@@ -27,8 +28,10 @@ from mirage.core.postgres.read import read as postgres_read
from mirage.core.postgres.readdir import readdir as _readdir
from mirage.core.postgres.scope import detect_scope
from mirage.core.postgres.search import (format_grep_results, search_database,
search_entity, search_kind,
search_schema)
search_database_metadata,
search_entity, search_entity_metadata,
search_kind, search_kind_metadata,
search_schema, search_schema_metadata)
from mirage.core.postgres.stat import stat as _stat
from mirage.io.types import ByteSource, IOResult
from mirage.types import PathSpec
@@ -49,45 +52,91 @@ async def grep(
) -> tuple[ByteSource | None, IOResult]:
fl = FlagView(flags, spec=SPECS["grep"])
pattern = pattern_arg(texts, fl)
ci = fl.as_bool("i")
limit = accessor.config.default_search_limit
if paths and pattern is not None and "\n" not in pattern:
# The push-down is a literal-substring search (case-sensitive unless -i)
# that prints each matching row as a whole line; it cannot honor
# output/match-shaping flags or a real regex, so those defer to the
# generic scan below.
if (paths and not has_unresolved_glob(paths) and pattern is not None
and search_pushdown_ok(flags, pattern)):
scope = detect_scope(paths[0])
if scope.level != "root":
await _stat(accessor, paths[0], index=index)
# Directory scopes cover every file under them, so the rendered
# schema.json / semantic.json are searched alongside the row
# push-down. Deliberate divergence from GNU: rows come first and
# metadata second, rather than in per-entity readdir order.
if scope.level == "root":
results = await search_database(accessor, pattern, limit)
results = await search_database(accessor,
pattern,
limit,
case_insensitive=ci)
all_lines = format_grep_results(results)
all_lines += await search_database_metadata(accessor,
pattern,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
if scope.level == "schema":
results = await search_schema(accessor, scope.schema, pattern,
limit)
results = await search_schema(accessor,
scope.schema,
pattern,
limit,
case_insensitive=ci)
all_lines = format_grep_results(results)
all_lines += await search_schema_metadata(accessor,
scope.schema,
pattern,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
if scope.level == "kind":
results = await search_kind(accessor, scope.schema, scope.kind,
pattern, limit)
results = await search_kind(accessor,
scope.schema,
scope.kind,
pattern,
limit,
case_insensitive=ci)
all_lines = format_grep_results(results)
all_lines += await search_kind_metadata(accessor,
scope.schema,
scope.kind,
pattern,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
if scope.level in ("entity", "entity_rows"):
rows = await search_entity(accessor, scope.schema, scope.kind,
scope.entity, pattern, limit)
if not rows:
return b"", IOResult(exit_code=1)
rows = await search_entity(accessor,
scope.schema,
scope.kind,
scope.entity,
pattern,
limit,
case_insensitive=ci)
results = [(scope.schema, scope.kind, scope.entity, rows)]
all_lines = format_grep_results(results)
# entity_rows names rows.jsonl explicitly; only the directory
# scope pulls in the sibling metadata files.
if scope.level == "entity":
all_lines += await search_entity_metadata(accessor,
scope.schema,
scope.kind,
scope.entity,
pattern,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
resolved = await resolve_glob(accessor, paths,
python/mirage/commands/builtin/postgres/rg.py
+61 -15
View File
@@ -16,9 +16,10 @@ from mirage.accessor.postgres import PostgresAccessor
from mirage.cache.index import IndexCacheStore
from mirage.commands.builtin.generic.rg import rg as generic_rg
from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.grep_helper import pattern_arg
from mirage.commands.builtin.grep_helper import pattern_arg, search_pushdown_ok
from mirage.commands.builtin.postgres.io import resolve_glob
from mirage.commands.builtin.utils.output import format_records
from mirage.commands.builtin.utils.paths import has_unresolved_glob
from mirage.commands.errors import UsageError
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
@@ -27,8 +28,10 @@ from mirage.core.postgres.read import read as postgres_read
from mirage.core.postgres.readdir import readdir as _readdir
from mirage.core.postgres.scope import detect_scope
from mirage.core.postgres.search import (format_grep_results, search_database,
search_entity, search_kind,
search_schema)
search_database_metadata,
search_entity, search_entity_metadata,
search_kind, search_kind_metadata,
search_schema, search_schema_metadata)
from mirage.core.postgres.stat import stat as _stat
from mirage.io.types import ByteSource, IOResult
from mirage.types import PathSpec
@@ -51,42 +54,85 @@ async def rg(
config = accessor.config
limit = config.default_search_limit
ci = fl.as_bool("i")
# Native search takes one pattern; a newline-joined multi -e set
# must fall through to the generic so each pattern matches (#347).
if paths and "\n" not in pattern_str:
# Native search takes one literal pattern and prints each matching row as
# a whole line; a multi -e set (#347), a real regex, or any match/output
# shaping flag must fall through to the generic scan below.
if (paths and not has_unresolved_glob(paths)
and search_pushdown_ok(flags, pattern_str)):
scope = detect_scope(paths[0])
# Directory scopes cover every file under them, so the rendered
# schema.json / semantic.json are searched alongside the row
# push-down. Deliberate divergence from GNU: rows come first and
# metadata second, rather than in per-entity readdir order.
if scope.level == "root":
results = await search_database(accessor, pattern_str, limit)
results = await search_database(accessor,
pattern_str,
limit,
case_insensitive=ci)
all_lines = format_grep_results(results)
all_lines += await search_database_metadata(accessor,
pattern_str,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
if scope.level == "schema":
results = await search_schema(accessor, scope.schema, pattern_str,
limit)
results = await search_schema(accessor,
scope.schema,
pattern_str,
limit,
case_insensitive=ci)
all_lines = format_grep_results(results)
all_lines += await search_schema_metadata(accessor,
scope.schema,
pattern_str,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
if scope.level == "kind":
results = await search_kind(accessor, scope.schema, scope.kind,
pattern_str, limit)
results = await search_kind(accessor,
scope.schema,
scope.kind,
pattern_str,
limit,
case_insensitive=ci)
all_lines = format_grep_results(results)
all_lines += await search_kind_metadata(accessor,
scope.schema,
scope.kind,
pattern_str,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
if scope.level in ("entity", "entity_rows"):
rows = await search_entity(accessor, scope.schema, scope.kind,
scope.entity, pattern_str, limit)
if not rows:
return b"", IOResult(exit_code=1)
rows = await search_entity(accessor,
scope.schema,
scope.kind,
scope.entity,
pattern_str,
limit,
case_insensitive=ci)
results = [(scope.schema, scope.kind, scope.entity, rows)]
all_lines = format_grep_results(results)
# entity_rows names rows.jsonl explicitly; only the directory
# scope pulls in the sibling metadata files.
if scope.level == "entity":
all_lines += await search_entity_metadata(accessor,
scope.schema,
scope.kind,
scope.entity,
pattern_str,
case_insensitive=ci)
if not all_lines:
return b"", IOResult(exit_code=1)
return format_records(all_lines), IOResult()
resolved = await resolve_glob(accessor, paths,
python/mirage/commands/builtin/postgres/tail.py
+3 -1
View File
@@ -22,6 +22,7 @@ from mirage.commands.builtin.generic_bind.adapter import bound_op
from mirage.commands.builtin.postgres._provision import head_tail_provision
from mirage.commands.builtin.postgres.io import resolve_glob
from mirage.commands.builtin.tail_helper import _parse_n
from mirage.commands.builtin.utils.paths import has_unresolved_glob
from mirage.commands.builtin.utils.stream import _resolve_source
from mirage.commands.registry import command
from mirage.commands.spec import SPECS
@@ -59,7 +60,8 @@ async def tail(
c_int = int(c) if c is not None else None
if paths:
scope = detect_scope(paths[0])
if (len(paths) == 1 and isinstance(scope, PostgresEntityRowsScope)
if (len(paths) == 1 and not has_unresolved_glob(paths)
and isinstance(scope, PostgresEntityRowsScope)
and c_int is None and n_int is not None):
limit = min(n_int, accessor.config.default_row_limit)
pool = await accessor.pool()
python/mirage/commands/builtin/utils/paths.py
+14
View File
@@ -16,6 +16,20 @@ from mirage.types import PathSpec
from mirage.utils.path import resolve_path
def has_unresolved_glob(paths: list[PathSpec]) -> bool:
"""True when any operand still carries a glob to expand.
Backend push-down branches read ``paths[0]`` directly to build SQL, so
they must not run before glob expansion: a pattern segment would be
taken for a literal entity name, and ``tables/*/rows.jsonl`` would
query a relation actually called ``*``.
Args:
paths (list[PathSpec]): operands as parsed.
"""
return any(p.pattern for p in paths)
def resolve_script(name: str, cwd: PathSpec | None) -> PathSpec:
"""Resolve a script operand to a fully-resolved PathSpec.
python/mirage/core/mongodb/_sampler.py
+12 -2
View File
@@ -13,6 +13,7 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import datetime as dt
import math
from collections.abc import Iterable
from typing import Any
@@ -34,7 +35,13 @@ def _scalar_tag(v) -> str:
if isinstance(v, int):
return BsonTypeTag.INT
if isinstance(v, float):
return BsonTypeTag.DOUBLE
# MongoDB is schemaless, so a field's type is inferred from values.
# The JS driver returns BSON double as a plain JS number, so it cannot
# tell a whole-valued double from an int (Number.isInteger). Mirror
# that here so py and ts classify identically (a whole-valued double
# is typed int, matching how it renders as a bare integer).
return (BsonTypeTag.INT
if math.isfinite(v) and v.is_integer() else BsonTypeTag.DOUBLE)
if isinstance(v, str):
return BsonTypeTag.STRING
if isinstance(v, ObjectId):
@@ -85,7 +92,10 @@ async def sample_field_types(col,
sample_size: int = 100) -> list[dict[str, Any]]:
counts: dict[str, dict[str, int]] = {}
total = 0
async for doc in await col.aggregate([{"$sample": {"size": sample_size}}]):
# Read the first sample_size docs sorted by _id (deterministic), not
# $sample (random): schema.json must be reproducible across reads, and
# this mirrors the TS sampler so the two languages infer identically.
async for doc in col.find({}, sort=[(PRIMARY_KEY, 1)], limit=sample_size):
total += 1
_walk(doc, "", counts)
if total == 0:
python/mirage/core/mongodb/read.py
+3 -7
View File
@@ -12,15 +12,13 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from bson.json_util import RELAXED_JSON_OPTIONS, dumps
from mirage.accessor.mongodb import MongoDBAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore
from mirage.core.mongodb._client import database_exists, entity_exists
from mirage.core.mongodb._schema_json import (build_collection_schema_json,
build_database_json)
from mirage.core.mongodb.scope import detect_scope
from mirage.core.mongodb.stream import read_stream
from mirage.core.mongodb.stream import read_stream, render_doc
from mirage.core.mongodb.types import ScopeLevel
from mirage.types import PathSpec
from mirage.utils.errors import enoent
@@ -48,13 +46,11 @@ async def read(
raise enoent(path)
payload = await build_collection_schema_json(accessor, scope.database,
scope.name)
return (dumps(payload, json_options=RELAXED_JSON_OPTIONS) +
"\n").encode()
return (render_doc(payload) + "\n").encode()
if scope.level == ScopeLevel.DATABASE_JSON:
if not await database_exists(accessor.client, accessor.config,
scope.database, accessor):
raise enoent(path)
payload = await build_database_json(accessor, scope.database)
return (dumps(payload, json_options=RELAXED_JSON_OPTIONS) +
"\n").encode()
return (render_doc(payload) + "\n").encode()
raise enoent(path)
python/mirage/core/mongodb/search.py
+2 -2
View File
@@ -15,10 +15,10 @@
import asyncio
from typing import Any
from bson.json_util import RELAXED_JSON_OPTIONS, dumps
from pymongo import AsyncMongoClient
from mirage.core.mongodb._client import list_collections
from mirage.core.mongodb.stream import render_doc
from mirage.core.mongodb.types import PRIMARY_KEY, EntityKind
@@ -93,6 +93,6 @@ def format_grep_results(
for db_name, col_name, docs in results:
path = f"{db_name}/collections/{col_name}/documents.jsonl"
for doc in docs:
line_json = dumps(doc, json_options=RELAXED_JSON_OPTIONS)
line_json = render_doc(doc)
lines.append(f"{path}:{line_json}")
return lines
python/mirage/core/mongodb/stream.py
+8 -3
View File
@@ -25,6 +25,11 @@ from mirage.core.mongodb.scope import detect_scope
from mirage.core.mongodb.types import PRIMARY_KEY, ScopeLevel
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.json_canonical import canonicalize_value
def render_doc(doc: dict[str, Any]) -> str:
return dumps(canonicalize_value(doc), json_options=RELAXED_JSON_OPTIONS)
def _apply_elision(value: dict[str, Any], paths: set[str]) -> dict[str, Any]:
@@ -88,7 +93,7 @@ async def read_tail(
for doc in docs:
if elide:
doc = _apply_elision(doc, elide)
lines.append(dumps(doc, json_options=RELAXED_JSON_OPTIONS))
lines.append(render_doc(doc))
return ("\n".join(lines) + "\n").encode()
@@ -111,7 +116,7 @@ async def read_stream(
):
if elide:
doc = _apply_elision(doc, elide)
yield (dumps(doc, json_options=RELAXED_JSON_OPTIONS) + "\n").encode()
yield (render_doc(doc) + "\n").encode()
async def watch_stream(
@@ -126,4 +131,4 @@ async def watch_stream(
async for doc in iter_inserts(accessor.client, scope.database, scope.name):
if elide:
doc = _apply_elision(doc, elide)
yield (dumps(doc, json_options=RELAXED_JSON_OPTIONS) + "\n").encode()
yield (render_doc(doc) + "\n").encode()
python/mirage/core/postgres/_client.py
+80 -4
View File
@@ -17,6 +17,18 @@ from typing import Any
import asyncpg
from mirage.utils.json_canonical import canonicalize_row, canonicalize_value
__all__ = ["canonicalize_row", "canonicalize_value"]
def quote_ident(ident: str) -> str:
return '"' + ident.replace('"', '""') + '"'
def qualified(schema: str, name: str) -> str:
return f"{quote_ident(schema)}.{quote_ident(name)}"
async def list_schemas(conn: asyncpg.Connection,
allowlist: list[str] | None) -> list[str]:
@@ -56,13 +68,14 @@ async def list_matviews(conn: asyncpg.Connection, schema: str) -> list[str]:
async def count_rows(conn: asyncpg.Connection, schema: str, name: str) -> int:
return await conn.fetchval(f'SELECT COUNT(*) FROM "{schema}"."{name}"')
return await conn.fetchval(
f"SELECT COUNT(*) FROM {qualified(schema, name)}")
async def estimate_size(conn: asyncpg.Connection, schema: str,
name: str) -> tuple[int, int]:
plan = await conn.fetchval(
f'EXPLAIN (FORMAT JSON) SELECT * FROM "{schema}"."{name}"')
f"EXPLAIN (FORMAT JSON) SELECT * FROM {qualified(schema, name)}")
if isinstance(plan, str):
plan = json.loads(plan)
top = plan[0]["Plan"]
@@ -90,8 +103,9 @@ async def table_size_bytes(conn: asyncpg.Connection, schema: str,
async def fetch_rows(conn: asyncpg.Connection, schema: str, name: str, *,
limit: int, offset: int) -> list[dict[str, Any]]:
rows = await conn.fetch(
f'SELECT * FROM "{schema}"."{name}" LIMIT $1 OFFSET $2', limit, offset)
return [dict(r) for r in rows]
f"SELECT * FROM {qualified(schema, name)} LIMIT $1 OFFSET $2", limit,
offset)
return [canonicalize_row(dict(r)) for r in rows]
async def fetch_columns(conn: asyncpg.Connection, schema: str,
@@ -108,6 +122,68 @@ async def fetch_columns(conn: asyncpg.Connection, schema: str,
} for r in rows]
async def fetch_table_comment(conn: asyncpg.Connection, schema: str,
name: str) -> str | None:
return await conn.fetchval(
"SELECT obj_description(c.oid, 'pg_class') "
"FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
"WHERE n.nspname = $1 AND c.relname = $2", schema, name)
async def fetch_column_comments(conn: asyncpg.Connection, schema: str,
name: str) -> dict[str, str]:
rows = await conn.fetch(
"SELECT a.attname, col_description(c.oid, a.attnum) AS comment "
"FROM pg_class c "
"JOIN pg_namespace n ON n.oid = c.relnamespace "
"JOIN pg_attribute a "
" ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped "
"WHERE n.nspname = $1 AND c.relname = $2 "
"ORDER BY a.attnum", schema, name)
return {r["attname"]: r["comment"] for r in rows if r["comment"]}
async def fetch_enum_columns(conn: asyncpg.Connection, schema: str,
name: str) -> dict[str, dict[str, Any]]:
rows = await conn.fetch(
"SELECT a.attname, t.typname, "
" array_agg(e.enumlabel ORDER BY e.enumsortorder)::text[] "
"AS labels "
"FROM pg_class c "
"JOIN pg_namespace n ON n.oid = c.relnamespace "
"JOIN pg_attribute a "
" ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped "
"JOIN pg_type t ON t.oid = a.atttypid "
"JOIN pg_enum e ON e.enumtypid = t.oid "
"WHERE n.nspname = $1 AND c.relname = $2 "
"GROUP BY a.attname, t.typname", schema, name)
return {
r["attname"]: {
"type": r["typname"],
"labels": list(r["labels"]),
}
for r in rows
}
async def fetch_column_stats(conn: asyncpg.Connection, schema: str,
name: str) -> dict[str, dict[str, Any]]:
# pg_stats is populated by ANALYZE, so it is empty for a freshly written
# relation until autovacuum gets to it. Callers treat it as best-effort;
# mirage never runs ANALYZE itself (a write and a cost on the user's DB).
rows = await conn.fetch(
"SELECT attname, n_distinct, "
" most_common_vals::text::text[] AS mcv "
"FROM pg_stats WHERE schemaname = $1 AND tablename = $2", schema, name)
return {
r["attname"]: {
"n_distinct": float(r["n_distinct"]),
"most_common_vals": list(r["mcv"]) if r["mcv"] else [],
}
for r in rows
}
async def fetch_primary_key(conn: asyncpg.Connection, schema: str,
name: str) -> list[str]:
rows = await conn.fetch(
python/mirage/core/postgres/read.py
+7
View File
@@ -20,6 +20,7 @@ from mirage.core.postgres import _client
from mirage.core.postgres._schema_json import (build_database_json,
build_entity_schema_json)
from mirage.core.postgres.scope import detect_scope
from mirage.core.postgres.semantic import build_entity_semantic_json
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
@@ -52,6 +53,12 @@ async def read(
scope.entity, kind)
return orjson.dumps(doc, option=orjson.OPT_INDENT_2)
if scope.level == "entity_semantic":
kind = "table" if scope.kind == "tables" else "view"
doc = await build_entity_semantic_json(accessor, scope.schema,
scope.entity, kind)
return orjson.dumps(doc, option=orjson.OPT_INDENT_2)
if scope.level == "entity_rows":
return await _read_rows(accessor,
scope.schema,
python/mirage/core/postgres/readdir.py
+2 -5
View File
@@ -15,7 +15,7 @@
from mirage.accessor.postgres import PostgresAccessor
from mirage.cache.index import NULL_INDEX, IndexCacheStore, IndexEntry
from mirage.core.postgres import _client
from mirage.core.postgres.scope import detect_scope
from mirage.core.postgres.scope import ENTITY_FILES, detect_scope
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of
@@ -53,10 +53,7 @@ async def readdir(accessor: PostgresAccessor,
virtual_key, index, prefix, raw)
if scope.level == "entity":
base = raw.rstrip("/")
return [
f"{prefix}{base}/schema.json",
f"{prefix}{base}/rows.jsonl",
]
return [f"{prefix}{base}/{name}" for name in ENTITY_FILES]
raise enoent(path)
python/mirage/core/postgres/scope.py
+21 -2
View File
@@ -69,6 +69,17 @@ class PostgresEntitySchemaScope:
init=False)
@dataclass(frozen=True)
class PostgresEntitySemanticScope:
schema: str
kind: PostgresKind
entity: str
resource_path: str
file: Literal["semantic.json"] = field(default="semantic.json", init=False)
level: Literal["entity_semantic"] = field(default="entity_semantic",
init=False)
@dataclass(frozen=True)
class PostgresEntityRowsScope:
schema: str
@@ -88,8 +99,11 @@ class PostgresInvalidScope:
PostgresScope: TypeAlias = (PostgresRootScope | PostgresDatabaseJSONScope
| PostgresSchemaScope | PostgresKindScope
| PostgresEntityScope | PostgresEntitySchemaScope
| PostgresEntitySemanticScope
| PostgresEntityRowsScope | PostgresInvalidScope)
ENTITY_FILES = ("schema.json", "semantic.json", "rows.jsonl")
def detect_scope(path: PathSpec) -> PostgresScope:
raw = path.mount_path
@@ -117,14 +131,19 @@ def detect_scope(path: PathSpec) -> PostgresScope:
entity=parts[2],
resource_path=raw)
if len(parts) == 4 and parts[1] in ("tables", "views") and parts[3] in (
"schema.json", "rows.jsonl"):
if len(parts) == 4 and parts[1] in ("tables",
"views") and parts[3] in ENTITY_FILES:
kind = "tables" if parts[1] == "tables" else "views"
if parts[3] == "schema.json":
return PostgresEntitySchemaScope(schema=parts[0],
kind=kind,
entity=parts[2],
resource_path=raw)
if parts[3] == "semantic.json":
return PostgresEntitySemanticScope(schema=parts[0],
kind=kind,
entity=parts[2],
resource_path=raw)
return PostgresEntityRowsScope(schema=parts[0],
kind=kind,
entity=parts[2],
python/mirage/core/postgres/search.py
+200 -22
View File
@@ -18,6 +18,10 @@ import orjson
from mirage.accessor.postgres import PostgresAccessor
from mirage.core.postgres import _client
from mirage.core.postgres._client import (canonicalize_row, qualified,
quote_ident)
from mirage.core.postgres._schema_json import build_entity_schema_json
from mirage.core.postgres.semantic import build_entity_semantic_json
_TEXT_TYPES = (
"text",
@@ -39,57 +43,231 @@ async def _text_columns(conn, schema: str, name: str) -> list[str]:
return [r["column_name"] for r in rows]
async def search_entity(accessor: PostgresAccessor, schema: str, kind: str,
entity: str, pattern: str,
limit: int) -> list[dict[str, Any]]:
def _escape_like(pattern: str) -> str:
"""Escape LIKE/ILIKE wildcards so the pattern matches as a literal.
Postgres LIKE treats % and _ as wildcards and \\ as the default escape
char; grep's substring pattern has no such meaning, so `user_id` must not
match `userXid`.
Args:
pattern (str): the literal substring to match.
"""
return (pattern.replace("\\", "\\\\").replace("%",
"\\%").replace("_", "\\_"))
async def search_entity(
accessor: PostgresAccessor,
schema: str,
kind: str,
entity: str,
pattern: str,
limit: int,
*,
case_insensitive: bool = False) -> list[dict[str, Any]]:
pool = await accessor.pool()
async with pool.acquire() as conn:
cols = await _text_columns(conn, schema, entity)
if not cols:
return []
where = " OR ".join(f'"{c}"::text ILIKE $1' for c in cols)
sql = f'SELECT * FROM "{schema}"."{entity}" WHERE {where} LIMIT $2'
rows = await conn.fetch(sql, f"%{pattern}%", limit)
return [dict(r) for r in rows]
op = "ILIKE" if case_insensitive else "LIKE"
where = " OR ".join(f"{quote_ident(c)}::text {op} $1" for c in cols)
sql = (f"SELECT * FROM {qualified(schema, entity)} "
f"WHERE {where} LIMIT $2")
rows = await conn.fetch(sql, f"%{_escape_like(pattern)}%", limit)
return [canonicalize_row(dict(r)) for r in rows]
async def search_kind(
accessor: PostgresAccessor, schema: str, kind: str, pattern: str,
limit: int) -> list[tuple[str, str, str, list[dict[str, Any]]]]:
async def search_entity_metadata(accessor: PostgresAccessor,
schema: str,
kind: str,
entity: str,
pattern: str,
*,
case_insensitive: bool = False) -> list[str]:
"""Grep an entity's rendered metadata files.
The LIKE push-down only ever sees row values, so schema.json and
semantic.json would be invisible at directory scope: `grep -r` would
report "not found" for content that is plainly there. These documents
are rendered, not stored, so the only honest way to match them is to
render and scan. Matching mirrors grep: case-sensitive unless -i is set.
Args:
accessor (PostgresAccessor): backend handle.
schema (str): the owning schema.
kind (str): "tables" or "views".
entity (str): the entity name.
pattern (str): the literal substring to match.
case_insensitive (bool): True when -i folds case.
"""
entity_kind = "table" if kind == "tables" else "view"
needle = pattern.lower() if case_insensitive else pattern
docs = (
("schema.json", await build_entity_schema_json(accessor, schema,
entity, entity_kind)),
("semantic.json", await
build_entity_semantic_json(accessor, schema, entity, entity_kind)),
)
lines: list[str] = []
for name, doc in docs:
rendered = orjson.dumps(doc, option=orjson.OPT_INDENT_2).decode()
for line in rendered.splitlines():
hay = line.lower() if case_insensitive else line
if needle in hay:
lines.append(f"{schema}/{kind}/{entity}/{name}:{line}")
return lines
async def search_kind_metadata(accessor: PostgresAccessor,
schema: str,
kind: str,
pattern: str,
*,
case_insensitive: bool = False) -> list[str]:
"""Grep every entity's metadata files under one kind directory.
Args:
accessor (PostgresAccessor): backend handle.
schema (str): the owning schema.
kind (str): "tables" or "views".
pattern (str): the literal substring to match.
case_insensitive (bool): True when -i folds case.
"""
names = await _entity_names(accessor, schema, kind)
lines: list[str] = []
for n in names:
lines.extend(await
search_entity_metadata(accessor,
schema,
kind,
n,
pattern,
case_insensitive=case_insensitive))
return lines
async def search_schema_metadata(accessor: PostgresAccessor,
schema: str,
pattern: str,
*,
case_insensitive: bool = False) -> list[str]:
"""Grep metadata files across both kinds of one schema.
Args:
accessor (PostgresAccessor): backend handle.
schema (str): the owning schema.
pattern (str): the literal substring to match.
case_insensitive (bool): True when -i folds case.
"""
lines: list[str] = []
for kind in ("tables", "views"):
lines.extend(await
search_kind_metadata(accessor,
schema,
kind,
pattern,
case_insensitive=case_insensitive))
return lines
async def search_database_metadata(
accessor: PostgresAccessor,
pattern: str,
*,
case_insensitive: bool = False) -> list[str]:
"""Grep metadata files across every visible schema.
Args:
accessor (PostgresAccessor): backend handle.
pattern (str): the literal substring to match.
case_insensitive (bool): True when -i folds case.
"""
pool = await accessor.pool()
async with pool.acquire() as conn:
schemas = await _client.list_schemas(conn, accessor.config.schemas)
lines: list[str] = []
for s in schemas:
lines.extend(await
search_schema_metadata(accessor,
s,
pattern,
case_insensitive=case_insensitive))
return lines
async def _entity_names(accessor: PostgresAccessor, schema: str,
kind: str) -> list[str]:
pool = await accessor.pool()
async with pool.acquire() as conn:
if kind == "tables":
names = await _client.list_tables(conn, schema)
else:
views = await _client.list_views(conn, schema)
mviews = await _client.list_matviews(conn, schema)
names = sorted(set(views) | set(mviews))
return await _client.list_tables(conn, schema)
views = await _client.list_views(conn, schema)
mviews = await _client.list_matviews(conn, schema)
return sorted(set(views) | set(mviews))
async def search_kind(
accessor: PostgresAccessor,
schema: str,
kind: str,
pattern: str,
limit: int,
*,
case_insensitive: bool = False
) -> list[tuple[str, str, str, list[dict[str, Any]]]]:
names = await _entity_names(accessor, schema, kind)
out: list[tuple[str, str, str, list[dict[str, Any]]]] = []
for n in names:
rows = await search_entity(accessor, schema, kind, n, pattern, limit)
rows = await search_entity(accessor,
schema,
kind,
n,
pattern,
limit,
case_insensitive=case_insensitive)
if rows:
out.append((schema, kind, n, rows))
return out
async def search_schema(
accessor: PostgresAccessor, schema: str, pattern: str,
limit: int) -> list[tuple[str, str, str, list[dict[str, Any]]]]:
accessor: PostgresAccessor,
schema: str,
pattern: str,
limit: int,
*,
case_insensitive: bool = False
) -> list[tuple[str, str, str, list[dict[str, Any]]]]:
out: list[tuple[str, str, str, list[dict[str, Any]]]] = []
for kind in ("tables", "views"):
out.extend(await search_kind(accessor, schema, kind, pattern, limit))
out.extend(await search_kind(accessor,
schema,
kind,
pattern,
limit,
case_insensitive=case_insensitive))
return out
async def search_database(
accessor: PostgresAccessor, pattern: str,
limit: int) -> list[tuple[str, str, str, list[dict[str, Any]]]]:
accessor: PostgresAccessor,
pattern: str,
limit: int,
*,
case_insensitive: bool = False
) -> list[tuple[str, str, str, list[dict[str, Any]]]]:
pool = await accessor.pool()
async with pool.acquire() as conn:
schemas = await _client.list_schemas(conn, accessor.config.schemas)
out: list[tuple[str, str, str, list[dict[str, Any]]]] = []
for s in schemas:
out.extend(await search_schema(accessor, s, pattern, limit))
out.extend(await search_schema(accessor,
s,
pattern,
limit,
case_insensitive=case_insensitive))
return out
python/mirage/core/postgres/semantic.py
+177
View File
@@ -0,0 +1,177 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from typing import Any
from mirage.accessor.postgres import PostgresAccessor
from mirage.core.postgres import _client
SAMPLE_VALUES_LIMIT = 10
TIME_TYPES = frozenset({
"date",
"time without time zone",
"time with time zone",
"timestamp without time zone",
"timestamp with time zone",
})
NUMERIC_TYPES = frozenset({
"bigint",
"double precision",
"integer",
"money",
"numeric",
"real",
"smallint",
})
def classify_column(name: str, data_type: str, key_columns: set[str]) -> str:
"""Assign a column its semantic role.
Mirrors the dimension / time_dimension / fact split of the Snowflake
semantic view vocabulary. Keys stay dimensions even when numeric: an
id is something you group or join by, never something you sum.
Args:
name (str): column name.
data_type (str): the information_schema data type.
key_columns (set[str]): primary-key and foreign-key column names.
"""
if name in key_columns:
return "dimensions"
if data_type in TIME_TYPES:
return "time_dimensions"
if data_type in NUMERIC_TYPES:
return "facts"
return "dimensions"
def build_column_entry(column: dict[str, Any], comment: str | None,
enum: dict[str, Any] | None,
stats: dict[str, Any] | None) -> dict[str, Any]:
"""Render one column in the semantic vocabulary.
Empty fields are omitted rather than emitted as null. The whole point
of this artifact is to fit an agent's context budget, so absent
metadata should cost nothing.
Args:
column (dict[str, Any]): the entry from fetch_columns.
comment (str | None): the column's COMMENT, if any.
enum (dict[str, Any] | None): declared enum type and labels, if the
column's type is an enum.
stats (dict[str, Any] | None): the pg_stats row, if ANALYZE has run.
"""
entry: dict[str, Any] = {
"name": column["name"],
"expr": column["name"],
"data_type": enum["type"] if enum else column["type"],
}
if comment:
entry["description"] = comment
if enum:
entry["is_enum"] = True
entry["sample_values"] = enum["labels"][:SAMPLE_VALUES_LIMIT]
elif stats and stats["most_common_vals"]:
# most_common_vals is null for high-cardinality columns, so this
# self-selects the ones where example values actually help.
entry["sample_values"] = (
stats["most_common_vals"][:SAMPLE_VALUES_LIMIT])
return entry
def build_relationships(foreign_keys: list[dict[str, Any]], schema: str,
name: str) -> list[dict[str, Any]]:
"""Render foreign keys as semantic relationships.
Args:
foreign_keys (list[dict[str, Any]]): entries from fetch_foreign_keys.
schema (str): the owning schema.
name (str): the owning entity.
"""
relationships: list[dict[str, Any]] = []
for fk in foreign_keys:
ref = fk["references"]
relationships.append({
"left_table":
f"{schema}.{name}",
"right_table":
f"{ref['schema']}.{ref['table']}",
"relationship_columns": [{
"left_column": left,
"right_column": right,
} for left, right in zip(fk["columns"], ref["columns"])],
})
return relationships
async def build_entity_semantic_json(accessor: PostgresAccessor, schema: str,
name: str, kind: str) -> dict[str, Any]:
"""Build the derived semantic model for one entity.
Uses the Snowflake semantic view field vocabulary so the artifact is
familiar to models and interchangeable with a curated one. Everything
here is derived from the catalog; synonyms, metrics and verified
queries have no catalog source and are left for a curated overlay.
Args:
accessor (PostgresAccessor): backend handle.
schema (str): the owning schema.
name (str): the entity name.
kind (str): "table" or "view".
"""
pool = await accessor.pool()
async with pool.acquire() as conn:
columns = await _client.fetch_columns(conn, schema, name)
pk = await _client.fetch_primary_key(conn, schema, name)
fks = await _client.fetch_foreign_keys(conn, schema, name)
table_comment = await _client.fetch_table_comment(conn, schema, name)
comments = await _client.fetch_column_comments(conn, schema, name)
enums = await _client.fetch_enum_columns(conn, schema, name)
stats = await _client.fetch_column_stats(conn, schema, name)
key_columns = set(pk)
for fk in fks:
key_columns.update(fk["columns"])
buckets: dict[str, list[dict[str, Any]]] = {
"dimensions": [],
"time_dimensions": [],
"facts": [],
}
for column in columns:
role = classify_column(column["name"], column["type"], key_columns)
buckets[role].append(
build_column_entry(column, comments.get(column["name"]),
enums.get(column["name"]),
stats.get(column["name"])))
doc: dict[str, Any] = {
"name": name,
"schema": schema,
"kind": kind,
}
if table_comment:
doc["description"] = table_comment
if pk:
doc["primary_key"] = pk
for role in ("dimensions", "time_dimensions", "facts"):
if buckets[role]:
doc[role] = buckets[role]
relationships = build_relationships(fks, schema, name)
if relationships:
doc["relationships"] = relationships
return doc
python/mirage/core/postgres/stat.py
+12
View File
@@ -84,6 +84,18 @@ async def stat(accessor: PostgresAccessor,
"name": scope.entity
})
if scope.level == "entity_semantic":
if not await _entity_exists(accessor, scope.schema, scope.kind,
scope.entity):
raise enoent(path)
return FileStat(name="semantic.json",
type=FileType.JSON,
extra={
"schema": scope.schema,
"kind": scope.kind,
"name": scope.entity
})
if scope.level == "entity_rows":
if not await _entity_exists(accessor, scope.schema, scope.kind,
scope.entity):
python/mirage/resource/postgres/prompt.py
+5
View File
@@ -18,10 +18,15 @@ PROMPT = """\
<schema>/ Postgres schema (namespace)
tables/<table>/
schema.json column types, PK/FK, indexes
semantic.json descriptions, roles, sample values
rows.jsonl data (size-guarded)
views/<view>/
schema.json
semantic.json
rows.jsonl
Read semantic.json to learn what a table means: it splits columns into
dimensions (group by), time_dimensions and facts (aggregate), and
carries COMMENTs, enum domains and example values.
Read database.json first to plan joins. Reading rows.jsonl is refused
for tables above the configured row/byte threshold; use head, tail, wc,
or grep, all of which push predicates down to SQL."""
python/mirage/utils/json_canonical.py
+31
View File
@@ -0,0 +1,31 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import math
from typing import Any
def canonicalize_value(value: Any) -> Any:
if isinstance(value, dict):
return {k: canonicalize_value(v) for k, v in value.items()}
if isinstance(value, list):
return [canonicalize_value(v) for v in value]
if (isinstance(value, float) and not isinstance(value, bool)
and math.isfinite(value) and value.is_integer()):
return int(value)
return value
def canonicalize_row(row: dict[str, Any]) -> dict[str, Any]:
return {k: canonicalize_value(v) for k, v in row.items()}
python/tests/commands/builtin/postgres/test_pushdown_glob.py
+249
View File
@@ -0,0 +1,249 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import AsyncMock, patch
import pytest
from mirage.accessor.postgres import PostgresAccessor
from mirage.cache.index import NULL_INDEX
from mirage.commands.builtin.postgres.grep import grep
from mirage.commands.builtin.postgres.rg import rg
from mirage.commands.builtin.postgres.tail import tail
from mirage.io.types import IOResult
from mirage.resource.postgres.config import PostgresConfig
from mirage.types import PathSpec
CONCRETE = "/public/tables/books/rows.jsonl"
GLOB = "/public/tables/*/rows.jsonl"
@pytest.fixture
def accessor():
return PostgresAccessor(config=PostgresConfig(
dsn="postgres://u:p@localhost:5432/db"))
def _glob_path() -> PathSpec:
# The dispatcher hands a glob operand through with the trailing segment
# in `pattern` and the wildcard still in `directory`; detect_scope would
# otherwise read the "*" as an entity literally named "*".
return PathSpec(virtual=GLOB,
directory="/public/tables",
resource_path=GLOB.strip("/"),
pattern="rows.jsonl",
resolved=False)
def _resolved_pair() -> list[PathSpec]:
return [
PathSpec(virtual=p,
directory="/public/tables",
resource_path=p.strip("/")) for p in (
"/public/tables/authors/rows.jsonl",
"/public/tables/books/rows.jsonl",
)
]
@pytest.mark.asyncio
async def test_grep_glob_skips_pushdown_and_expands(accessor):
seen: dict[str, object] = {}
async def fake_resolve(_accessor, _paths, index=None):
return _resolved_pair()
async def fake_generic(paths, _texts, _flags, **_kwargs):
seen["generic"] = [p.virtual for p in paths]
return b"", IOResult()
with patch(
"mirage.commands.builtin.postgres.grep.search_entity",
new=AsyncMock(side_effect=AssertionError("pushdown ran on glob")),
), patch(
"mirage.commands.builtin.postgres.grep._stat",
new=AsyncMock(side_effect=AssertionError("stat ran on glob")),
), patch(
"mirage.commands.builtin.postgres.grep.resolve_glob",
new=fake_resolve,
), patch(
"mirage.commands.builtin.postgres.grep.generic_grep",
new=fake_generic,
):
_, io = await grep(accessor, [_glob_path()], "ada", index=NULL_INDEX)
assert io.exit_code == 0
assert seen["generic"] == [
"/public/tables/authors/rows.jsonl",
"/public/tables/books/rows.jsonl",
]
@pytest.mark.asyncio
async def test_grep_concrete_path_still_uses_pushdown(accessor):
search = AsyncMock(return_value=[])
with patch(
"mirage.commands.builtin.postgres.grep.search_entity",
new=search,
), patch(
"mirage.commands.builtin.postgres.grep._stat",
new=AsyncMock(),
), patch(
"mirage.commands.builtin.postgres.grep.resolve_glob",
new=AsyncMock(side_effect=AssertionError("glob ran")),
):
_, io = await grep(accessor, [
PathSpec(virtual=CONCRETE,
directory="/public/tables/books",
resource_path=CONCRETE.strip("/"))
],
"ada",
index=NULL_INDEX)
assert io.exit_code == 1
search.assert_awaited_once()
def _concrete_path() -> PathSpec:
return PathSpec(virtual=CONCRETE,
directory="/public/tables/books",
resource_path=CONCRETE.strip("/"))
@pytest.mark.asyncio
@pytest.mark.parametrize("flags", [
{
"v": True
},
{
"c": True
},
{
"args_l": True
},
{
"n": True
},
])
async def test_grep_shaping_flag_skips_pushdown(accessor, flags):
# A shaping flag cannot be honored by the ILIKE push-down (which prints
# whole matching rows), so the wrapper must defer to the generic scan.
seen: dict[str, object] = {}
async def fake_resolve(_accessor, _paths, index=None):
return [_concrete_path()]
async def fake_generic(paths, _texts, _flags, **_kwargs):
seen["generic"] = [p.virtual for p in paths]
return b"", IOResult()
with patch(
"mirage.commands.builtin.postgres.grep.search_entity",
new=AsyncMock(side_effect=AssertionError("pushdown ran w/ flag")),
), patch(
"mirage.commands.builtin.postgres.grep.resolve_glob",
new=fake_resolve,
), patch(
"mirage.commands.builtin.postgres.grep.generic_grep",
new=fake_generic,
):
await grep(accessor, [_concrete_path()],
"ada",
index=NULL_INDEX,
**flags)
assert seen["generic"] == [CONCRETE]
@pytest.mark.asyncio
async def test_grep_regex_pattern_skips_pushdown(accessor):
# A pattern with regex meaning is matched literally by ILIKE, so it must
# take the generic scan rather than silently mis-matching.
seen: dict[str, object] = {}
async def fake_resolve(_accessor, _paths, index=None):
return [_concrete_path()]
async def fake_generic(paths, _texts, _flags, **_kwargs):
seen["generic"] = [p.virtual for p in paths]
return b"", IOResult()
with patch(
"mirage.commands.builtin.postgres.grep.search_entity",
new=AsyncMock(side_effect=AssertionError("pushdown ran on regex")),
), patch(
"mirage.commands.builtin.postgres.grep.resolve_glob",
new=fake_resolve,
), patch(
"mirage.commands.builtin.postgres.grep.generic_grep",
new=fake_generic,
):
await grep(accessor, [_concrete_path()], "a.b", index=NULL_INDEX)
assert seen["generic"] == [CONCRETE]
@pytest.mark.asyncio
async def test_rg_glob_skips_pushdown_and_expands(accessor):
seen: dict[str, object] = {}
async def fake_resolve(_accessor, _paths, index=None):
return _resolved_pair()
async def fake_generic(paths, _texts, _flags, **_kwargs):
seen["generic"] = [p.virtual for p in paths]
return b"", IOResult()
with patch(
"mirage.commands.builtin.postgres.rg.search_entity",
new=AsyncMock(side_effect=AssertionError("pushdown ran on glob")),
), patch(
"mirage.commands.builtin.postgres.rg._stat",
new=AsyncMock(side_effect=AssertionError("stat ran on glob")),
), patch(
"mirage.commands.builtin.postgres.rg.resolve_glob",
new=fake_resolve,
), patch(
"mirage.commands.builtin.postgres.rg.generic_rg",
new=fake_generic,
):
_, io = await rg(accessor, [_glob_path()], "ada", index=NULL_INDEX)
assert io.exit_code == 0
assert seen["generic"] == [
"/public/tables/authors/rows.jsonl",
"/public/tables/books/rows.jsonl",
]
@pytest.mark.asyncio
async def test_tail_glob_does_not_query_a_relation_named_star(accessor):
# Before the fix this reached count_rows with entity="*" and surfaced
# 'relation "public.*" does not exist' to the user.
async def fake_resolve(_accessor, _paths, index=None):
return _resolved_pair()
with patch(
"mirage.commands.builtin.postgres.tail._client.count_rows",
new=AsyncMock(side_effect=AssertionError("pushdown ran on glob")),
), patch(
"mirage.commands.builtin.postgres.tail.resolve_glob",
new=fake_resolve,
), patch(
"mirage.commands.builtin.postgres.tail.tail_multi",
new=lambda paths, **_kw: b"",
):
_, io = await tail(accessor, [_glob_path()], n="1", index=NULL_INDEX)
assert io.exit_code == 0
python/tests/commands/builtin/test_grep_helper.py
+85
View File
@@ -176,3 +176,88 @@ async def test_grep_files_only_recursive_scans_file_operands():
warnings=[],
)
assert hits == ["/data/notes.txt"]
@pytest.mark.parametrize("pattern,fixed,expected", [
("abc", False, True),
("a-b_c.d", False, False),
("plain text", False, True),
("a.b", False, False),
("a*b", False, False),
("^start", False, False),
("a.b", True, True),
("a\nb", False, False),
("a\nb", True, True),
])
def test_is_literal_pattern(pattern, fixed, expected):
assert grep_helper.is_literal_pattern(pattern, fixed) is expected
@pytest.mark.parametrize("flags,expected", [
({}, False),
({
"i": True
}, False),
({
"F": True
}, False),
({
"r": True
}, False),
({
"v": True
}, True),
({
"n": True
}, True),
({
"c": True
}, True),
({
"args_l": True
}, True),
({
"w": True
}, True),
({
"o": True
}, True),
({
"q": True
}, True),
({
"H": True
}, True),
({
"h": True
}, True),
({
"m": "3"
}, True),
({
"A": "2"
}, True),
({
"B": "2"
}, True),
({
"C": "2"
}, True),
])
def test_has_search_shaping_flags(flags, expected):
assert grep_helper.has_search_shaping_flags(flags) is expected
def test_search_pushdown_ok_plain_literal():
assert grep_helper.search_pushdown_ok({}, "ada") is True
assert grep_helper.search_pushdown_ok({"i": True}, "ada") is True
def test_search_pushdown_ok_rejects_shaping_flag():
assert grep_helper.search_pushdown_ok({"v": True}, "ada") is False
assert grep_helper.search_pushdown_ok({"c": True}, "ada") is False
def test_search_pushdown_ok_rejects_regex_but_allows_fixed_string():
assert grep_helper.search_pushdown_ok({}, "a.b") is False
assert grep_helper.search_pushdown_ok({"F": True}, "a.b") is True
python/tests/commands/builtin/utils/test_paths.py
+33 -1
View File
@@ -12,7 +12,9 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.commands.builtin.utils.paths import default_paths, resolve_script
from mirage.commands.builtin.utils.paths import (default_paths,
has_unresolved_glob,
resolve_script)
from mirage.types import PathSpec
@@ -22,6 +24,36 @@ def _spec(virtual: str) -> PathSpec:
resource_path=virtual.strip("/"))
def _glob_spec(virtual: str, directory: str, pattern: str) -> PathSpec:
return PathSpec(virtual=virtual,
directory=directory,
resource_path=virtual.strip("/"),
pattern=pattern,
resolved=False)
def test_has_unresolved_glob_false_for_concrete_paths():
assert has_unresolved_glob([_spec("/pg/public/tables/books/rows.jsonl")
]) is False
def test_has_unresolved_glob_false_for_empty():
assert has_unresolved_glob([]) is False
def test_has_unresolved_glob_true_for_pattern():
spec = _glob_spec("/pg/public/tables/*/rows.jsonl", "/pg/public/tables",
"rows.jsonl")
assert has_unresolved_glob([spec]) is True
def test_has_unresolved_glob_true_when_any_operand_globs():
concrete = _spec("/pg/public/tables/books/rows.jsonl")
globbed = _glob_spec("/pg/public/tables/*/schema.json",
"/pg/public/tables", "schema.json")
assert has_unresolved_glob([concrete, globbed]) is True
def test_resolve_script_absolute_is_normalized():
spec = resolve_script("/data/../data/run.py", _spec("/cwd"))
assert spec.virtual == "/data/run.py"
python/tests/core/mongodb/test_sampler.py
+13 -2
View File
@@ -12,7 +12,7 @@
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from unittest.mock import AsyncMock, MagicMock
from unittest.mock import MagicMock
import pytest
from bson import Decimal128, ObjectId
@@ -36,7 +36,7 @@ class _AsyncIter:
def _col(docs):
col = MagicMock()
col.aggregate = AsyncMock(return_value=_AsyncIter(docs))
col.find = MagicMock(return_value=_AsyncIter(docs))
return col
@@ -116,6 +116,17 @@ async def test_sample_field_types_handles_bson_scalars():
assert by_path["oid"]["types"] == {"objectId": 1.0}
@pytest.mark.asyncio
async def test_sample_field_types_whole_valued_double_classified_as_int():
# The JS driver returns BSON double as a plain number and cannot tell a
# whole-valued double from an int, so both languages classify by value:
# 5.0 -> int, 4.5 -> double. Keeps py/ts schema.json inference identical.
docs = [{"_id": 1, "rating": 5.0}, {"_id": 2, "rating": 4.5}]
out = await sample_field_types(_col(docs), sample_size=2)
by_path = {f["path"]: f for f in out}
assert by_path["rating"]["types"] == {"int": 0.5, "double": 0.5}
@pytest.mark.asyncio
async def test_sample_field_types_empty_sample_returns_empty_list():
out = await sample_field_types(_col([]), sample_size=5)
python/tests/core/postgres/test_client.py
+58
View File
@@ -28,6 +28,64 @@ def mock_conn():
return conn
def test_quote_ident_escapes_embedded_quotes():
assert _client.quote_ident("users") == '"users"'
assert _client.quote_ident('a"b') == '"a""b"'
def test_qualified_composes_both_parts():
assert _client.qualified("public", "users") == '"public"."users"'
def test_quote_ident_neutralizes_injection():
payload = 'books" CROSS JOIN secret AS "s'
quoted = _client.quote_ident(payload)
# Every embedded quote is doubled, so the payload cannot terminate the
# identifier and become executable SQL: it stays one (nonexistent) name.
assert quoted == '"books"" CROSS JOIN secret AS ""s"'
# After stripping the outer wrapper and collapsing every doubled-quote
# pair, no lone quote survives to break out of the identifier.
assert '"' not in quoted[1:-1].replace('""', "")
def test_canonicalize_value_drops_trailing_zero_on_whole_floats():
# Postgres renders a whole-valued double as `5` (to_jsonb), and node-pg
# gives JS number 5; asyncpg gives Python float 5.0 which orjson would
# print as `5.0`. Canonicalizing to int keeps py rows.jsonl byte-identical
# to ts (and to Postgres canonical JSON).
assert _client.canonicalize_value(5.0) == 5
assert isinstance(_client.canonicalize_value(5.0), int)
assert _client.canonicalize_value(4.5) == 4.5
assert isinstance(_client.canonicalize_value(4.5), float)
def test_canonicalize_value_leaves_non_floats_and_specials():
assert _client.canonicalize_value(True) is True
assert _client.canonicalize_value("5.0") == "5.0"
assert _client.canonicalize_value(float("inf")) == float("inf")
def test_canonicalize_row_recurses_into_nested_structures():
row = {"r": 5.0, "tags": [1.0, 2.5], "meta": {"n": 3.0}}
assert _client.canonicalize_row(row) == {
"r": 5,
"tags": [1, 2.5],
"meta": {
"n": 3
},
}
@pytest.mark.asyncio
async def test_count_rows_quotes_a_malicious_name(mock_conn):
mock_conn.fetchval.return_value = 0
await _client.count_rows(mock_conn, "public",
'books" CROSS JOIN secret AS "s')
sql = mock_conn.fetchval.call_args.args[0]
assert '"books"" CROSS JOIN secret AS ""s"' in sql
assert 'FROM "public"."books" CROSS JOIN secret' not in sql
@pytest.mark.asyncio
async def test_list_schemas(mock_conn):
mock_conn.fetch.return_value = [
python/tests/core/postgres/test_readdir.py
+2
View File
@@ -106,6 +106,7 @@ async def test_readdir_entity_lists_schema_and_rows(accessor, index):
directory="/public/tables/users"), index)
assert result == [
"/public/tables/users/schema.json",
"/public/tables/users/semantic.json",
"/public/tables/users/rows.jsonl",
]
@@ -119,6 +120,7 @@ async def test_readdir_view_entity_lists_schema_and_rows(accessor, index):
directory="/analytics/views/daily_revenue"), index)
assert result == [
"/analytics/views/daily_revenue/schema.json",
"/analytics/views/daily_revenue/semantic.json",
"/analytics/views/daily_revenue/rows.jsonl",
]
python/tests/core/postgres/test_scope.py
+9
View File
@@ -87,6 +87,15 @@ def test_entity_schema_file():
assert s.file == "schema.json"
def test_entity_semantic_file():
s = detect_scope(_ps("/public/tables/users/semantic.json"))
assert s.level == "entity_semantic"
assert s.schema == "public"
assert s.kind == "tables"
assert s.entity == "users"
assert s.file == "semantic.json"
def test_entity_rows_file():
s = detect_scope(_ps("/public/tables/users/rows.jsonl"))
assert s.level == "entity_rows"
python/tests/core/postgres/test_search.py
+21 -2
View File
@@ -106,12 +106,31 @@ async def test_search_entity_builds_or_clause():
accessor = _accessor_with_conn(conn)
await search_entity(accessor, "public", "tables", "t1", "pat", limit=5)
final_call_sql = conn.fetch.await_args_list[1].args[0]
assert "ILIKE" in final_call_sql
assert final_call_sql.count("ILIKE") == 3
# grep is case-sensitive by default, so the push-down uses LIKE, not ILIKE.
assert "ILIKE" not in final_call_sql
assert final_call_sql.count("LIKE") == 3
assert "$1" in final_call_sql
assert "LIMIT $2" in final_call_sql
@pytest.mark.asyncio
async def test_search_entity_case_insensitive_uses_ilike_and_escapes():
conn = MagicMock()
conn.fetch = AsyncMock(side_effect=[[{"column_name": "a"}], []])
accessor = _accessor_with_conn(conn)
await search_entity(accessor,
"public",
"tables",
"t1",
"user_id",
limit=5,
case_insensitive=True)
final_call_sql = conn.fetch.await_args_list[1].args[0]
assert "ILIKE" in final_call_sql
# `_` is escaped so it matches literally, not as a LIKE wildcard.
assert conn.fetch.await_args_list[1].args[1] == "%user\\_id%"
@pytest.mark.asyncio
async def test_search_kind_iterates_tables():
accessor = _accessor()
python/tests/core/postgres/test_semantic.py
+293
View File
@@ -0,0 +1,293 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from contextlib import asynccontextmanager
from unittest.mock import AsyncMock, MagicMock
import pytest
from mirage.accessor.postgres import PostgresAccessor
from mirage.core.postgres.semantic import (SAMPLE_VALUES_LIMIT,
build_column_entry,
build_entity_semantic_json,
build_relationships,
classify_column)
from mirage.resource.postgres.config import PostgresConfig
COLUMNS = [
{
"name": "order_id",
"type": "integer",
"nullable": False
},
{
"name": "customer_id",
"type": "integer",
"nullable": True
},
{
"name": "status",
"type": "USER-DEFINED",
"nullable": True
},
{
"name": "channel",
"type": "text",
"nullable": True
},
{
"name": "total_amount",
"type": "numeric",
"nullable": True
},
{
"name": "placed_at",
"type": "timestamp with time zone",
"nullable": True
},
]
FOREIGN_KEYS = [{
"columns": ["customer_id"],
"references": {
"schema": "public",
"table": "customers",
"columns": ["id"],
},
}]
@asynccontextmanager
async def _fake_acquire():
yield MagicMock()
def _accessor() -> PostgresAccessor:
a = PostgresAccessor(PostgresConfig(dsn="postgres://localhost/db"))
pool = MagicMock()
pool.acquire = lambda: _fake_acquire()
a.pool = AsyncMock(return_value=pool)
return a
@pytest.fixture
def accessor():
return _accessor()
@pytest.fixture
def client(monkeypatch):
fakes = {
"fetch_columns":
AsyncMock(return_value=COLUMNS),
"fetch_primary_key":
AsyncMock(return_value=["order_id"]),
"fetch_foreign_keys":
AsyncMock(return_value=FOREIGN_KEYS),
"fetch_table_comment":
AsyncMock(return_value="Customer orders."),
"fetch_column_comments":
AsyncMock(return_value={"total_amount": "Order total in USD."}),
"fetch_enum_columns":
AsyncMock(
return_value={
"status": {
"type": "order_status",
"labels": ["pending", "shipped", "cancelled"],
}
}),
"fetch_column_stats":
AsyncMock(
return_value={
"channel": {
"n_distinct": 3.0,
"most_common_vals": ["web", "retail", "partner"],
},
"total_amount": {
"n_distinct": -1.0,
"most_common_vals": [],
},
}),
}
for name, fake in fakes.items():
monkeypatch.setattr(f"mirage.core.postgres._client.{name}", fake)
return fakes
def test_classify_key_column_is_dimension_even_when_numeric():
assert classify_column("order_id", "integer", {"order_id"}) == "dimensions"
def test_classify_numeric_non_key_is_fact():
assert classify_column("total_amount", "numeric", set()) == "facts"
def test_classify_timestamp_is_time_dimension():
role = classify_column("placed_at", "timestamp with time zone", set())
assert role == "time_dimensions"
def test_classify_text_is_dimension():
assert classify_column("channel", "text", set()) == "dimensions"
def test_column_entry_omits_absent_metadata():
entry = build_column_entry({
"name": "channel",
"type": "text"
}, None, None, None)
assert entry == {"name": "channel", "expr": "channel", "data_type": "text"}
def test_column_entry_uses_enum_type_and_labels():
entry = build_column_entry({
"name": "status",
"type": "USER-DEFINED"
}, None, {
"type": "order_status",
"labels": ["pending", "shipped"]
}, None)
assert entry["data_type"] == "order_status"
assert entry["is_enum"] is True
assert entry["sample_values"] == ["pending", "shipped"]
def test_column_entry_takes_sample_values_from_stats():
entry = build_column_entry({
"name": "channel",
"type": "text"
}, None, None, {
"n_distinct": 3.0,
"most_common_vals": ["web", "retail"]
})
assert entry["sample_values"] == ["web", "retail"]
assert "is_enum" not in entry
def test_column_entry_skips_sample_values_when_stats_empty():
entry = build_column_entry({
"name": "total_amount",
"type": "numeric"
}, None, None, {
"n_distinct": -1.0,
"most_common_vals": []
})
assert "sample_values" not in entry
def test_column_entry_caps_sample_values():
many = [str(i) for i in range(SAMPLE_VALUES_LIMIT + 5)]
entry = build_column_entry({
"name": "channel",
"type": "text"
}, None, None, {
"n_distinct": 15.0,
"most_common_vals": many
})
assert len(entry["sample_values"]) == SAMPLE_VALUES_LIMIT
def test_build_relationships_pairs_columns():
rels = build_relationships(FOREIGN_KEYS, "public", "orders")
assert rels == [{
"left_table":
"public.orders",
"right_table":
"public.customers",
"relationship_columns": [{
"left_column": "customer_id",
"right_column": "id",
}],
}]
def test_build_relationships_empty_without_foreign_keys():
assert build_relationships([], "public", "orders") == []
@pytest.mark.asyncio
async def test_semantic_json_splits_roles(accessor, client):
doc = await build_entity_semantic_json(accessor, "public", "orders",
"table")
assert [d["name"] for d in doc["dimensions"]
] == ["order_id", "customer_id", "status", "channel"]
assert [d["name"] for d in doc["time_dimensions"]] == ["placed_at"]
assert [d["name"] for d in doc["facts"]] == ["total_amount"]
@pytest.mark.asyncio
async def test_semantic_json_carries_comments(accessor, client):
doc = await build_entity_semantic_json(accessor, "public", "orders",
"table")
assert doc["description"] == "Customer orders."
total = next(f for f in doc["facts"] if f["name"] == "total_amount")
assert total["description"] == "Order total in USD."
@pytest.mark.asyncio
async def test_semantic_json_carries_enum_and_samples(accessor, client):
doc = await build_entity_semantic_json(accessor, "public", "orders",
"table")
status = next(d for d in doc["dimensions"] if d["name"] == "status")
assert status["data_type"] == "order_status"
assert status["sample_values"] == ["pending", "shipped", "cancelled"]
channel = next(d for d in doc["dimensions"] if d["name"] == "channel")
assert channel["sample_values"] == ["web", "retail", "partner"]
@pytest.mark.asyncio
async def test_semantic_json_head_fields(accessor, client):
doc = await build_entity_semantic_json(accessor, "public", "orders",
"table")
assert doc["name"] == "orders"
assert doc["schema"] == "public"
assert doc["kind"] == "table"
assert doc["primary_key"] == ["order_id"]
assert doc["relationships"][0]["right_table"] == "public.customers"
@pytest.mark.asyncio
async def test_semantic_json_omits_empty_sections(accessor, monkeypatch,
client):
monkeypatch.setattr(
"mirage.core.postgres._client.fetch_columns",
AsyncMock(return_value=[{
"name": "note",
"type": "text",
"nullable": True
}]))
monkeypatch.setattr("mirage.core.postgres._client.fetch_primary_key",
AsyncMock(return_value=[]))
monkeypatch.setattr("mirage.core.postgres._client.fetch_foreign_keys",
AsyncMock(return_value=[]))
monkeypatch.setattr("mirage.core.postgres._client.fetch_table_comment",
AsyncMock(return_value=None))
doc = await build_entity_semantic_json(accessor, "public", "notes",
"table")
assert "facts" not in doc
assert "time_dimensions" not in doc
assert "relationships" not in doc
assert "primary_key" not in doc
assert "description" not in doc
@pytest.mark.asyncio
async def test_semantic_json_survives_missing_pg_stats(accessor, monkeypatch,
client):
monkeypatch.setattr("mirage.core.postgres._client.fetch_column_stats",
AsyncMock(return_value={}))
doc = await build_entity_semantic_json(accessor, "public", "orders",
"table")
channel = next(d for d in doc["dimensions"] if d["name"] == "channel")
assert "sample_values" not in channel
python/tests/utils/test_json_canonical.py
+54
View File
@@ -0,0 +1,54 @@
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
from mirage.utils.json_canonical import canonicalize_row, canonicalize_value
def test_whole_valued_floats_become_int():
assert canonicalize_value(5.0) == 5
assert isinstance(canonicalize_value(5.0), int)
def test_fractional_floats_are_unchanged():
assert canonicalize_value(4.5) == 4.5
assert isinstance(canonicalize_value(4.5), float)
def test_bool_and_string_and_non_finite_untouched():
assert canonicalize_value(True) is True
assert canonicalize_value("5.0") == "5.0"
assert canonicalize_value(float("inf")) == float("inf")
assert canonicalize_value(float("nan")) != canonicalize_value(float("nan"))
def test_recurses_into_dicts_and_lists():
assert canonicalize_value({
"r": 5.0,
"xs": [1.0, 2.5]
}) == {
"r": 5,
"xs": [1, 2.5],
}
def test_canonicalize_row_maps_every_value():
assert canonicalize_row({
"a": 1.0,
"b": "x",
"c": 2.5
}) == {
"a": 1,
"b": "x",
"c": 2.5,
}
typescript/packages/core/src/commands/builtin/grep_helper.test.ts
+63
View File
@@ -20,9 +20,12 @@ import {
compilePattern,
extractRequiredLiteral,
grepFilesOnly,
hasSearchShapingFlags,
isLiteralPattern,
isRegexPattern,
mergePatternList,
NEVER_MATCH,
searchPushdownOk,
searchQuery,
} from './grep_helper.ts'
@@ -199,3 +202,63 @@ describe('grepFilesOnly', () => {
expect(hits).toEqual(['/data/notes.txt'])
})
})
describe('isLiteralPattern', () => {
it.each([
['abc', false, true],
['a-b_c.d', false, false],
['plain text', false, true],
['a.b', false, false],
['a*b', false, false],
['^start', false, false],
['a.b', true, true],
['a\nb', false, false],
['a\nb', true, true],
])('isLiteralPattern(%j, %j) === %j', (pattern, fixed, expected) => {
expect(isLiteralPattern(pattern, fixed)).toBe(expected)
})
})
describe('hasSearchShapingFlags', () => {
it.each([
[{}, false],
[{ i: true }, false],
[{ F: true }, false],
[{ r: true }, false],
[{ v: true }, true],
[{ n: true }, true],
[{ c: true }, true],
[{ args_l: true }, true],
[{ l: true }, true],
[{ w: true }, true],
[{ o: true }, true],
[{ q: true }, true],
[{ H: true }, true],
[{ h: true }, true],
[{ m: '3' }, true],
[{ A: '2' }, true],
[{ B: '2' }, true],
[{ C: '2' }, true],
])('hasSearchShapingFlags(%j) === %j', (flags, expected) => {
expect(hasSearchShapingFlags(flags as Record<string, string | boolean | string[]>)).toBe(
expected,
)
})
})
describe('searchPushdownOk', () => {
it('allows a plain literal, with or without -i', () => {
expect(searchPushdownOk({}, 'ada')).toBe(true)
expect(searchPushdownOk({ i: true }, 'ada')).toBe(true)
})
it('rejects any shaping flag', () => {
expect(searchPushdownOk({ v: true }, 'ada')).toBe(false)
expect(searchPushdownOk({ c: true }, 'ada')).toBe(false)
})
it('rejects a regex pattern but allows it under -F', () => {
expect(searchPushdownOk({}, 'a.b')).toBe(false)
expect(searchPushdownOk({ F: true }, 'a.b')).toBe(true)
})
})
typescript/packages/core/src/commands/builtin/grep_helper.ts
+44
View File
@@ -220,6 +220,50 @@ export function isLiteralPattern(pattern: string, fixedString: boolean): boolean
return pt === PatternType.EXACT || (pt === PatternType.SIMPLE && !pattern.includes('.'))
}
// True when a flag alters the match set or output shape of grep/rg. A search
// push-down prints each matching record as one whole line, so it cannot honor
// -v/-n/-c/-l/-w/-o/-m/-A/-B/-C/-q/-H/-h, rg's -I (no filename), nor rg's
// file-filtering --glob/--type; the wrapper must defer to the generic scan
// when any is present.
export function hasSearchShapingFlags(flags: Record<string, string | boolean | string[]>): boolean {
if (
flags.v === true ||
flags.n === true ||
flags.c === true ||
flags.args_l === true ||
flags.l === true ||
flags.w === true ||
flags.o === true ||
flags.q === true ||
flags.H === true ||
flags.h === true ||
flags.args_I === true
) {
return true
}
return (
typeof flags.m === 'string' ||
typeof flags.A === 'string' ||
typeof flags.B === 'string' ||
typeof flags.C === 'string' ||
flags.type !== undefined ||
flags.glob !== undefined
)
}
// True when a literal-substring push-down (LIKE/ILIKE) faithfully reproduces
// grep/rg: a literal pattern with no shaping flags. A newline-joined pattern
// list (-F with multiple -e) is a set of independent alternatives LIKE cannot
// express, so it stays on the generic path. Backends that push a real regex
// down (mongodb) gate on hasSearchShapingFlags alone instead.
export function searchPushdownOk(
flags: Record<string, string | boolean | string[]>,
pattern: string,
): boolean {
if (pattern.includes('\n')) return false
return isLiteralPattern(pattern, flags.F === true) && !hasSearchShapingFlags(flags)
}
export interface GrepLinesOptions {
invert: boolean
lineNumbers: boolean
typescript/packages/core/src/commands/builtin/mongodb/grep.ts
+11 -2
View File
@@ -16,6 +16,7 @@ import type { MongoDBAccessor } from '../../../accessor/mongodb.ts'
import type { IndexCacheStore } from '../../../cache/index/store.ts'
import { listDatabases } from '../../../core/mongodb/_client.ts'
import { resolveGlobOf } from '../generic_bind/index.ts'
import { hasUnresolvedGlob } from '../utils/operands.ts'
import { MONGODB_IO } from './io.ts'
import { read as mongoRead } from '../../../core/mongodb/read.ts'
import { readdir as mongoReaddir } from '../../../core/mongodb/readdir.ts'
@@ -32,7 +33,7 @@ import { type FileStat, type PathSpec, ResourceName } from '../../../types.ts'
import { command, type CommandFnResult, type CommandOpts } from '../../config.ts'
import { specOf } from '../../spec/builtins.ts'
import { grepGeneric } from '../generic/grep.ts'
import { patternArg } from '../grep_helper.ts'
import { hasSearchShapingFlags, patternArg } from '../grep_helper.ts'
import { formatRecords } from '../utils/output.ts'
import { searchProvision } from './_provision.ts'
@@ -55,8 +56,16 @@ async function grepCommand(
const pattern = patternArg(texts, opts.flags)
const limit = accessor.config.defaultSearchLimit
// The $regex push-down prints each matching document as a whole line, so
// output/match-shaping flags must defer to the generic scan below.
const first = paths[0]
if (first !== undefined && pattern !== null && !pattern.includes('\n')) {
if (
first !== undefined &&
!hasUnresolvedGlob(paths) &&
pattern !== null &&
!pattern.includes('\n') &&
!hasSearchShapingFlags(opts.flags)
) {
const scope = detectScope(first)
if (scope.level !== ScopeLevel.ROOT) {
typescript/packages/core/src/commands/builtin/mongodb/rg.ts
+58
View File
@@ -13,15 +13,27 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import type { MongoDBAccessor } from '../../../accessor/mongodb.ts'
import { listDatabases } from '../../../core/mongodb/_client.ts'
import { resolveGlobOf } from '../generic_bind/index.ts'
import { hasUnresolvedGlob } from '../utils/operands.ts'
import { MONGODB_IO } from './io.ts'
import { streamAny } from '../../../core/mongodb/read.ts'
import { readdir as mongoReaddir } from '../../../core/mongodb/readdir.ts'
import { detectScope } from '../../../core/mongodb/scope.ts'
import {
formatGrepResults,
searchCollection,
searchDatabase,
} from '../../../core/mongodb/search.ts'
import { stat as mongoStat } from '../../../core/mongodb/stat.ts'
import { ScopeLevel } from '../../../core/mongodb/types.ts'
import { IOResult } from '../../../io/types.ts'
import { type FileStat, ResourceName, type PathSpec } from '../../../types.ts'
import { command, type CommandFnResult, type CommandOpts } from '../../config.ts'
import { specOf } from '../../spec/builtins.ts'
import { rgGeneric } from '../generic/rg.ts'
import { hasSearchShapingFlags, patternArg } from '../grep_helper.ts'
import { formatRecords } from '../utils/output.ts'
const resolveGlob = resolveGlobOf(MONGODB_IO)
@@ -31,6 +43,52 @@ async function rgCommand(
texts: string[],
opts: CommandOpts,
): Promise<CommandFnResult> {
const pattern = patternArg(texts, opts.flags)
const limit = accessor.config.defaultSearchLimit
// The $regex push-down prints each matching document as a whole line, so
// output/match-shaping flags must defer to the generic scan below.
const first = paths[0]
if (
first !== undefined &&
!hasUnresolvedGlob(paths) &&
pattern !== null &&
!pattern.includes('\n') &&
!hasSearchShapingFlags(opts.flags)
) {
const scope = detectScope(first)
if (scope.level !== ScopeLevel.ROOT) {
await mongoStat(accessor, first, opts.index ?? undefined)
}
if (scope.level === ScopeLevel.ROOT) {
const dbs = await listDatabases(accessor)
const results: Awaited<ReturnType<typeof searchDatabase>> = []
for (const db of dbs) {
results.push(...(await searchDatabase(accessor, db, pattern, limit)))
}
const allLines = formatGrepResults(results)
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === ScopeLevel.DATABASE && scope.database !== null) {
const results = await searchDatabase(accessor, scope.database, pattern, limit)
const allLines = formatGrepResults(results)
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === ScopeLevel.ENTITY && scope.database !== null && scope.name !== null) {
const docs = await searchCollection(accessor, scope.database, scope.name, pattern, limit)
if (docs.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
const results = [{ database: scope.database, collection: scope.name, docs }]
const allLines = formatGrepResults(results)
return [formatRecords(allLines), new IOResult()]
}
}
const resolved =
paths.length > 0 ? await resolveGlob(accessor, paths, opts.index ?? undefined) : []
const stat = (p: PathSpec): Promise<FileStat> => mongoStat(accessor, p, opts.index ?? undefined)
typescript/packages/core/src/commands/builtin/postgres/grep.ts
+43 -6
View File
@@ -15,6 +15,7 @@
import type { PostgresAccessor } from '../../../accessor/postgres.ts'
import type { IndexCacheStore } from '../../../cache/index/store.ts'
import { resolveGlobOf } from '../generic_bind/index.ts'
import { hasUnresolvedGlob } from '../utils/operands.ts'
import { POSTGRES_IO } from './io.ts'
import { read as postgresRead } from '../../../core/postgres/read.ts'
import { readdir as postgresReaddir } from '../../../core/postgres/readdir.ts'
@@ -22,9 +23,13 @@ import { detectScope } from '../../../core/postgres/scope.ts'
import {
formatGrepResults,
searchDatabase,
searchDatabaseMetadata,
searchEntity,
searchEntityMetadata,
searchKind,
searchKindMetadata,
searchSchema,
searchSchemaMetadata,
} from '../../../core/postgres/search.ts'
import { stat as postgresStat } from '../../../core/postgres/stat.ts'
import { IOResult } from '../../../io/types.ts'
@@ -32,7 +37,7 @@ import { type FileStat, type PathSpec, ResourceName } from '../../../types.ts'
import { command, type CommandFnResult, type CommandOpts } from '../../config.ts'
import { specOf } from '../../spec/builtins.ts'
import { grepGeneric } from '../generic/grep.ts'
import { patternArg } from '../grep_helper.ts'
import { patternArg, searchPushdownOk } from '../grep_helper.ts'
import { formatRecords } from '../utils/output.ts'
import { searchProvision } from './_provision.ts'
@@ -55,31 +60,48 @@ async function grepCommand(
const pattern = patternArg(texts, opts.flags)
const limit = accessor.config.defaultSearchLimit
// The push-down is a literal-substring search (case-sensitive unless -i)
// that prints
// each matching row as a whole line; it cannot honor output/match-shaping
// flags or a real regex, so those defer to the generic scan below.
const first = paths[0]
if (first !== undefined && pattern !== null && !pattern.includes('\n')) {
const ci = opts.flags.i === true
if (
first !== undefined &&
!hasUnresolvedGlob(paths) &&
pattern !== null &&
searchPushdownOk(opts.flags, pattern)
) {
const scope = detectScope(first)
if (scope.level !== 'root') {
await postgresStat(accessor, first, opts.index ?? undefined)
}
// Directory scopes cover every file under them, so the rendered
// schema.json / semantic.json are searched alongside the row push-down.
// Deliberate divergence from GNU: rows come first and metadata second,
// rather than in per-entity readdir order.
if (scope.level === 'root') {
const results = await searchDatabase(accessor, pattern, limit)
const results = await searchDatabase(accessor, pattern, limit, ci)
const allLines = formatGrepResults(results)
allLines.push(...(await searchDatabaseMetadata(accessor, pattern, ci)))
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === 'schema') {
const results = await searchSchema(accessor, scope.schema, pattern, limit)
const results = await searchSchema(accessor, scope.schema, pattern, limit, ci)
const allLines = formatGrepResults(results)
allLines.push(...(await searchSchemaMetadata(accessor, scope.schema, pattern, ci)))
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === 'kind') {
const results = await searchKind(accessor, scope.schema, scope.kind, pattern, limit)
const results = await searchKind(accessor, scope.schema, scope.kind, pattern, limit, ci)
const allLines = formatGrepResults(results)
allLines.push(...(await searchKindMetadata(accessor, scope.schema, scope.kind, pattern, ci)))
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
@@ -92,10 +114,25 @@ async function grepCommand(
scope.entity,
pattern,
limit,
ci,
)
if (rows.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
const results = [{ schema: scope.schema, kind: scope.kind, entity: scope.entity, rows }]
const allLines = formatGrepResults(results)
// entity_rows names rows.jsonl explicitly; only the directory scope
// pulls in the sibling metadata files.
if (scope.level === 'entity') {
allLines.push(
...(await searchEntityMetadata(
accessor,
scope.schema,
scope.kind,
scope.entity,
pattern,
ci,
)),
)
}
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
}
typescript/packages/core/src/commands/builtin/postgres/pushdown_glob.test.ts
+252
View File
@@ -0,0 +1,252 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock('../../../core/postgres/search.ts', () => ({
searchEntity: vi.fn(),
searchKind: vi.fn(),
searchSchema: vi.fn(),
searchDatabase: vi.fn(),
searchEntityMetadata: vi.fn(() => []),
searchKindMetadata: vi.fn(() => []),
searchSchemaMetadata: vi.fn(() => []),
searchDatabaseMetadata: vi.fn(() => []),
formatGrepResults: vi.fn(() => []),
}))
vi.mock('../../../core/postgres/stat.ts', () => ({ stat: vi.fn() }))
// When the push-down is skipped the wrapper falls through to the generic
// scan, which reads the rendered file; stub the read so those cases exercise
// only the push-down/no-push-down decision, not real row fetching.
vi.mock('../../../core/postgres/read.ts', () => ({
read: vi.fn(() => Promise.resolve(new Uint8Array(0))),
// eslint-disable-next-line @typescript-eslint/require-await
readStream: vi.fn(async function* () {
yield new Uint8Array(0)
}),
}))
import { PostgresAccessor } from '../../../accessor/postgres.ts'
import type { PgDriver, PgQueryResult } from '../../../core/postgres/_driver.ts'
import * as searchModule from '../../../core/postgres/search.ts'
import * as statModule from '../../../core/postgres/stat.ts'
import { resolvePostgresConfig } from '../../../resource/postgres/config.ts'
import { FileStat, FileType, PathSpec } from '../../../types.ts'
import { hasUnresolvedGlob } from '../utils/operands.ts'
import { POSTGRES_COMMANDS } from './index.ts'
const POSTGRES_GREP = POSTGRES_COMMANDS.filter((c) => c.name === 'grep' && c.filetype == null)
const POSTGRES_RG = POSTGRES_COMMANDS.filter((c) => c.name === 'rg' && c.filetype == null)
class StubDriver implements PgDriver {
query<R = Record<string, unknown>>(): Promise<PgQueryResult<R>> {
return Promise.resolve({ rows: [] as R[], rowCount: 0 })
}
close(): Promise<void> {
return Promise.resolve()
}
}
function makeAccessor(): PostgresAccessor {
return new PostgresAccessor(new StubDriver(), resolvePostgresConfig({ dsn: 'postgres://h/db' }))
}
// The dispatcher hands a glob operand through with the trailing segment in
// `pattern` and the wildcard still in `directory`; detectScope would
// otherwise read the "*" as an entity literally named "*".
function globPath(): PathSpec {
return new PathSpec({
virtual: '/public/tables/*/rows.jsonl',
directory: '/public/tables/',
resourcePath: 'public/tables/*/rows.jsonl',
pattern: 'rows.jsonl',
resolved: false,
})
}
function concretePath(): PathSpec {
return new PathSpec({
virtual: '/public/tables/books/rows.jsonl',
directory: '/public/tables/books/',
resourcePath: 'public/tables/books/rows.jsonl',
resolved: true,
})
}
describe('hasUnresolvedGlob', () => {
it('is false for concrete operands', () => {
expect(hasUnresolvedGlob([concretePath()])).toBe(false)
})
it('is false for no operands', () => {
expect(hasUnresolvedGlob([])).toBe(false)
})
it('is true when any operand still carries a pattern', () => {
expect(hasUnresolvedGlob([concretePath(), globPath()])).toBe(true)
})
})
describe('postgres grep push-down and globs', () => {
beforeEach(() => {
vi.mocked(searchModule.searchEntity).mockReset()
vi.mocked(statModule.stat).mockReset()
})
it('skips the SQL push-down when the operand is an unresolved glob', async () => {
const cmd = POSTGRES_GREP[0]
if (cmd === undefined) throw new Error('grep not registered')
// stat must resolve here: glob expansion legitimately stats the paths it
// discovers, so only searchEntity discriminates "push-down ran" from
// "push-down was skipped". Before the fix, detectScope read the "*" as an
// entity and searchEntity was called with it.
vi.mocked(statModule.stat).mockResolvedValue(undefined as never)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
const result = await cmd.fn(makeAccessor(), [globPath()], ['ada'], {
stdin: null,
flags: { l: true },
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(result).not.toBeNull()
expect(vi.mocked(searchModule.searchEntity)).not.toHaveBeenCalled()
})
it('still uses the SQL push-down for a concrete operand', async () => {
const cmd = POSTGRES_GREP[0]
if (cmd === undefined) throw new Error('grep not registered')
vi.mocked(statModule.stat).mockResolvedValue(undefined as never)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
await cmd.fn(makeAccessor(), [concretePath()], ['ada'], {
stdin: null,
flags: {},
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(vi.mocked(searchModule.searchEntity)).toHaveBeenCalledTimes(1)
})
it.each([{ v: true }, { c: true }, { args_l: true }, { n: true }])(
'skips the SQL push-down when a shaping flag is set (%j)',
async (flags) => {
const cmd = POSTGRES_GREP[0]
if (cmd === undefined) throw new Error('grep not registered')
// A shaping flag cannot be honored by the ILIKE push-down, so the
// wrapper must defer to the generic scan; searchEntity must not run.
vi.mocked(statModule.stat).mockResolvedValue(
new FileStat({ name: 'rows.jsonl', type: FileType.TEXT }),
)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
await cmd.fn(makeAccessor(), [concretePath()], ['ada'], {
stdin: null,
flags,
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(vi.mocked(searchModule.searchEntity)).not.toHaveBeenCalled()
},
)
it('skips the SQL push-down for a regex pattern', async () => {
const cmd = POSTGRES_GREP[0]
if (cmd === undefined) throw new Error('grep not registered')
vi.mocked(statModule.stat).mockResolvedValue(
new FileStat({ name: 'rows.jsonl', type: FileType.TEXT }),
)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
await cmd.fn(makeAccessor(), [concretePath()], ['a.b'], {
stdin: null,
flags: {},
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(vi.mocked(searchModule.searchEntity)).not.toHaveBeenCalled()
})
})
describe('postgres rg push-down and globs', () => {
beforeEach(() => {
vi.mocked(searchModule.searchEntity).mockReset()
vi.mocked(statModule.stat).mockReset()
})
it('skips the SQL push-down when the operand is an unresolved glob', async () => {
const cmd = POSTGRES_RG[0]
if (cmd === undefined) throw new Error('rg not registered')
vi.mocked(statModule.stat).mockResolvedValue(
new FileStat({ name: 'rows.jsonl', type: FileType.TEXT }),
)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
await cmd.fn(makeAccessor(), [globPath()], ['ada'], {
stdin: null,
flags: {},
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(vi.mocked(searchModule.searchEntity)).not.toHaveBeenCalled()
})
it('still uses the SQL push-down for a concrete operand', async () => {
const cmd = POSTGRES_RG[0]
if (cmd === undefined) throw new Error('rg not registered')
vi.mocked(statModule.stat).mockResolvedValue(undefined as never)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
await cmd.fn(makeAccessor(), [concretePath()], ['ada'], {
stdin: null,
flags: {},
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(vi.mocked(searchModule.searchEntity)).toHaveBeenCalledTimes(1)
})
it.each([{ v: true }, { c: true }, { args_l: true }, { n: true }])(
'skips the SQL push-down when a shaping flag is set (%j)',
async (flags) => {
const cmd = POSTGRES_RG[0]
if (cmd === undefined) throw new Error('rg not registered')
vi.mocked(statModule.stat).mockResolvedValue(
new FileStat({ name: 'rows.jsonl', type: FileType.TEXT }),
)
vi.mocked(searchModule.searchEntity).mockResolvedValue([])
await cmd.fn(makeAccessor(), [concretePath()], ['ada'], {
stdin: null,
flags,
filetypeFns: null,
cwd: '/',
resource: { kind: 'postgres' } as never,
})
expect(vi.mocked(searchModule.searchEntity)).not.toHaveBeenCalled()
},
)
})
typescript/packages/core/src/commands/builtin/postgres/rg.ts
+95
View File
@@ -14,14 +14,30 @@
import type { PostgresAccessor } from '../../../accessor/postgres.ts'
import { resolveGlobOf } from '../generic_bind/index.ts'
import { hasUnresolvedGlob } from '../utils/operands.ts'
import { POSTGRES_IO } from './io.ts'
import { readStream } from '../../../core/postgres/read.ts'
import { readdir as postgresReaddir } from '../../../core/postgres/readdir.ts'
import { detectScope } from '../../../core/postgres/scope.ts'
import {
formatGrepResults,
searchDatabase,
searchDatabaseMetadata,
searchEntity,
searchEntityMetadata,
searchKind,
searchKindMetadata,
searchSchema,
searchSchemaMetadata,
} from '../../../core/postgres/search.ts'
import { stat as postgresStat } from '../../../core/postgres/stat.ts'
import { IOResult } from '../../../io/types.ts'
import { type FileStat, ResourceName, type PathSpec } from '../../../types.ts'
import { command, type CommandFnResult, type CommandOpts } from '../../config.ts'
import { specOf } from '../../spec/builtins.ts'
import { rgGeneric } from '../generic/rg.ts'
import { patternArg, searchPushdownOk } from '../grep_helper.ts'
import { formatRecords } from '../utils/output.ts'
const resolveGlob = resolveGlobOf(POSTGRES_IO)
@@ -31,6 +47,85 @@ async function rgCommand(
texts: string[],
opts: CommandOpts,
): Promise<CommandFnResult> {
const pattern = patternArg(texts, opts.flags)
const limit = accessor.config.defaultSearchLimit
// Native search takes one literal pattern and prints each matching row as a
// whole line; a multi -e set (#347), a real regex, or any match/output
// shaping flag must fall through to the generic scan below.
const first = paths[0]
const ci = opts.flags.i === true
if (
first !== undefined &&
!hasUnresolvedGlob(paths) &&
pattern !== null &&
searchPushdownOk(opts.flags, pattern)
) {
const scope = detectScope(first)
if (scope.level !== 'root') {
await postgresStat(accessor, first, opts.index ?? undefined)
}
// Directory scopes cover every file under them, so the rendered
// schema.json / semantic.json are searched alongside the row push-down.
// Deliberate divergence from GNU: rows come first and metadata second,
// rather than in per-entity readdir order.
if (scope.level === 'root') {
const results = await searchDatabase(accessor, pattern, limit, ci)
const allLines = formatGrepResults(results)
allLines.push(...(await searchDatabaseMetadata(accessor, pattern, ci)))
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === 'schema') {
const results = await searchSchema(accessor, scope.schema, pattern, limit, ci)
const allLines = formatGrepResults(results)
allLines.push(...(await searchSchemaMetadata(accessor, scope.schema, pattern, ci)))
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === 'kind') {
const results = await searchKind(accessor, scope.schema, scope.kind, pattern, limit, ci)
const allLines = formatGrepResults(results)
allLines.push(...(await searchKindMetadata(accessor, scope.schema, scope.kind, pattern, ci)))
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
if (scope.level === 'entity' || scope.level === 'entity_rows') {
const rows = await searchEntity(
accessor,
scope.schema,
scope.kind,
scope.entity,
pattern,
limit,
ci,
)
const results = [{ schema: scope.schema, kind: scope.kind, entity: scope.entity, rows }]
const allLines = formatGrepResults(results)
// entity_rows names rows.jsonl explicitly; only the directory scope
// pulls in the sibling metadata files.
if (scope.level === 'entity') {
allLines.push(
...(await searchEntityMetadata(
accessor,
scope.schema,
scope.kind,
scope.entity,
pattern,
ci,
)),
)
}
if (allLines.length === 0) return [new Uint8Array(0), new IOResult({ exitCode: 1 })]
return [formatRecords(allLines), new IOResult()]
}
}
const resolved =
paths.length > 0 ? await resolveGlob(accessor, paths, opts.index ?? undefined) : []
const stat = (p: PathSpec): Promise<FileStat> =>
typescript/packages/core/src/commands/builtin/utils/operands.ts
+8
View File
@@ -23,6 +23,14 @@ const ENC = new TextEncoder()
type Stat = (p: PathSpec) => Promise<FileStat>
// True when any operand still carries a glob to expand. Backend push-down
// branches read paths[0] directly to build SQL, so they must not run before
// glob expansion: a pattern segment would be taken for a literal entity
// name, and tables/*/rows.jsonl would query a relation actually called "*".
export function hasUnresolvedGlob(paths: PathSpec[]): boolean {
return paths.some((p) => p.pattern !== null && p.pattern !== '')
}
// Resolve a script operand (absolute or cwd-relative) to a fully-resolved
// PathSpec, the way python3/js locate a mounted script before running it.
export function resolveScript(name: string, cwd: string): PathSpec {
typescript/packages/core/src/core/postgres/_client.test.ts
+28
View File
@@ -33,6 +33,34 @@ function makeAccessor(
return { accessor: new PostgresAccessor(driver, cfg), query }
}
describe('quoteIdent', () => {
it('escapes embedded quotes', () => {
expect(_client.quoteIdent('users')).toBe('"users"')
expect(_client.quoteIdent('a"b')).toBe('"a""b"')
})
it('neutralizes an injection payload', () => {
const payload = 'books" CROSS JOIN secret AS "s'
const quoted = _client.quoteIdent(payload)
// Every embedded quote is doubled, so the payload cannot terminate the
// identifier and become executable SQL: it stays one (nonexistent) name.
expect(quoted).toBe('"books"" CROSS JOIN secret AS ""s"')
// After stripping the outer wrapper and collapsing every doubled-quote
// pair, no lone quote survives to break out of the identifier.
expect(quoted.slice(1, -1).replace(/""/g, '')).not.toContain('"')
})
})
describe('countRows', () => {
it('quotes a malicious relation name instead of interpolating it raw', async () => {
const { accessor, query } = makeAccessor([{ count: 0 }])
await _client.countRows(accessor, 'public', 'books" CROSS JOIN secret AS "s')
const sql = query.mock.calls[0]?.[0] as string
expect(sql).toContain('"books"" CROSS JOIN secret AS ""s"')
expect(sql).not.toContain('FROM "public"."books" CROSS JOIN secret')
})
})
describe('listSchemas', () => {
it('filters system schemas via SQL and returns names', async () => {
const { accessor, query } = makeAccessor([
typescript/packages/core/src/core/postgres/_client.ts
+103
View File
@@ -37,6 +37,16 @@ export interface Relationship {
kind: 'many_to_one'
}
export interface EnumInfo {
type: string
labels: string[]
}
export interface ColumnStats {
n_distinct: number
most_common_vals: string[]
}
export function quoteIdent(ident: string): string {
return `"${ident.replace(/"/g, '""')}"`
}
@@ -344,3 +354,96 @@ export async function fetchAllRelationships(
}
return [...grouped.values()]
}
export async function fetchTableComment(
accessor: PostgresAccessor,
schema: string,
name: string,
): Promise<string | null> {
const result = await accessor.store.query<{ obj_description: string | null }>(
"SELECT obj_description(c.oid, 'pg_class') " +
'FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace ' +
'WHERE n.nspname = $1 AND c.relname = $2',
[schema, name],
)
return result.rows[0]?.obj_description ?? null
}
export async function fetchColumnComments(
accessor: PostgresAccessor,
schema: string,
name: string,
): Promise<Map<string, string>> {
const result = await accessor.store.query<{ attname: string; comment: string | null }>(
'SELECT a.attname, col_description(c.oid, a.attnum) AS comment ' +
'FROM pg_class c ' +
'JOIN pg_namespace n ON n.oid = c.relnamespace ' +
'JOIN pg_attribute a ' +
' ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped ' +
'WHERE n.nspname = $1 AND c.relname = $2 ' +
'ORDER BY a.attnum',
[schema, name],
)
const out = new Map<string, string>()
for (const r of result.rows) {
if (r.comment) out.set(r.attname, r.comment)
}
return out
}
export async function fetchEnumColumns(
accessor: PostgresAccessor,
schema: string,
name: string,
): Promise<Map<string, EnumInfo>> {
const result = await accessor.store.query<{
attname: string
typname: string
labels: string[]
}>(
'SELECT a.attname, t.typname, ' +
' array_agg(e.enumlabel ORDER BY e.enumsortorder)::text[] AS labels ' +
'FROM pg_class c ' +
'JOIN pg_namespace n ON n.oid = c.relnamespace ' +
'JOIN pg_attribute a ' +
' ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped ' +
'JOIN pg_type t ON t.oid = a.atttypid ' +
'JOIN pg_enum e ON e.enumtypid = t.oid ' +
'WHERE n.nspname = $1 AND c.relname = $2 ' +
'GROUP BY a.attname, t.typname',
[schema, name],
)
const out = new Map<string, EnumInfo>()
for (const r of result.rows) {
out.set(r.attname, { type: r.typname, labels: [...r.labels] })
}
return out
}
export async function fetchColumnStats(
accessor: PostgresAccessor,
schema: string,
name: string,
): Promise<Map<string, ColumnStats>> {
// pg_stats is populated by ANALYZE, so it is empty for a freshly written
// relation until autovacuum gets to it. Callers treat it as best-effort;
// mirage never runs ANALYZE itself (a write and a cost on the user's DB).
const result = await accessor.store.query<{
attname: string
n_distinct: number | string
mcv: string[] | null
}>(
'SELECT attname, n_distinct, ' +
' most_common_vals::text::text[] AS mcv ' +
'FROM pg_stats WHERE schemaname = $1 AND tablename = $2',
[schema, name],
)
const out = new Map<string, ColumnStats>()
for (const r of result.rows) {
out.set(r.attname, {
n_distinct: Number(r.n_distinct),
most_common_vals: r.mcv ? [...r.mcv] : [],
})
}
return out
}
typescript/packages/core/src/core/postgres/read.ts
+6
View File
@@ -19,6 +19,7 @@ import { encodeBase64 } from '../../utils/base64.ts'
import type { PostgresAccessor } from '../../accessor/postgres.ts'
import { estimateSize, fetchRows } from './_client.ts'
import { buildDatabaseJson, buildEntitySchemaJson } from './_schema_json.ts'
import { buildEntitySemanticJson } from './semantic.ts'
import { detectScope } from './scope.ts'
import { enoent } from '../../utils/errors.ts'
@@ -61,6 +62,11 @@ export async function read(
const doc = await buildEntitySchemaJson(accessor, scope.schema, scope.entity, kind)
return new TextEncoder().encode(JSON.stringify(doc, null, 2))
}
if (scope.level === 'entity_semantic') {
const kind = scope.kind === 'tables' ? 'table' : 'view'
const doc = await buildEntitySemanticJson(accessor, scope.schema, scope.entity, kind)
return new TextEncoder().encode(JSON.stringify(doc, null, 2))
}
if (scope.level === 'entity_rows') {
return readRows(accessor, scope.schema, scope.kind, scope.entity, options)
}
typescript/packages/core/src/core/postgres/readdir.test.ts
+2 -1
View File
@@ -92,7 +92,7 @@ describe('readdir', () => {
expect(out).toEqual(['/pg/public/views/a_mview', '/pg/public/views/z_view'])
})
it('lists entity: schema.json + rows.jsonl', async () => {
it('lists entity: schema.json + semantic.json + rows.jsonl', async () => {
const out = await readdir(
makeAccessor(),
new PathSpec({
@@ -103,6 +103,7 @@ describe('readdir', () => {
)
expect(out).toEqual([
'/pg/public/tables/users/schema.json',
'/pg/public/tables/users/semantic.json',
'/pg/public/tables/users/rows.jsonl',
])
})
typescript/packages/core/src/core/postgres/readdir.ts
+2 -2
View File
@@ -18,7 +18,7 @@ import type { IndexCacheStore } from '../../cache/index/store.ts'
import { PathSpec } from '../../types.ts'
import type { PostgresAccessor } from '../../accessor/postgres.ts'
import { listMatviews, listSchemas, listTables, listViews } from './_client.ts'
import { detectScope } from './scope.ts'
import { detectScope, ENTITY_FILES } from './scope.ts'
import { rstripSlash } from '../../utils/slash.ts'
export async function readdir(
@@ -51,7 +51,7 @@ export async function readdir(
}
if (scope.level === 'entity') {
const base = rstripSlash(raw)
return [`${prefix}${base}/schema.json`, `${prefix}${base}/rows.jsonl`]
return ENTITY_FILES.map((name) => `${prefix}${base}/${name}`)
}
const err = new Error(raw) as Error & { code?: string }
err.code = 'ENOENT'
typescript/packages/core/src/core/postgres/scope.test.ts
+10
View File
@@ -97,6 +97,16 @@ describe('detectScope', () => {
expect(s.file).toBe('schema.json')
})
it('detects entity_semantic file', () => {
const s = detectScope(ps('/public/tables/users/semantic.json'))
expect(s.level).toBe('entity_semantic')
if (s.level !== 'entity_semantic') return
expect(s.schema).toBe('public')
expect(s.kind).toBe('tables')
expect(s.entity).toBe('users')
expect(s.file).toBe('semantic.json')
})
it('detects entity_rows file', () => {
const s = detectScope(ps('/public/tables/users/rows.jsonl'))
expect(s.level).toBe('entity_rows')
typescript/packages/core/src/core/postgres/scope.ts
+20
View File
@@ -17,6 +17,8 @@ import { stripSlash } from '../../utils/slash.ts'
type EntityKind = 'tables' | 'views'
export const ENTITY_FILES = ['schema.json', 'semantic.json', 'rows.jsonl'] as const
export type PostgresScope =
| { level: 'root'; resourcePath: string }
| { level: 'database_json'; file: 'database.json'; resourcePath: string }
@@ -37,6 +39,14 @@ export type PostgresScope =
file: 'schema.json'
resourcePath: string
}
| {
level: 'entity_semantic'
schema: string
kind: EntityKind
entity: string
file: 'semantic.json'
resourcePath: string
}
| {
level: 'entity_rows'
schema: string
@@ -100,6 +110,16 @@ export function detectScope(path: PathSpec | string): PostgresScope {
resourcePath: raw,
}
}
if (file === 'semantic.json') {
return {
level: 'entity_semantic',
schema,
kind,
entity,
file: 'semantic.json',
resourcePath: raw,
}
}
if (file === 'rows.jsonl') {
return {
level: 'entity_rows',

Some files were not shown because too many files have changed in this diff Show More