6bf8bebf51
CI / Test and Build (push) Failing after 1s
CI / Migrate Dev DB (push) Has been skipped
CI / Migrate DB (push) Has been skipped
CodeQL / Analyze actions (push) Has been cancelled
CodeQL / Analyze javascript-typescript (push) Has been cancelled
CI / Detect Version (push) Has been cancelled
CI / Detect Desktop Changes (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/cron.Dockerfile, ubuntu-latest, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build AMD64 (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/cron.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/db.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/pii.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/realtime.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-8vcpu-ubuntu-2404-arm, ./docker/app.Dockerfile, linux-arm64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
Helm Chart / Lint, test, and validate chart (push) Has been cancelled
Helm Chart / Chart version bumped (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Build Dev ECR (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core) (push) Has been cancelled
CI / Promote Images (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Check Desktop Signing Secrets (push) Has been cancelled
CI / Desktop Release (push) Has been cancelled
CI / Create Desktop Prerelease (push) Has been cancelled
CI / Desktop Prerelease Build (push) Has been cancelled
CI / Publish Desktop Prerelease (push) Has been cancelled
CI / Prune Desktop Prereleases (push) Has been cancelled
Helm Chart / Install on kind and run helm test (push) Has been cancelled
709 lines
28 KiB
TypeScript
709 lines
28 KiB
TypeScript
import { createLogger } from '@sim/logger'
|
|
import { omit } from '@sim/utils/object'
|
|
import { hasWorkspaceSandboxAccess } from '@/lib/billing/core/subscription'
|
|
import { applySecretMountPolicy } from '@/lib/copilot/secret-mount-policy'
|
|
import type { ToolExecutionContext, ToolExecutionResult } from '@/lib/copilot/tool-executor/types'
|
|
import {
|
|
CopilotCodeSecretAccessError,
|
|
type MaterializedCopilotCodeSecrets,
|
|
materializeCopilotCodeSecrets,
|
|
} from '@/lib/copilot/tools/secret-mount-materializer.server'
|
|
import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils'
|
|
import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
|
|
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
|
|
import type { PrivateSecretProvenanceBundleV1 } from '@/lib/execution/model-input-provenance'
|
|
import {
|
|
MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY,
|
|
PRIVATE_SECRET_PROVENANCE_FIELD,
|
|
} from '@/lib/execution/private-tool-metadata'
|
|
import { MAX_PLAN_REQUIRED } from '@/lib/execution/remote-sandbox/workspace-sandboxes'
|
|
import { getColumnId } from '@/lib/table/column-keys'
|
|
import { TABLE_LIMITS } from '@/lib/table/constants'
|
|
import { formatCsvCell, neutralizeCsvFormula, toCsvRow } from '@/lib/table/export-format'
|
|
import {
|
|
isTableSnapshotSafeForModelMount,
|
|
loadTableRowSecretProvenance,
|
|
} from '@/lib/table/rows/secret-provenance'
|
|
import { queryRows } from '@/lib/table/rows/service'
|
|
import { getTableById, listTables } from '@/lib/table/service'
|
|
import { getOrCreateTableSnapshot, SNAPSHOT_MAX_BYTES } from '@/lib/table/snapshot-cache'
|
|
import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
|
|
import {
|
|
fetchServableWorkspaceFileBuffer,
|
|
fetchWorkspaceFileBuffer,
|
|
findWorkspaceFileRecord,
|
|
getSandboxWorkspaceFilePath,
|
|
listWorkspaceFiles,
|
|
type WorkspaceFileRecord,
|
|
} from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
|
import { importWorkspaceFileSecretProvenanceForRuntime } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
|
|
import {
|
|
downloadFile,
|
|
generatePresignedDownloadUrl,
|
|
hasCloudStorage,
|
|
} from '@/lib/uploads/core/storage-service'
|
|
import { isGeneratedDocumentSourceType } from '@/lib/uploads/utils/file-utils'
|
|
import { extractCodeSecretNames } from '@/executor/utils/code-secret-references'
|
|
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
|
|
import { executeTool as executeAppTool } from '@/tools'
|
|
|
|
const logger = createLogger('CopilotFunctionExecute')
|
|
|
|
const MAX_FILE_SIZE = 10 * 1024 * 1024
|
|
const MAX_TOTAL_SIZE = 50 * 1024 * 1024
|
|
const MAX_MOUNTED_FILES = 500
|
|
|
|
/**
|
|
* Below this row count a table mounts via the direct inline CSV path — the version-keyed snapshot
|
|
* cache (storage round-trip) only pays off for larger/hot tables. Behind the feature flag either
|
|
* way; this just keeps tiny one-shot tables on the cheaper path.
|
|
*/
|
|
const SNAPSHOT_MIN_ROWS = 500
|
|
|
|
/**
|
|
* Lifetime of a presigned URL handed to the sandbox to fetch a mounted object (table snapshot or
|
|
* workspace file). Long enough to download a large file at sandbox startup; the URL grants read to
|
|
* only that one object.
|
|
*/
|
|
const MOUNT_URL_TTL_SECONDS = 600
|
|
|
|
/**
|
|
* Per-file ceiling for URL-mounted workspace files. The bytes never transit the web process — the
|
|
* sandbox curls them straight from storage — so the bound is sandbox disk, not web heap (unlike the
|
|
* inline MAX_FILE_SIZE path).
|
|
*/
|
|
const MOUNT_URL_MAX_BYTES = 500 * 1024 * 1024
|
|
|
|
/**
|
|
* Aggregate ceiling across all URL-mounted files in one request. URL mounts bypass the web heap (so
|
|
* they don't count against MAX_TOTAL_SIZE), but the sandbox still curls every byte onto its disk —
|
|
* this rejects an oversized request up front instead of filling the sandbox disk one slow curl at a
|
|
* time. Generous vs MAX_TOTAL_SIZE since the bytes never transit web memory.
|
|
*/
|
|
const MAX_TOTAL_URL_BYTES = 2 * 1024 * 1024 * 1024
|
|
|
|
type SandboxFile =
|
|
| { type?: 'content'; path: string; content: string; encoding?: 'base64' }
|
|
| { type: 'url'; path: string; url: string }
|
|
|
|
/**
|
|
* Running byte totals for one resolveInputFiles call. `buffered` bytes pass through the web process
|
|
* (capped by MAX_TOTAL_SIZE); `url` bytes are curled straight into the sandbox (capped by
|
|
* MAX_TOTAL_URL_BYTES). Tracked separately because the two ceilings protect different resources —
|
|
* web heap vs sandbox disk.
|
|
*/
|
|
interface MountedBytes {
|
|
buffered: number
|
|
url: number
|
|
}
|
|
|
|
async function importMountedWorkspaceFileProvenance(args: {
|
|
workspaceId: string
|
|
record: WorkspaceFileRecord
|
|
mountPath: string
|
|
registry?: ResolvedSecretTraceRegistry
|
|
}): Promise<void> {
|
|
if (!args.registry) {
|
|
throw new Error(
|
|
`Input file "${args.mountPath}" cannot be mounted because its secret provenance is unavailable.`
|
|
)
|
|
}
|
|
try {
|
|
const imported = await importWorkspaceFileSecretProvenanceForRuntime({
|
|
workspaceId: args.workspaceId,
|
|
identity: {
|
|
fileId: args.record.id,
|
|
key: args.record.key,
|
|
context: args.record.storageContext ?? 'workspace',
|
|
},
|
|
registry: args.registry,
|
|
})
|
|
if (!imported) args.registry.markIncomplete()
|
|
} catch {
|
|
args.registry.markIncomplete()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Mounts a stored workspace file into the sandbox and records its bytes against the running totals.
|
|
* With cloud storage the sandbox fetches the bytes itself from a presigned URL (no web-heap transit,
|
|
* per-file ceiling MOUNT_URL_MAX_BYTES, aggregate ceiling MAX_TOTAL_URL_BYTES); with local storage a
|
|
* presigned URL is an app-internal serve path a remote sandbox can't reach, so we buffer the bytes
|
|
* through the web process under the inline MAX_FILE_SIZE / MAX_TOTAL_SIZE guards.
|
|
*/
|
|
async function pushWorkspaceFileMount(
|
|
sandboxFiles: SandboxFile[],
|
|
record: WorkspaceFileRecord,
|
|
mountPath: string,
|
|
mounted: MountedBytes,
|
|
workspaceId: string,
|
|
registry?: ResolvedSecretTraceRegistry
|
|
): Promise<void> {
|
|
await importMountedWorkspaceFileProvenance({ workspaceId, record, mountPath, registry })
|
|
|
|
// A generated document stores its generator source, so a presigned URL for
|
|
// `record.key` would hand the sandbox source text under a `.docx` name and the
|
|
// user's script would fail on a file that looks fine. Those resolve through the
|
|
// servable reader instead — they are bounded by the render ceiling, so routing them
|
|
// through the web process rather than presigning is affordable.
|
|
const rendersFromSource = isGeneratedDocumentSourceType(record.type)
|
|
|
|
if (hasCloudStorage() && !rendersFromSource) {
|
|
if (record.size > MOUNT_URL_MAX_BYTES) {
|
|
throw new Error(
|
|
`Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MOUNT_URL_MAX_BYTES / 1024 / 1024}MB per-file mount limit.`
|
|
)
|
|
}
|
|
if (mounted.url + record.size > MAX_TOTAL_URL_BYTES) {
|
|
throw new Error(
|
|
`Mounting "${mountPath}" would exceed the ${MAX_TOTAL_URL_BYTES / 1024 / 1024 / 1024}GB total mount limit. Mount fewer or smaller files.`
|
|
)
|
|
}
|
|
const url = await generatePresignedDownloadUrl(
|
|
record.key,
|
|
record.storageContext ?? 'workspace',
|
|
MOUNT_URL_TTL_SECONDS
|
|
)
|
|
sandboxFiles.push({ type: 'url', path: mountPath, url })
|
|
mounted.url += record.size
|
|
return
|
|
}
|
|
|
|
const remainingBudget = Math.max(0, MAX_TOTAL_SIZE - mounted.buffered)
|
|
|
|
// A source-backed document declares the size of its generator, not of the document,
|
|
// so these pre-checks say nothing about what is about to be mounted. Its read is
|
|
// capped instead, and the real length is checked once it is known.
|
|
if (!rendersFromSource) {
|
|
if (record.size > MAX_FILE_SIZE) {
|
|
throw new Error(
|
|
`Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.`
|
|
)
|
|
}
|
|
if (record.size > remainingBudget) {
|
|
throw new Error(
|
|
`Mounting "${mountPath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.`
|
|
)
|
|
}
|
|
}
|
|
|
|
const { buffer, contentType } = rendersFromSource
|
|
? await fetchServableWorkspaceFileBuffer(record, {
|
|
maxBytes: Math.min(MAX_FILE_SIZE, remainingBudget),
|
|
}).catch((error) => {
|
|
if (!isPayloadSizeLimitError(error)) throw error
|
|
throw new Error(
|
|
`Input file "${mountPath}" renders to more than the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit, or than the mount budget left. Mount fewer or smaller files.`
|
|
)
|
|
})
|
|
: { buffer: await fetchWorkspaceFileBuffer(record), contentType: record.type }
|
|
// Keyed off the resolved type: a rendered document's source MIME is `text/x-…`, and
|
|
// decoding the binary as UTF-8 would corrupt it just as surely as shipping the source.
|
|
const isText = /^text\/|application\/json|application\/xml|application\/csv/.test(
|
|
contentType || ''
|
|
)
|
|
sandboxFiles.push({
|
|
path: mountPath,
|
|
content: isText ? buffer.toString('utf-8') : buffer.toString('base64'),
|
|
encoding: isText ? undefined : 'base64',
|
|
})
|
|
mounted.buffered += buffer.length
|
|
}
|
|
|
|
/**
|
|
* Explains why a VFS path the agent legitimately discovered cannot be mounted, and
|
|
* what to do instead. Only workspace `files/` are backed by storage the sandbox can
|
|
* fetch from — `internal/` is served by the copilot backend and its bytes never reach
|
|
* Sim, `uploads/` is chat-scoped, `recently-deleted/` is archived, and the remaining
|
|
* namespaces are metadata views rather than stored file bytes. Returns null for
|
|
* `files/` references, where "not found" is the honest answer.
|
|
*
|
|
* These paths are correct and are advertised to the model as read/grep-able, so the
|
|
* generic not-found message ("copy the exact canonical path") is actively wrong for
|
|
* them: it sends the agent hunting for a path that does not exist.
|
|
*/
|
|
function unmountableNamespaceReason(filePath: string): string | null {
|
|
// Trailing slash so a bare namespace passed as a directory matches the same prefixes
|
|
// as a file path inside it.
|
|
const path = `${filePath.replace(/^\/+|\/+$/g, '')}/`
|
|
|
|
if (path.startsWith('uploads/')) {
|
|
return 'uploads/ files are not mountable into the sandbox. Use materialize_file to save it to a files/... path first, then mount that canonical path.'
|
|
}
|
|
if (path.startsWith('internal/tool-results/')) {
|
|
return 'tool-result artifacts are stored by the copilot backend, not in workspace storage, so read and grep reach them but the sandbox cannot. This path is correct — searching for a different one will not find anything. Either read or grep the artifact and inline the values you need in code, or re-run the tool that produced it with an output path under files/ (function_execute: outputs.files[].path, user_table: outputPath) and mount that files/... path.'
|
|
}
|
|
if (path.startsWith('internal/')) {
|
|
return 'internal/ paths are served by the copilot backend, not from workspace storage, so read and grep reach them but the sandbox cannot. This path is correct — read or grep it and inline the values you need in code instead of mounting it.'
|
|
}
|
|
if (path.startsWith('recently-deleted/')) {
|
|
return 'deleted resources are not mountable into the sandbox. Use restore_resource to restore it first, then mount the restored files/... path.'
|
|
}
|
|
if (path.startsWith('tables/')) {
|
|
return 'tables are not mounted as files. Pass the table in inputs.tables instead and it is mounted as CSV.'
|
|
}
|
|
const namespace = /^(workflows|knowledgebases|components|environment|agent)\//.exec(path)?.[1]
|
|
if (namespace) {
|
|
return `${namespace}/ paths are VFS metadata views, not stored file bytes, so the sandbox cannot mount them. This path is correct — read or grep it and inline the values you need in code.`
|
|
}
|
|
return null
|
|
}
|
|
|
|
interface CanonicalFileInput {
|
|
path: string
|
|
sandboxPath?: string
|
|
}
|
|
|
|
interface CanonicalDirectoryInput {
|
|
path: string
|
|
sandboxPath?: string
|
|
}
|
|
|
|
interface CanonicalTableInput {
|
|
tableId?: string
|
|
path?: string
|
|
sandboxPath?: string
|
|
}
|
|
|
|
function tableNameFromVfsPath(tableRef: string): string | null {
|
|
if (!tableRef.startsWith('tables/')) return null
|
|
const segments = decodeVfsPathSegments(tableRef)
|
|
const metaIndex = segments.lastIndexOf('meta.json')
|
|
return segments[metaIndex > 0 ? metaIndex - 1 : segments.length - 1] ?? null
|
|
}
|
|
|
|
async function resolveTableRef(
|
|
tableRef: string,
|
|
tablePathLookup?: Map<string, Awaited<ReturnType<typeof listTables>>[number]>
|
|
) {
|
|
if (!tableRef.startsWith('tables/')) {
|
|
return getTableById(tableRef)
|
|
}
|
|
|
|
const tableName = tableNameFromVfsPath(tableRef)
|
|
if (!tableName) return null
|
|
return tablePathLookup?.get(tableName) ?? null
|
|
}
|
|
|
|
export async function resolveInputFiles(
|
|
workspaceId: string,
|
|
inputFiles?: unknown[],
|
|
inputTables?: unknown[],
|
|
inputDirectories?: unknown[],
|
|
provenanceUserId?: string,
|
|
resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry
|
|
): Promise<SandboxFile[]> {
|
|
const sandboxFiles: SandboxFile[] = []
|
|
const mounted: MountedBytes = { buffered: 0, url: 0 }
|
|
|
|
if (inputFiles?.length && workspaceId) {
|
|
if (inputFiles.length > MAX_MOUNTED_FILES) {
|
|
throw new Error(
|
|
`Too many input files (${inputFiles.length}). Maximum is ${MAX_MOUNTED_FILES}. Mount fewer files.`
|
|
)
|
|
}
|
|
const allFiles = await listWorkspaceFiles(workspaceId)
|
|
for (const fileRef of inputFiles) {
|
|
const filePath =
|
|
typeof fileRef === 'string'
|
|
? fileRef
|
|
: fileRef && typeof fileRef === 'object'
|
|
? (fileRef as CanonicalFileInput).path
|
|
: undefined
|
|
if (!filePath) continue
|
|
const record = findWorkspaceFileRecord(allFiles, filePath)
|
|
if (!record) {
|
|
const unmountable = unmountableNamespaceReason(filePath)
|
|
if (unmountable) {
|
|
throw new Error(`Cannot mount "${filePath}": ${unmountable}`)
|
|
}
|
|
throw new Error(
|
|
`Input file not found: "${filePath}". Pass the exact canonical VFS path copied from glob/read (e.g. "files/Reports/data.csv").`
|
|
)
|
|
}
|
|
const explicitSandboxPath =
|
|
typeof fileRef === 'object' && fileRef !== null
|
|
? (fileRef as CanonicalFileInput).sandboxPath
|
|
: undefined
|
|
const mountPath = explicitSandboxPath || getSandboxWorkspaceFilePath(record)
|
|
await pushWorkspaceFileMount(
|
|
sandboxFiles,
|
|
record,
|
|
mountPath,
|
|
mounted,
|
|
workspaceId,
|
|
resolvedSecretTraceRegistry
|
|
)
|
|
}
|
|
}
|
|
|
|
if (inputDirectories?.length && workspaceId) {
|
|
const folders = await listWorkspaceFileFolders(workspaceId)
|
|
const allFiles = await listWorkspaceFiles(workspaceId, { folders })
|
|
for (const dirRef of inputDirectories) {
|
|
const dirPath =
|
|
typeof dirRef === 'string'
|
|
? dirRef
|
|
: dirRef && typeof dirRef === 'object'
|
|
? (dirRef as CanonicalDirectoryInput).path
|
|
: undefined
|
|
if (!dirPath) continue
|
|
const folderSegments = decodeVfsPathSegments(dirPath.replace(/^\/?files\/?/, ''))
|
|
const folderDisplayPath = folderSegments.join('/')
|
|
const folder = folders.find((candidate) => candidate.path === folderDisplayPath)
|
|
if (!folder) {
|
|
const unmountable = unmountableNamespaceReason(dirPath)
|
|
throw new Error(
|
|
unmountable
|
|
? `Cannot mount "${dirPath}": ${unmountable}`
|
|
: `Input directory not found: "${dirPath}". Pass a canonical workspace folder path copied from glob/read (e.g. "files/Reports").`
|
|
)
|
|
}
|
|
const mountRoot =
|
|
typeof dirRef === 'object' &&
|
|
dirRef !== null &&
|
|
(dirRef as CanonicalDirectoryInput).sandboxPath
|
|
? (dirRef as CanonicalDirectoryInput).sandboxPath!
|
|
: `/home/user/files/${encodeVfsPathSegments(folder.path.split('/'))}`
|
|
const descendants = allFiles.filter((file) => {
|
|
if (!file.folderPath) return false
|
|
return file.folderPath === folder.path || file.folderPath.startsWith(`${folder.path}/`)
|
|
})
|
|
if (descendants.length > MAX_MOUNTED_FILES) {
|
|
throw new Error(
|
|
`Input directory contains too many files (${descendants.length}). Maximum is ${MAX_MOUNTED_FILES}. Mount a smaller directory or individual files.`
|
|
)
|
|
}
|
|
logger.info('Mounting workspace directory for function_execute', {
|
|
vfsPath: dirPath,
|
|
sandboxPath: mountRoot,
|
|
fileCount: descendants.length,
|
|
})
|
|
const childFolders = folders.filter(
|
|
(candidate) =>
|
|
candidate.path !== folder.path && candidate.path.startsWith(`${folder.path}/`)
|
|
)
|
|
if (descendants.length === 0 && childFolders.length === 0) {
|
|
sandboxFiles.push({ path: `${mountRoot}/.keep`, content: '' })
|
|
continue
|
|
}
|
|
for (const childFolder of childFolders) {
|
|
const hasFiles = descendants.some((file) => {
|
|
if (!file.folderPath) return false
|
|
return (
|
|
file.folderPath === childFolder.path ||
|
|
file.folderPath.startsWith(`${childFolder.path}/`)
|
|
)
|
|
})
|
|
if (!hasFiles) {
|
|
const relativeFolder = childFolder.path.slice(folder.path.length).replace(/^\/+/, '')
|
|
sandboxFiles.push({ path: `${mountRoot}/${relativeFolder}/.keep`, content: '' })
|
|
}
|
|
}
|
|
for (const record of descendants) {
|
|
const relativeFolder =
|
|
record.folderPath?.slice(folder.path.length).replace(/^\/+/, '') ?? ''
|
|
const relativePath = [relativeFolder, record.name].filter(Boolean).join('/')
|
|
await pushWorkspaceFileMount(
|
|
sandboxFiles,
|
|
record,
|
|
`${mountRoot}/${relativePath}`,
|
|
mounted,
|
|
workspaceId,
|
|
resolvedSecretTraceRegistry
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
if (inputTables?.length) {
|
|
const hasTablePathRefs = inputTables.some((tableRef) => {
|
|
const tableId =
|
|
typeof tableRef === 'string'
|
|
? tableRef
|
|
: tableRef && typeof tableRef === 'object'
|
|
? (tableRef as CanonicalTableInput).tableId || (tableRef as CanonicalTableInput).path
|
|
: undefined
|
|
return typeof tableId === 'string' && tableId.startsWith('tables/')
|
|
})
|
|
const tablePathLookup = hasTablePathRefs
|
|
? new Map((await listTables(workspaceId)).map((table) => [table.name, table]))
|
|
: undefined
|
|
const snapshotCacheEnabled = await isFeatureEnabled('table-snapshot-cache')
|
|
for (const tableRef of inputTables) {
|
|
const tableId =
|
|
typeof tableRef === 'string'
|
|
? tableRef
|
|
: tableRef && typeof tableRef === 'object'
|
|
? (tableRef as CanonicalTableInput).tableId || (tableRef as CanonicalTableInput).path
|
|
: undefined
|
|
if (!tableId) continue
|
|
const table = await resolveTableRef(tableId, tablePathLookup)
|
|
if (!table || table.workspaceId !== workspaceId) {
|
|
throw new Error(
|
|
`Input table not found: "${tableId}". Pass the table id (tbl_...) from tables/{name}/meta.json, or a tables/{name}/meta.json path.`
|
|
)
|
|
}
|
|
const sandboxPath =
|
|
typeof tableRef === 'object' && tableRef !== null
|
|
? (tableRef as CanonicalTableInput).sandboxPath
|
|
: undefined
|
|
const mountPath = sandboxPath || `/home/user/tables/${table.id}.csv`
|
|
|
|
// Large/hot tables mount by reference from a version-keyed CSV snapshot in object storage.
|
|
if (snapshotCacheEnabled && table.rowCount >= SNAPSHOT_MIN_ROWS) {
|
|
const snapshot = await getOrCreateTableSnapshot(table, 'copilot-fn-exec')
|
|
if (!resolvedSecretTraceRegistry) {
|
|
throw new Error(
|
|
`Input table "${tableId}" cannot be mounted because its secret provenance is unavailable.`
|
|
)
|
|
}
|
|
try {
|
|
const safeForModelMount = await isTableSnapshotSafeForModelMount({
|
|
tableId: table.id,
|
|
workspaceId,
|
|
rowsVersion: snapshot.version,
|
|
})
|
|
if (!safeForModelMount) resolvedSecretTraceRegistry.markIncomplete()
|
|
} catch {
|
|
resolvedSecretTraceRegistry.markIncomplete()
|
|
}
|
|
|
|
if (hasCloudStorage()) {
|
|
// Mount by reference: the sandbox fetches the snapshot straight from storage via a
|
|
// presigned URL, so the bytes never pass through the web process — the only ceiling is
|
|
// sandbox disk (enforced at materialization by SNAPSHOT_MAX_BYTES).
|
|
if (snapshot.size > SNAPSHOT_MAX_BYTES) {
|
|
throw new Error(
|
|
`Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${SNAPSHOT_MAX_BYTES / 1024 / 1024}MB table mount limit.`
|
|
)
|
|
}
|
|
const url = await generatePresignedDownloadUrl(
|
|
snapshot.key,
|
|
'execution',
|
|
MOUNT_URL_TTL_SECONDS
|
|
)
|
|
sandboxFiles.push({ type: 'url', path: mountPath, url })
|
|
continue
|
|
}
|
|
|
|
// Local storage: a presigned URL is an app-internal serve path a remote sandbox can't
|
|
// reach, so fall back to buffering the bytes through the web process (file-mount guards).
|
|
if (snapshot.size > MAX_FILE_SIZE) {
|
|
throw new Error(
|
|
`Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.`
|
|
)
|
|
}
|
|
if (mounted.buffered + snapshot.size > MAX_TOTAL_SIZE) {
|
|
throw new Error(
|
|
`Mounting "${tableId}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller tables.`
|
|
)
|
|
}
|
|
const buffer = await downloadFile({
|
|
key: snapshot.key,
|
|
context: 'execution',
|
|
maxBytes: MAX_FILE_SIZE,
|
|
})
|
|
mounted.buffered += buffer.length
|
|
sandboxFiles.push({ path: mountPath, content: buffer.toString('utf-8') })
|
|
continue
|
|
}
|
|
|
|
// Keep the prior bounded mount — draining the whole table here was backed
|
|
// out for OOM, so don't ride the new unbounded queryRows default.
|
|
const rows = await queryRows(
|
|
table,
|
|
{ limit: TABLE_LIMITS.DEFAULT_QUERY_LIMIT },
|
|
'copilot-fn-exec'
|
|
)
|
|
if (!resolvedSecretTraceRegistry) {
|
|
throw new Error(
|
|
`Input table "${tableId}" cannot be mounted because its secret provenance is unavailable.`
|
|
)
|
|
}
|
|
try {
|
|
const provenance = await loadTableRowSecretProvenance(rows.rows, {
|
|
userId: provenanceUserId ?? 'opaque-model-mount',
|
|
workspaceId,
|
|
})
|
|
if (
|
|
!provenance.complete ||
|
|
!(await resolvedSecretTraceRegistry.importProvenance(provenance, { trusted: true }))
|
|
) {
|
|
resolvedSecretTraceRegistry.markIncomplete()
|
|
}
|
|
} catch {
|
|
resolvedSecretTraceRegistry.markIncomplete()
|
|
}
|
|
|
|
const columns = table.schema.columns
|
|
const csvLines = [toCsvRow(columns.map((column) => neutralizeCsvFormula(column.name)))]
|
|
for (const row of rows.rows) {
|
|
csvLines.push(
|
|
toCsvRow(columns.map((column) => formatCsvCell(column, row.data[getColumnId(column)])))
|
|
)
|
|
}
|
|
const csvContent = csvLines.join('\n')
|
|
sandboxFiles.push({ path: mountPath, content: csvContent })
|
|
}
|
|
}
|
|
|
|
return sandboxFiles
|
|
}
|
|
|
|
async function importMountedProvenance(
|
|
source: ResolvedSecretTraceRegistry,
|
|
target: ResolvedSecretTraceRegistry | undefined,
|
|
crossingValue: unknown
|
|
): Promise<void> {
|
|
if (!target) return
|
|
|
|
try {
|
|
const provenance = source.exportProvenanceForValue(crossingValue)
|
|
const imported = await target.importCrossingProvenance(provenance, crossingValue, {
|
|
trusted: true,
|
|
})
|
|
if (!imported) target.markIncomplete()
|
|
} catch {
|
|
target.markIncomplete()
|
|
}
|
|
}
|
|
|
|
export async function executeFunctionExecute(
|
|
params: Record<string, unknown>,
|
|
context: ToolExecutionContext
|
|
): Promise<ToolExecutionResult> {
|
|
const enrichedParams = omit(params, [
|
|
'sandboxProfile',
|
|
'internalSandboxProfile',
|
|
PRIVATE_SECRET_PROVENANCE_FIELD,
|
|
])
|
|
if (params.sandboxId !== undefined) {
|
|
if (typeof params.sandboxId !== 'string' || !params.sandboxId.trim()) {
|
|
throw new Error('sandboxId must be a non-empty Sim sandbox id')
|
|
}
|
|
if (!context.workspaceId) {
|
|
throw new Error('A workspace is required to select a Sim sandbox')
|
|
}
|
|
if (!(await hasWorkspaceSandboxAccess(context.workspaceId))) {
|
|
throw new Error(MAX_PLAN_REQUIRED)
|
|
}
|
|
enrichedParams.sandboxId = params.sandboxId.trim()
|
|
}
|
|
const requestedNames = applySecretMountPolicy(
|
|
await extractCodeSecretNames(params.code, params.language),
|
|
context.secretMountPolicy
|
|
)
|
|
const completePendingActivation =
|
|
requestedNames.length > 0
|
|
? context.resolvedSecretTraceRegistry?.beginPendingActivation()
|
|
: undefined
|
|
let mountedRegistry: ResolvedSecretTraceRegistry | undefined
|
|
let crossingValue: unknown
|
|
|
|
try {
|
|
const secretActorUserId =
|
|
context.secretActorUserId === undefined ? context.userId : context.secretActorUserId
|
|
let mounted: MaterializedCopilotCodeSecrets = { envVars: {}, catalogEntries: [] }
|
|
if (requestedNames.length > 0) {
|
|
if (!secretActorUserId) {
|
|
throw new CopilotCodeSecretAccessError('Secret access is unavailable for this Copilot run')
|
|
}
|
|
if (!context.workspaceId) {
|
|
throw new CopilotCodeSecretAccessError(
|
|
'A workspace is required to mount secrets into Copilot code'
|
|
)
|
|
}
|
|
mounted = await materializeCopilotCodeSecrets({
|
|
actorUserId: secretActorUserId,
|
|
workspaceId: context.workspaceId,
|
|
requestedNames,
|
|
})
|
|
}
|
|
mountedRegistry = new ResolvedSecretTraceRegistry(mounted.catalogEntries, {
|
|
userId: secretActorUserId ?? context.userId,
|
|
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
|
})
|
|
|
|
enrichedParams.envVars = mounted.envVars
|
|
enrichedParams.secretScope = 'selected'
|
|
enrichedParams.mountedSecrets = requestedNames
|
|
|
|
if (context.workspaceId) {
|
|
const inputs = enrichedParams.inputs as
|
|
| {
|
|
files?: CanonicalFileInput[]
|
|
directories?: CanonicalDirectoryInput[]
|
|
tables?: CanonicalTableInput[]
|
|
}
|
|
| undefined
|
|
const inputFiles = [
|
|
...((enrichedParams.inputFiles as unknown[] | undefined) ?? []),
|
|
...(inputs?.files ?? []),
|
|
]
|
|
const inputDirectories = inputs?.directories ?? []
|
|
const inputTables = [
|
|
...((enrichedParams.inputTables as unknown[] | undefined) ?? []),
|
|
...(inputs?.tables ?? []),
|
|
]
|
|
|
|
if (inputFiles?.length || inputTables?.length || inputDirectories.length) {
|
|
const resolved = await resolveInputFiles(
|
|
context.workspaceId,
|
|
inputFiles,
|
|
inputTables,
|
|
inputDirectories,
|
|
secretActorUserId ?? context.userId,
|
|
mountedRegistry
|
|
)
|
|
if (resolved.length > 0) {
|
|
const existing = (enrichedParams._sandboxFiles as SandboxFile[]) || []
|
|
enrichedParams._sandboxFiles = [...existing, ...resolved]
|
|
|
|
const provenance = mountedRegistry.exportProvenance()
|
|
const bundle: PrivateSecretProvenanceBundleV1 = {
|
|
version: 1,
|
|
complete: provenance.complete,
|
|
selections: provenance.complete
|
|
? [{ key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, provenance }]
|
|
: [],
|
|
}
|
|
enrichedParams[PRIVATE_SECRET_PROVENANCE_FIELD] = bundle
|
|
}
|
|
}
|
|
}
|
|
|
|
enrichedParams._context = {
|
|
userId: context.userId,
|
|
workflowId: context.workflowId,
|
|
workspaceId: context.workspaceId,
|
|
chatId: context.chatId,
|
|
executionId: context.executionId,
|
|
runId: context.runId,
|
|
enforceCredentialAccess: true,
|
|
}
|
|
|
|
try {
|
|
const result = await executeAppTool('function_execute', enrichedParams, {
|
|
resolvedSecretTraceRegistry: mountedRegistry,
|
|
...(context.abortSignal ? { signal: context.abortSignal } : {}),
|
|
...(context.sandboxProfile ? { internalSandboxProfile: context.sandboxProfile } : {}),
|
|
})
|
|
crossingValue = result
|
|
return result
|
|
} catch (error) {
|
|
crossingValue = error
|
|
throw error
|
|
}
|
|
} finally {
|
|
if (mountedRegistry && crossingValue !== undefined) {
|
|
await importMountedProvenance(
|
|
mountedRegistry,
|
|
context.resolvedSecretTraceRegistry,
|
|
crossingValue
|
|
)
|
|
}
|
|
completePendingActivation?.()
|
|
}
|
|
}
|