6bf8bebf51
CI / Test and Build (push) Failing after 1s
CI / Migrate Dev DB (push) Has been skipped
CI / Migrate DB (push) Has been skipped
CodeQL / Analyze actions (push) Has been cancelled
CodeQL / Analyze javascript-typescript (push) Has been cancelled
CI / Detect Version (push) Has been cancelled
CI / Detect Desktop Changes (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/cron.Dockerfile, ubuntu-latest, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build AMD64 (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/cron.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/db.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/pii.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/realtime.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-8vcpu-ubuntu-2404-arm, ./docker/app.Dockerfile, linux-arm64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
Helm Chart / Lint, test, and validate chart (push) Has been cancelled
Helm Chart / Chart version bumped (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Build Dev ECR (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core) (push) Has been cancelled
CI / Promote Images (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Check Desktop Signing Secrets (push) Has been cancelled
CI / Desktop Release (push) Has been cancelled
CI / Create Desktop Prerelease (push) Has been cancelled
CI / Desktop Prerelease Build (push) Has been cancelled
CI / Publish Desktop Prerelease (push) Has been cancelled
CI / Prune Desktop Prereleases (push) Has been cancelled
Helm Chart / Install on kind and run helm test (push) Has been cancelled
907 lines
30 KiB
TypeScript
907 lines
30 KiB
TypeScript
import { afterAll, describe, expect, it, vi } from 'vitest'
|
|
import { mergeToolParameters } from '@/tools/merge-params'
|
|
import * as toolMetadata from '@/tools/metadata'
|
|
import {
|
|
createExecutionToolSchema,
|
|
createLLMToolSchema,
|
|
createUserToolSchema,
|
|
filterSchemaForLLM,
|
|
formatParameterLabel,
|
|
getSubBlocksForToolInput,
|
|
getToolParametersConfig,
|
|
isPasswordParameter,
|
|
type ToolParameterConfig,
|
|
type ToolSchema,
|
|
ToolSchemaEnrichmentError,
|
|
type ValidationResult,
|
|
validateToolParameters,
|
|
} from '@/tools/params'
|
|
import type { HttpMethod, ParameterVisibility } from '@/tools/types'
|
|
|
|
const mockToolConfig = {
|
|
id: 'test_tool',
|
|
name: 'Test Tool',
|
|
description: 'A test tool for parameter handling',
|
|
version: '1.0.0',
|
|
params: {
|
|
apiKey: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'API key for authentication',
|
|
},
|
|
message: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Message to send',
|
|
},
|
|
channel: {
|
|
type: 'string',
|
|
required: false,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'Channel to send message to',
|
|
},
|
|
timeout: {
|
|
type: 'number',
|
|
required: false,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
default: 5000,
|
|
description: 'Request timeout in milliseconds',
|
|
},
|
|
},
|
|
request: {
|
|
url: 'https://api.example.com/test',
|
|
method: 'POST' as HttpMethod,
|
|
headers: () => ({}),
|
|
},
|
|
}
|
|
|
|
/**
|
|
* Spy on the real module namespace instead of vi.mock: under `isolate: false`
|
|
* `@/tools/params` may already be cached bound to the real `@/tools/metadata`
|
|
* module, so patching the shared namespace is the only wiring that always
|
|
* applies.
|
|
*/
|
|
const getToolSpy = vi.spyOn(toolMetadata, 'getToolMetadata').mockImplementation(((
|
|
toolId: string
|
|
) => {
|
|
if (toolId === 'test_tool') {
|
|
return mockToolConfig
|
|
}
|
|
if (toolId === 'workflow_executor') {
|
|
return {
|
|
id: 'workflow_executor',
|
|
name: 'Workflow Executor',
|
|
description: '',
|
|
version: '1.0.0',
|
|
params: {},
|
|
}
|
|
}
|
|
return null
|
|
}) as unknown as typeof toolMetadata.getToolMetadata)
|
|
|
|
afterAll(() => {
|
|
getToolSpy.mockRestore()
|
|
})
|
|
|
|
describe('Tool Parameters Utils', () => {
|
|
describe('getToolParametersConfig', () => {
|
|
it.concurrent('should return tool parameters configuration', () => {
|
|
const result = getToolParametersConfig('test_tool')
|
|
|
|
expect(result).toBeDefined()
|
|
expect(result?.toolConfig).toEqual(mockToolConfig)
|
|
expect(result?.allParameters).toHaveLength(4)
|
|
expect(result?.userInputParameters).toHaveLength(4) // apiKey, message, channel, timeout (all have visibility)
|
|
expect(result?.requiredParameters).toHaveLength(2) // apiKey, message (both required: true)
|
|
expect(result?.optionalParameters).toHaveLength(2) // channel, timeout (both user-only + required: false)
|
|
})
|
|
|
|
it.concurrent('should return null for non-existent tool', () => {
|
|
const result = getToolParametersConfig('non_existent_tool')
|
|
expect(result).toBeNull()
|
|
})
|
|
})
|
|
|
|
describe('createLLMToolSchema', () => {
|
|
it.concurrent('should create schema excluding user-provided parameters', async () => {
|
|
const userProvidedParams = {
|
|
apiKey: 'user-provided-key',
|
|
channel: '#general',
|
|
}
|
|
|
|
const { schema } = await createLLMToolSchema(mockToolConfig, userProvidedParams)
|
|
|
|
expect(schema.properties).not.toHaveProperty('apiKey') // user-only, excluded
|
|
expect(schema.properties).not.toHaveProperty('channel') // user-provided, excluded
|
|
expect(schema.properties).toHaveProperty('message') // user-or-llm, included
|
|
expect(schema.properties).not.toHaveProperty('timeout') // user-only, excluded
|
|
expect(schema.required).toContain('message') // user-or-llm + required: true
|
|
expect(schema.required).not.toContain('apiKey') // user-only, never required for LLM
|
|
})
|
|
|
|
it.concurrent('should include all parameters when none are user-provided', async () => {
|
|
const { schema } = await createLLMToolSchema(mockToolConfig, {})
|
|
|
|
expect(schema.properties).not.toHaveProperty('apiKey') // user-only, never shown to LLM
|
|
expect(schema.properties).toHaveProperty('message') // user-or-llm, shown to LLM
|
|
expect(schema.properties).not.toHaveProperty('channel') // user-only, never shown to LLM
|
|
expect(schema.properties).not.toHaveProperty('timeout') // user-only, never shown to LLM
|
|
expect(schema.required).not.toContain('apiKey') // user-only, never required for LLM
|
|
expect(schema.required).toContain('message') // user-or-llm + required: true
|
|
})
|
|
|
|
it('wraps tool enrichment failures so execution boundaries can fail fast', async () => {
|
|
const cause = new Error('table metadata unavailable')
|
|
const toolConfig = {
|
|
...mockToolConfig,
|
|
toolEnrichment: {
|
|
dependsOn: 'tableId',
|
|
enrichTool: vi.fn().mockRejectedValue(cause),
|
|
},
|
|
}
|
|
|
|
const error = await createLLMToolSchema(toolConfig, { tableId: 'tbl_123' }).catch(
|
|
(caught) => caught
|
|
)
|
|
|
|
expect(error).toBeInstanceOf(ToolSchemaEnrichmentError)
|
|
expect(error).toMatchObject({
|
|
message: 'Failed to enrich schema for tool "test_tool"',
|
|
cause,
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('createUserToolSchema', () => {
|
|
it.concurrent('should include user-only parameters and omit hidden ones', () => {
|
|
const toolWithHiddenParam = {
|
|
...mockToolConfig,
|
|
id: 'user_schema_tool',
|
|
params: {
|
|
...mockToolConfig.params,
|
|
spreadsheetId: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'Spreadsheet ID to operate on',
|
|
},
|
|
accessToken: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'hidden' as ParameterVisibility,
|
|
description: 'OAuth access token',
|
|
},
|
|
},
|
|
}
|
|
|
|
const schema = createUserToolSchema(toolWithHiddenParam)
|
|
|
|
expect(schema.properties).toHaveProperty('spreadsheetId')
|
|
expect(schema.required).toContain('spreadsheetId')
|
|
expect(schema.properties).not.toHaveProperty('accessToken')
|
|
expect(schema.required).not.toContain('accessToken')
|
|
expect(schema.properties).toHaveProperty('message')
|
|
})
|
|
|
|
it.concurrent('keeps the hosted key param optional when hosted keys are supported', () => {
|
|
const hostedTool = {
|
|
...mockToolConfig,
|
|
id: 'hosted_key_tool',
|
|
params: {
|
|
query: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Search query',
|
|
},
|
|
apiKey: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'Exa AI API Key',
|
|
},
|
|
},
|
|
hosting: {
|
|
envKeyPrefix: 'EXA_API_KEY',
|
|
apiKeyParam: 'apiKey',
|
|
byokProviderId: 'exa',
|
|
pricing: { type: 'per_request' as const, cost: 0.005 },
|
|
rateLimit: { mode: 'per_request' as const, requestsPerMinute: 100 },
|
|
},
|
|
}
|
|
|
|
const hostedSchema = createUserToolSchema(hostedTool, {
|
|
surface: 'copilot',
|
|
hostedKeySupport: true,
|
|
})
|
|
|
|
// The key stays available as a bring-your-own-key override but is never
|
|
// a required argument — the executor injects the hosted key server-side.
|
|
expect(hostedSchema.properties).toHaveProperty('apiKey')
|
|
expect(hostedSchema.required).not.toContain('apiKey')
|
|
expect(hostedSchema.properties.apiKey.description).toContain('hosted key')
|
|
expect(hostedSchema.required).toContain('query')
|
|
})
|
|
|
|
it.concurrent('keeps the hosted key param required without hosted key support', () => {
|
|
const hostedTool = {
|
|
...mockToolConfig,
|
|
id: 'hosted_key_tool_self_hosted',
|
|
params: {
|
|
apiKey: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'Exa AI API Key',
|
|
},
|
|
},
|
|
hosting: {
|
|
envKeyPrefix: 'EXA_API_KEY',
|
|
apiKeyParam: 'apiKey',
|
|
byokProviderId: 'exa',
|
|
pricing: { type: 'per_request' as const, cost: 0.005 },
|
|
rateLimit: { mode: 'per_request' as const, requestsPerMinute: 100 },
|
|
},
|
|
}
|
|
|
|
const selfHostedSchema = createUserToolSchema(hostedTool, { surface: 'copilot' })
|
|
|
|
expect(selfHostedSchema.required).toContain('apiKey')
|
|
expect(selfHostedSchema.properties.apiKey.description).not.toContain('hosted key')
|
|
})
|
|
|
|
it.concurrent('keeps the key required for conditionally hosted tools', () => {
|
|
const enabled = Object.assign(
|
|
(params: Record<string, unknown>) => params.provider === 'falai',
|
|
{
|
|
condition: { field: 'provider', operator: 'equals' as const, value: 'falai' },
|
|
}
|
|
)
|
|
const conditionalTool = {
|
|
...mockToolConfig,
|
|
id: 'conditional_hosted_tool',
|
|
params: {
|
|
apiKey: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'Provider API Key',
|
|
},
|
|
},
|
|
hosting: {
|
|
enabled,
|
|
envKeyPrefix: 'FALAI_API_KEY',
|
|
apiKeyParam: 'apiKey',
|
|
byokProviderId: 'falai',
|
|
pricing: { type: 'per_request' as const, cost: 0.01 },
|
|
rateLimit: { mode: 'per_request' as const, requestsPerMinute: 100 },
|
|
},
|
|
}
|
|
|
|
const schema = createUserToolSchema(conditionalTool, {
|
|
surface: 'copilot',
|
|
hostedKeySupport: true,
|
|
})
|
|
|
|
// Injection only happens when the predicate passes at runtime, so the
|
|
// schema must not promise a hosted key for every configuration.
|
|
expect(schema.required).toContain('apiKey')
|
|
expect(schema.properties.apiKey.description).not.toContain('hosted key')
|
|
})
|
|
|
|
it.concurrent('does not relax required keys on tools without hosting', () => {
|
|
const plainTool = {
|
|
...mockToolConfig,
|
|
id: 'plain_key_tool',
|
|
params: {
|
|
apiKey: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-only' as ParameterVisibility,
|
|
description: 'Service API Key',
|
|
},
|
|
},
|
|
}
|
|
|
|
const schema = createUserToolSchema(plainTool, {
|
|
surface: 'copilot',
|
|
hostedKeySupport: true,
|
|
})
|
|
|
|
expect(schema.required).toContain('apiKey')
|
|
})
|
|
|
|
it.concurrent('adds credentialId only for copilot-facing oauth schemas', () => {
|
|
const oauthTool = {
|
|
...mockToolConfig,
|
|
id: 'oauth_schema_tool',
|
|
oauth: {
|
|
required: true,
|
|
provider: 'google-email',
|
|
},
|
|
params: {
|
|
message: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Message to send',
|
|
},
|
|
accessToken: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'hidden' as ParameterVisibility,
|
|
description: 'OAuth access token',
|
|
},
|
|
},
|
|
}
|
|
|
|
const defaultSchema = createUserToolSchema(oauthTool)
|
|
const copilotSchema = createUserToolSchema(oauthTool, { surface: 'copilot' })
|
|
|
|
expect(defaultSchema.properties).not.toHaveProperty('credentialId')
|
|
expect(copilotSchema.properties).toHaveProperty('credentialId')
|
|
expect(copilotSchema.properties.credentialId).toMatchObject({
|
|
type: 'string',
|
|
})
|
|
expect(copilotSchema.required).toContain('credentialId')
|
|
})
|
|
|
|
it.concurrent('keeps shared file params unchanged by default', () => {
|
|
const toolWithFileParam = {
|
|
...mockToolConfig,
|
|
id: 'file_schema_tool',
|
|
params: {
|
|
attachment: {
|
|
type: 'file',
|
|
required: true,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Attachment file',
|
|
},
|
|
},
|
|
}
|
|
|
|
const schema = createUserToolSchema(toolWithFileParam)
|
|
|
|
expect(schema.properties.attachment).toMatchObject({
|
|
type: 'file',
|
|
description: 'Attachment file',
|
|
})
|
|
})
|
|
|
|
it.concurrent('expands file params for copilot-facing schemas', () => {
|
|
const toolWithFileParams = {
|
|
...mockToolConfig,
|
|
id: 'copilot_file_schema_tool',
|
|
params: {
|
|
attachment: {
|
|
type: 'file',
|
|
required: true,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Attachment file',
|
|
},
|
|
attachments: {
|
|
type: 'file[]',
|
|
required: false,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Attachment files',
|
|
},
|
|
},
|
|
}
|
|
|
|
const schema = createUserToolSchema(toolWithFileParams, { surface: 'copilot' })
|
|
|
|
expect(schema.properties.attachment).toMatchObject({
|
|
type: 'object',
|
|
required: ['id', 'name', 'url', 'size', 'type', 'key'],
|
|
})
|
|
expect(schema.properties.attachment.description).toContain('canonical workspace file IDs')
|
|
expect(schema.properties.attachments).toMatchObject({
|
|
type: 'array',
|
|
})
|
|
expect(schema.properties.attachments.description).toContain('canonical workspace file IDs')
|
|
})
|
|
})
|
|
|
|
describe('createExecutionToolSchema', () => {
|
|
it.concurrent('should create complete schema with all parameters', () => {
|
|
const schema = createExecutionToolSchema(mockToolConfig)
|
|
|
|
expect(schema.properties).toHaveProperty('apiKey')
|
|
expect(schema.properties).toHaveProperty('message')
|
|
expect(schema.properties).toHaveProperty('channel')
|
|
expect(schema.properties).toHaveProperty('timeout')
|
|
expect(schema.required).toContain('apiKey')
|
|
expect(schema.required).toContain('message')
|
|
expect(schema.required).not.toContain('channel')
|
|
expect(schema.required).not.toContain('timeout')
|
|
})
|
|
})
|
|
|
|
describe('mergeToolParameters', () => {
|
|
it.concurrent('should merge parameters with user-provided taking precedence', () => {
|
|
const userProvided = {
|
|
apiKey: 'user-key',
|
|
channel: '#general',
|
|
}
|
|
const llmGenerated = {
|
|
message: 'Hello world',
|
|
channel: '#random',
|
|
timeout: 10000,
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.apiKey).toBe('user-key')
|
|
expect(merged.channel).toBe('#general')
|
|
expect(merged.message).toBe('Hello world')
|
|
expect(merged.timeout).toBe(10000)
|
|
})
|
|
|
|
it.concurrent('should skip empty strings so LLM values are used', () => {
|
|
const userProvided = {
|
|
apiKey: 'user-key',
|
|
channel: '', // User cleared this field
|
|
message: '', // User cleared this field too
|
|
}
|
|
const llmGenerated = {
|
|
message: 'Hello world',
|
|
channel: '#random',
|
|
timeout: 10000,
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.apiKey).toBe('user-key') // Non-empty user value preserved
|
|
expect(merged.channel).toBe('#random') // LLM value used because user value was empty
|
|
expect(merged.message).toBe('Hello world') // LLM value used because user value was empty
|
|
expect(merged.timeout).toBe(10000)
|
|
})
|
|
|
|
it.concurrent('should skip null and undefined values', () => {
|
|
const userProvided = {
|
|
apiKey: 'user-key',
|
|
channel: null,
|
|
message: undefined,
|
|
}
|
|
const llmGenerated = {
|
|
message: 'Hello world',
|
|
channel: '#random',
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.apiKey).toBe('user-key')
|
|
expect(merged.channel).toBe('#random') // LLM value used
|
|
expect(merged.message).toBe('Hello world') // LLM value used
|
|
})
|
|
})
|
|
|
|
describe('validateToolParameters', () => {
|
|
it.concurrent('should validate successfully with all required parameters', () => {
|
|
const finalParams = {
|
|
apiKey: 'test-key',
|
|
message: 'Hello world',
|
|
channel: '#general',
|
|
}
|
|
|
|
const result = validateToolParameters(mockToolConfig, finalParams)
|
|
|
|
expect(result.valid).toBe(true)
|
|
expect(result.missingParams).toHaveLength(0)
|
|
})
|
|
|
|
it.concurrent('should fail validation with missing required parameters', () => {
|
|
const finalParams = {
|
|
channel: '#general',
|
|
}
|
|
|
|
const result = validateToolParameters(mockToolConfig, finalParams)
|
|
|
|
expect(result.valid).toBe(false)
|
|
expect(result.missingParams).toContain('apiKey')
|
|
expect(result.missingParams).toContain('message')
|
|
})
|
|
})
|
|
|
|
describe('filterSchemaForLLM', () => {
|
|
it.concurrent('should filter out user-provided parameters from schema', () => {
|
|
const originalSchema: ToolSchema = {
|
|
type: 'object' as const,
|
|
properties: {
|
|
apiKey: { type: 'string', description: 'API key' },
|
|
message: { type: 'string', description: 'Message' },
|
|
channel: { type: 'string', description: 'Channel' },
|
|
},
|
|
required: ['apiKey', 'message'],
|
|
}
|
|
|
|
const userProvidedParams = {
|
|
apiKey: 'user-key',
|
|
channel: '#general',
|
|
}
|
|
|
|
const filtered = filterSchemaForLLM(originalSchema, userProvidedParams)
|
|
|
|
expect(filtered.properties).not.toHaveProperty('apiKey')
|
|
expect(filtered.properties).not.toHaveProperty('channel')
|
|
expect(filtered.properties).toHaveProperty('message')
|
|
expect(filtered.required).not.toContain('apiKey')
|
|
expect(filtered.required).toContain('message')
|
|
})
|
|
})
|
|
|
|
describe('formatParameterLabel', () => {
|
|
it.concurrent('should format parameter labels correctly', () => {
|
|
expect(formatParameterLabel('apiKey')).toBe('API Key')
|
|
expect(formatParameterLabel('apiVersion')).toBe('API Version')
|
|
expect(formatParameterLabel('userName')).toBe('User Name')
|
|
expect(formatParameterLabel('user_name')).toBe('User Name')
|
|
expect(formatParameterLabel('user-name')).toBe('User Name')
|
|
expect(formatParameterLabel('message')).toBe('Message')
|
|
expect(formatParameterLabel('a')).toBe('A')
|
|
})
|
|
})
|
|
|
|
describe('isPasswordParameter', () => {
|
|
it.concurrent('should identify password parameters correctly', () => {
|
|
expect(isPasswordParameter('password')).toBe(true)
|
|
expect(isPasswordParameter('apiKey')).toBe(true)
|
|
expect(isPasswordParameter('token')).toBe(true)
|
|
expect(isPasswordParameter('secret')).toBe(true)
|
|
expect(isPasswordParameter('accessToken')).toBe(true)
|
|
expect(isPasswordParameter('message')).toBe(false)
|
|
expect(isPasswordParameter('channel')).toBe(false)
|
|
expect(isPasswordParameter('timeout')).toBe(false)
|
|
})
|
|
})
|
|
|
|
describe('workflow_executor inputMapping handling', () => {
|
|
const mockWorkflowExecutorConfig = {
|
|
id: 'workflow_executor',
|
|
name: 'Workflow Executor',
|
|
description: 'Execute another workflow',
|
|
version: '1.0.0',
|
|
params: {
|
|
workflowId: {
|
|
type: 'string',
|
|
required: true,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'The ID of the workflow to execute',
|
|
},
|
|
inputMapping: {
|
|
type: 'object',
|
|
required: false,
|
|
visibility: 'user-or-llm' as ParameterVisibility,
|
|
description: 'Map inputs to the selected workflow',
|
|
},
|
|
},
|
|
request: {
|
|
url: 'https://api.example.com/workflows',
|
|
method: 'POST' as HttpMethod,
|
|
headers: () => ({}),
|
|
},
|
|
}
|
|
|
|
describe('createLLMToolSchema - inputMapping always included', () => {
|
|
it.concurrent(
|
|
'should include inputMapping in schema even when user provides empty object',
|
|
async () => {
|
|
const userProvidedParams = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{}',
|
|
}
|
|
|
|
const { schema } = await createLLMToolSchema(
|
|
mockWorkflowExecutorConfig,
|
|
userProvidedParams
|
|
)
|
|
|
|
expect(schema.properties).toHaveProperty('inputMapping')
|
|
expect(schema.properties.inputMapping.type).toBe('object')
|
|
}
|
|
)
|
|
|
|
it.concurrent(
|
|
'should include inputMapping in schema even when user provides object with empty values',
|
|
async () => {
|
|
const userProvidedParams = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{"query": "", "limit": ""}',
|
|
}
|
|
|
|
const { schema } = await createLLMToolSchema(
|
|
mockWorkflowExecutorConfig,
|
|
userProvidedParams
|
|
)
|
|
|
|
expect(schema.properties).toHaveProperty('inputMapping')
|
|
}
|
|
)
|
|
|
|
it.concurrent(
|
|
'should include inputMapping when user has not provided it at all',
|
|
async () => {
|
|
const userProvidedParams = {
|
|
workflowId: 'workflow-123',
|
|
}
|
|
|
|
const { schema } = await createLLMToolSchema(
|
|
mockWorkflowExecutorConfig,
|
|
userProvidedParams
|
|
)
|
|
|
|
expect(schema.properties).toHaveProperty('inputMapping')
|
|
}
|
|
)
|
|
|
|
it.concurrent('should exclude workflowId from schema when user provides it', async () => {
|
|
const userProvidedParams = {
|
|
workflowId: 'workflow-123',
|
|
}
|
|
|
|
const { schema } = await createLLMToolSchema(mockWorkflowExecutorConfig, userProvidedParams)
|
|
|
|
expect(schema.properties).not.toHaveProperty('workflowId')
|
|
expect(schema.properties).toHaveProperty('inputMapping')
|
|
})
|
|
})
|
|
|
|
describe('mergeToolParameters - inputMapping deep merge', () => {
|
|
it.concurrent('should deep merge inputMapping when user provides empty object', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{}',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'search term', limit: 10 },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({ query: 'search term', limit: 10 })
|
|
expect(merged.workflowId).toBe('workflow-123')
|
|
})
|
|
|
|
it.concurrent('should deep merge inputMapping when user provides partial values', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{"query": "", "customField": "user-value"}',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'llm-search', limit: 10 },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({
|
|
query: 'llm-search',
|
|
limit: 10,
|
|
customField: 'user-value',
|
|
})
|
|
})
|
|
|
|
it.concurrent('should preserve user inputMapping values when they are non-empty', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{"query": "user-search", "limit": 5}',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'llm-search', limit: 10, extra: 'field' },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({
|
|
query: 'user-search',
|
|
limit: 5,
|
|
extra: 'field',
|
|
})
|
|
})
|
|
|
|
it.concurrent('should handle inputMapping as object (not JSON string)', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: { query: '', customField: 'user-value' },
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'llm-search', limit: 10 },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({
|
|
query: 'llm-search',
|
|
limit: 10,
|
|
customField: 'user-value',
|
|
})
|
|
})
|
|
|
|
it.concurrent('should use LLM inputMapping when user does not provide it', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'llm-search', limit: 10 },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({ query: 'llm-search', limit: 10 })
|
|
})
|
|
|
|
it.concurrent('should use user inputMapping when LLM does not provide it', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{"query": "user-search"}',
|
|
}
|
|
const llmGenerated = {}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({ query: 'user-search' })
|
|
})
|
|
|
|
it.concurrent('should handle invalid JSON in user inputMapping gracefully', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: 'not valid json {',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'llm-search' },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({ query: 'llm-search' })
|
|
})
|
|
|
|
it.concurrent(
|
|
'should fill field when user typed something then removed it (field becomes empty string)',
|
|
() => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{"query": ""}',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { query: 'llm-generated-search' },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.inputMapping).toEqual({ query: 'llm-generated-search' })
|
|
}
|
|
)
|
|
|
|
it.concurrent('should not affect other parameters - normal override behavior', () => {
|
|
const userProvided = {
|
|
apiKey: 'user-key',
|
|
channel: '#general',
|
|
}
|
|
const llmGenerated = {
|
|
message: 'Hello world',
|
|
channel: '#random',
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
expect(merged.apiKey).toBe('user-key')
|
|
expect(merged.channel).toBe('#general')
|
|
expect(merged.message).toBe('Hello world')
|
|
})
|
|
|
|
it.concurrent('should preserve 0 and false as valid user values in inputMapping', () => {
|
|
const userProvided = {
|
|
workflowId: 'workflow-123',
|
|
inputMapping: '{"limit": 0, "enabled": false, "query": ""}',
|
|
}
|
|
const llmGenerated = {
|
|
inputMapping: { limit: 10, enabled: true, query: 'llm-search' },
|
|
}
|
|
|
|
const merged = mergeToolParameters(userProvided, llmGenerated)
|
|
|
|
// 0 and false should be preserved (they're valid values)
|
|
// empty string should be filled by LLM
|
|
expect(merged.inputMapping).toEqual({
|
|
limit: 0,
|
|
enabled: false,
|
|
query: 'llm-search',
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('Type Interface Validation', () => {
|
|
it.concurrent('should have properly typed ToolSchema', async () => {
|
|
const { schema } = await createLLMToolSchema(mockToolConfig, {})
|
|
|
|
expect(schema.type).toBe('object')
|
|
expect(typeof schema.properties).toBe('object')
|
|
expect(Array.isArray(schema.required)).toBe(true)
|
|
|
|
Object.values(schema.properties).forEach((prop) => {
|
|
expect(prop).toHaveProperty('type')
|
|
expect(prop).toHaveProperty('description')
|
|
expect(typeof prop.type).toBe('string')
|
|
expect(typeof prop.description).toBe('string')
|
|
})
|
|
})
|
|
|
|
it.concurrent('should have properly typed ValidationResult', () => {
|
|
const result: ValidationResult = validateToolParameters(mockToolConfig, {})
|
|
|
|
expect(typeof result.valid).toBe('boolean')
|
|
expect(Array.isArray(result.missingParams)).toBe(true)
|
|
expect(result.missingParams.every((param) => typeof param === 'string')).toBe(true)
|
|
})
|
|
|
|
it.concurrent('should have properly typed ToolParameterConfig', () => {
|
|
const config = getToolParametersConfig('test_tool')
|
|
expect(config).toBeDefined()
|
|
|
|
if (config) {
|
|
config.allParameters.forEach((param: ToolParameterConfig) => {
|
|
expect(typeof param.id).toBe('string')
|
|
expect(typeof param.type).toBe('string')
|
|
expect(typeof param.required).toBe('boolean')
|
|
expect(
|
|
['user-or-llm', 'user-only', 'llm-only', 'hidden'].includes(param.visibility!)
|
|
).toBe(true)
|
|
if (param.description) expect(typeof param.description).toBe('string')
|
|
if (param.uiComponent) {
|
|
expect(typeof param.uiComponent.type).toBe('string')
|
|
}
|
|
})
|
|
}
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('custom block agent-tool rendering', () => {
|
|
// Mirrors buildCustomBlockConfig: hidden workflowId/inputMapping wiring + per-field
|
|
// sub-blocks keyed by the source field's stable id.
|
|
const customBlockConfig = {
|
|
subBlocks: [
|
|
{ id: 'workflowId', type: 'short-input', hidden: true },
|
|
{ id: 'inputMapping', type: 'code', language: 'json', hidden: true },
|
|
{ id: 'field-question', title: 'Question', type: 'short-input', required: true },
|
|
{ id: 'field-files', title: 'Attachments', type: 'file-upload', multiple: true },
|
|
],
|
|
} as any
|
|
|
|
describe('getToolParametersConfig', () => {
|
|
it('surfaces the field sub-blocks, never workflowId/inputMapping', () => {
|
|
const result = getToolParametersConfig(
|
|
'workflow_executor',
|
|
'custom_block_abc',
|
|
undefined,
|
|
customBlockConfig
|
|
)
|
|
expect(result).not.toBeNull()
|
|
const ids = result!.userInputParameters.map((p) => p.id)
|
|
expect(ids).toEqual(['field-question', 'field-files'])
|
|
expect(ids).not.toContain('workflowId')
|
|
expect(ids).not.toContain('inputMapping')
|
|
expect(result!.userInputParameters.every((p) => p.visibility === 'user-or-llm')).toBe(true)
|
|
expect(result!.requiredParameters.map((p) => p.id)).toEqual(['field-question'])
|
|
})
|
|
})
|
|
|
|
describe('getSubBlocksForToolInput', () => {
|
|
it('returns field sub-blocks as user-or-llm and drops reserved/hidden wiring', () => {
|
|
const result = getSubBlocksForToolInput(
|
|
'workflow_executor',
|
|
'custom_block_abc',
|
|
undefined,
|
|
undefined,
|
|
customBlockConfig
|
|
)
|
|
expect(result).not.toBeNull()
|
|
expect(result!.subBlocks.map((sb) => sb.id)).toEqual(['field-question', 'field-files'])
|
|
expect(result!.subBlocks.every((sb) => sb.paramVisibility === 'user-or-llm')).toBe(true)
|
|
})
|
|
})
|
|
})
|