simstudioai--sim/scripts/sync-agent-stream-docs.ts

/**
 * Generates the "Streamed thinking and tool calls" support tables on the Agent
 * block docs page from the provider registry, so the docs can never drift from
 * the code:
 *
 * - Thinking visibility per model comes from `capabilities.thinking.streamed`
 *   (explicit) or the per-provider defaults in `getThinkingStreamVisibility`.
 * - Live tool-call streaming comes from `STREAMING_TOOL_CALL_PROVIDERS`.
 *
 * Content is rewritten between the `agent-stream-capabilities` markers in
 * `apps/docs/content/docs/en/workflows/blocks/agent.mdx`.
 *
 * Usage:
 *   bun run scripts/sync-agent-stream-docs.ts           # write
 *   bun run scripts/sync-agent-stream-docs.ts --check   # fail on drift or missing metadata
 */

import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
import {
  getThinkingStreamVisibility,
  PROVIDER_DEFINITIONS,
  type ThinkingStreamVisibility,
} from '../apps/sim/providers/models'
import { STREAMING_TOOL_CALL_PROVIDERS } from '../apps/sim/providers/streaming-tool-loop-shared'

const __filename = fileURLToPath(import.meta.url)
const rootDir = path.resolve(path.dirname(__filename), '..')

const AGENT_DOC_PATH = path.join(rootDir, 'apps/docs/content/docs/en/workflows/blocks/agent.mdx')

const BEGIN_MARKER =
  '{/* agent-stream-capabilities:begin — generated by `bun run agent-stream-docs:generate`; do not edit between markers */}'
const END_MARKER = '{/* agent-stream-capabilities:end */}'

/**
 * Providers whose thinking visibility varies per model generation and must
 * therefore be declared explicitly on every thinking-capable model.
 */
const EXPLICIT_VISIBILITY_PROVIDERS = new Set(['anthropic', 'azure-anthropic'])

const VISIBILITY_LABELS: Record<ThinkingStreamVisibility, string> = {
  full: 'Full thinking deltas',
  summary: 'Summaries only',
  none: 'Not streamed',
}

const VISIBILITY_NOTES: Partial<Record<string, string>> = {
  'openai:summary': 'Requires OpenAI organization verification; falls back to no summaries.',
  'azure-openai:summary': 'Requires OpenAI organization verification; falls back to no summaries.',
  'anthropic:summary':
    'These generations omit full thinking; Sim requests summarized thinking on streaming runs.',
  'azure-anthropic:summary':
    'These generations omit full thinking; Sim requests summarized thinking on streaming runs.',
  'anthropic:none': 'These model generations return thinking with omitted display by default.',
  'azure-anthropic:none':
    'These model generations return thinking with omitted display by default.',
  'bedrock:none': 'Sim does not request reasoning on Bedrock.',
}

interface VisibilityRow {
  providerName: string
  visibility: ThinkingStreamVisibility
  note: string
  models: string[]
}

function buildVisibilityRows(): { rows: VisibilityRow[]; errors: string[] } {
  const rows: VisibilityRow[] = []
  const errors: string[] = []

  for (const provider of Object.values(PROVIDER_DEFINITIONS)) {
    const grouped = new Map<ThinkingStreamVisibility, string[]>()

    for (const model of provider.models) {
      if (model.sunset?.status === 'deprecated') continue
      const reasoningCapable = model.capabilities.thinking || model.capabilities.reasoningEffort
      if (!reasoningCapable) continue

      if (
        EXPLICIT_VISIBILITY_PROVIDERS.has(provider.id) &&
        model.capabilities.thinking &&
        model.capabilities.thinking.streamed === undefined
      ) {
        errors.push(
          `${provider.id}/${model.id}: thinking-capable models on this provider must declare capabilities.thinking.streamed ('full' | 'summary' | 'none') — visibility varies per Claude generation`
        )
        continue
      }

      const visibility = getThinkingStreamVisibility(model.id)
      if (!visibility) continue
      const models = grouped.get(visibility) ?? []
      models.push(model.id)
      grouped.set(visibility, models)
    }

    for (const visibility of ['full', 'summary', 'none'] as const) {
      const models = grouped.get(visibility)
      if (!models?.length) continue
      rows.push({
        providerName: provider.name,
        visibility,
        note: VISIBILITY_NOTES[`${provider.id}:${visibility}`] ?? '',
        models,
      })
    }
  }

  return { rows, errors }
}

function buildGeneratedContent(): { content: string; errors: string[] } {
  const { rows, errors } = buildVisibilityRows()

  const liveToolProviders = Object.values(PROVIDER_DEFINITIONS)
    .filter((provider) => STREAMING_TOOL_CALL_PROVIDERS.has(provider.id))
    .map((provider) => provider.name)

  const lines: string[] = []
  lines.push('')
  lines.push(
    `Live tool-call chips stream for **${liveToolProviders.join(', ')}** models. Other providers run tools without live chips and project the settled final answer when the run completes; they do not ask the model to regenerate that answer just to create a stream.`
  )
  lines.push('')
  lines.push('| Provider | Streamed thinking | Models |')
  lines.push('|----------|-------------------|--------|')
  for (const row of rows) {
    const models = row.models.map((id) => `\`${id}\``).join(', ')
    const visibility = row.note
      ? `${VISIBILITY_LABELS[row.visibility]} — ${row.note}`
      : VISIBILITY_LABELS[row.visibility]
    lines.push(`| ${row.providerName} | ${visibility} | ${models} |`)
  }
  lines.push('')

  return { content: lines.join('\n'), errors }
}

function main(): void {
  const checkMode = process.argv.includes('--check')

  const { content, errors } = buildGeneratedContent()
  if (errors.length > 0) {
    console.error('agent-stream-docs: missing stream-visibility metadata:')
    for (const error of errors) {
      console.error(`  - ${error}`)
    }
    process.exit(1)
  }

  const doc = fs.readFileSync(AGENT_DOC_PATH, 'utf8')
  const beginIndex = doc.indexOf(BEGIN_MARKER)
  const endIndex = doc.indexOf(END_MARKER)
  if (beginIndex === -1 || endIndex === -1 || endIndex < beginIndex) {
    console.error(
      `agent-stream-docs: markers not found in ${path.relative(rootDir, AGENT_DOC_PATH)}`
    )
    process.exit(1)
  }

  const next =
    doc.slice(0, beginIndex + BEGIN_MARKER.length) + `\n${content}\n` + doc.slice(endIndex)

  if (checkMode) {
    if (next !== doc) {
      console.error(
        'agent-stream-docs: docs are out of date — run `bun run agent-stream-docs:generate`'
      )
      process.exit(1)
    }
    console.log('agent-stream-docs: up to date.')
    return
  }

  if (next !== doc) {
    fs.writeFileSync(AGENT_DOC_PATH, next)
    console.log(`agent-stream-docs: updated ${path.relative(rootDir, AGENT_DOC_PATH)}`)
  } else {
    console.log('agent-stream-docs: no changes.')
  }
}

main()