WeHub Mirror
6bf8bebf51
CI / Test and Build (push) Failing after 1s
CI / Migrate Dev DB (push) Has been skipped
CI / Migrate DB (push) Has been skipped
CodeQL / Analyze actions (push) Has been cancelled
CodeQL / Analyze javascript-typescript (push) Has been cancelled
CI / Detect Version (push) Has been cancelled
CI / Detect Desktop Changes (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/cron.Dockerfile, ubuntu-latest, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build AMD64 (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/cron.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/db.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/pii.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/realtime.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-8vcpu-ubuntu-2404-arm, ./docker/app.Dockerfile, linux-arm64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
Helm Chart / Lint, test, and validate chart (push) Has been cancelled
Helm Chart / Chart version bumped (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Build Dev ECR (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core) (push) Has been cancelled
CI / Promote Images (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Check Desktop Signing Secrets (push) Has been cancelled
CI / Desktop Release (push) Has been cancelled
CI / Create Desktop Prerelease (push) Has been cancelled
CI / Desktop Prerelease Build (push) Has been cancelled
CI / Publish Desktop Prerelease (push) Has been cancelled
CI / Prune Desktop Prereleases (push) Has been cancelled
Helm Chart / Install on kind and run helm test (push) Has been cancelled
107 lines
2.6 KiB
TypeScript
107 lines
2.6 KiB
TypeScript
/**
|
|
* Configuration constants for tokenization functionality
|
|
*/
|
|
|
|
import type { ProviderTokenizationConfig } from '@/lib/tokenization/types'
|
|
|
|
export const TOKENIZATION_CONFIG = {
|
|
providers: {
|
|
openai: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
'azure-openai': {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
anthropic: {
|
|
avgCharsPerToken: 4.5,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
'azure-anthropic': {
|
|
avgCharsPerToken: 4.5,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
google: {
|
|
avgCharsPerToken: 5,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
deepseek: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
xai: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
cerebras: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
mistral: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
groq: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
sakana: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
nvidia: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
meta: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
zai: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
kimi: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
ollama: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'low',
|
|
supportedMethods: ['fallback'],
|
|
},
|
|
} satisfies Record<string, ProviderTokenizationConfig>,
|
|
|
|
fallback: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'low',
|
|
supportedMethods: ['fallback'],
|
|
} satisfies ProviderTokenizationConfig,
|
|
|
|
defaults: {
|
|
model: 'gpt-4o',
|
|
provider: 'openai',
|
|
},
|
|
} as const
|
|
|
|
export const LLM_BLOCK_TYPES = ['agent', 'router', 'evaluator'] as const
|
|
|
|
export const MIN_TEXT_LENGTH_FOR_ESTIMATION = 1
|
|
export const MAX_PREVIEW_LENGTH = 100
|