Eric Allam
1cfc296c6b
- Automatic LLM cost enrichment for AI SDK spans (streamText, generateText, generateObject) or any other spans that use semantic gen_ai attributes with support for 145+ models - New AI span inspector sidebar showing model, tokens, cost, messages, tool calls, and response text - LLM metrics dual-write to ClickHouse `llm_metrics_v1` table for analytics - LLM metrics built-in dashboard (unlinked at the moment) - Provider cost fallback — uses gateway/OpenRouter reported costs from `providerMetadata` when registry pricing is unavailable - Prefix-stripping for gateway/OpenRouter model names (e.g. `mistral/mistral-large-3` matches `mistral-large-3` pricing) - Admin dashboard for managing LLM model pricing (list, create, edit, delete, search, test pattern matching) - Missing models detection page — queries ClickHouse for unpriced models with sample spans and Claude Code-ready prompts for adding pricing - AI span seed script (`pnpm run db:seed:ai-spans`) with 51 spans across 12 provider systems for local dev testing - UI fixes: `completionTokens`/`promptTokens` aliases, `ai.response.object` display for generateObject, cache read/write token breakdown ## Screenshots: <img width="1030" height="104" alt="CleanShot 2026-03-17 at 16 48 54@2x" src="https://github.com/user-attachments/assets/bc8fccda-e48b-4d0c-bfb1-e620064e5979" /> <img width="1094" height="1512" alt="CleanShot 2026-03-17 at 16 49 23@2x" src="https://github.com/user-attachments/assets/c2424569-d07e-4d67-a436-e8250043a1ee" /> <img width="1074" height="1412" alt="CleanShot 2026-03-17 at 16 49 18@2x" src="https://github.com/user-attachments/assets/22342ac4-4769-45d1-a328-a24fb9a82a50" /> <img width="1012" height="2292" alt="CleanShot 2026-03-17 at 16 39 01@2x" src="https://github.com/user-attachments/assets/59e327d1-6652-4293-8be0-bb8326e5fbc5" /> <img width="3680" height="2392" alt="CleanShot 2026-03-15 at 08 29 38@2x" src="https://github.com/user-attachments/assets/1f77beb8-de67-495b-b890-bcdb8d7f1fe8" /> --------- Co-authored-by: James Ritchie <james@trigger.dev>
65 lines
2.2 KiB
SQL
65 lines
2.2 KiB
SQL
-- +goose Up
|
|
CREATE TABLE IF NOT EXISTS trigger_dev.llm_metrics_v1
|
|
(
|
|
-- Tenant context
|
|
organization_id LowCardinality(String),
|
|
project_id LowCardinality(String),
|
|
environment_id String CODEC(ZSTD(1)),
|
|
run_id String CODEC(ZSTD(1)),
|
|
task_identifier LowCardinality(String),
|
|
trace_id String CODEC(ZSTD(1)),
|
|
span_id String CODEC(ZSTD(1)),
|
|
|
|
-- Model & provider
|
|
gen_ai_system LowCardinality(String),
|
|
request_model String CODEC(ZSTD(1)),
|
|
response_model String CODEC(ZSTD(1)),
|
|
matched_model_id String CODEC(ZSTD(1)),
|
|
operation_id LowCardinality(String),
|
|
finish_reason LowCardinality(String),
|
|
cost_source LowCardinality(String),
|
|
|
|
-- Pricing
|
|
pricing_tier_id String CODEC(ZSTD(1)),
|
|
pricing_tier_name LowCardinality(String),
|
|
|
|
-- Token usage
|
|
input_tokens UInt64 DEFAULT 0,
|
|
output_tokens UInt64 DEFAULT 0,
|
|
total_tokens UInt64 DEFAULT 0,
|
|
usage_details Map(LowCardinality(String), UInt64),
|
|
|
|
-- Cost
|
|
input_cost Decimal64(12) DEFAULT 0,
|
|
output_cost Decimal64(12) DEFAULT 0,
|
|
total_cost Decimal64(12) DEFAULT 0,
|
|
cost_details Map(LowCardinality(String), Decimal64(12)),
|
|
provider_cost Decimal64(12) DEFAULT 0,
|
|
|
|
-- Performance
|
|
ms_to_first_chunk Float64 DEFAULT 0,
|
|
tokens_per_second Float64 DEFAULT 0,
|
|
|
|
-- Attribution
|
|
metadata Map(LowCardinality(String), String),
|
|
|
|
-- Timing
|
|
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
|
duration UInt64 DEFAULT 0 CODEC(ZSTD(1)),
|
|
inserted_at DateTime64(3) DEFAULT now64(3),
|
|
|
|
-- Indexes
|
|
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
|
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
|
INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1,
|
|
INDEX idx_metadata_keys mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
|
|
)
|
|
ENGINE = MergeTree
|
|
PARTITION BY toDate(inserted_at)
|
|
ORDER BY (organization_id, project_id, environment_id, toDate(inserted_at), run_id)
|
|
TTL toDateTime(inserted_at) + INTERVAL 365 DAY
|
|
SETTINGS ttl_only_drop_parts = 1;
|
|
|
|
-- +goose Down
|
|
DROP TABLE IF EXISTS trigger_dev.llm_metrics_v1;
|