github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
57254b57fbc46f2f524541c8a29d8aee17fa7b5b
triggerdotdev--trigger.dev/package.json
T
Eric Allam 0445b8ec27 fix(webapp,clickhouse): keep the rest of a ClickHouse batch when one run or span has un-ingestable JSON (#4358) 
## Summary

A single run output, trace span, or payload carrying JSON that
ClickHouse can't ingest (for example nesting past its depth limit) used
to fail the whole insert batch, so unrelated runs and spans silently
disappeared from the runs list, traces, and logs. This keeps the rest of
the batch and handles the offending row instead of dropping everything
around it.

## Fix

Recovery is per-table, matched to what each table needs:

- **Runs** (`task_runs_v2`) keep their status. We follow ClickHouse's
failing-row hint to strip just the un-ingestable JSON column(s) so the
run still lands (its output reads from Postgres on the detail page), up
to a configurable limit (`RUN_REPLICATION_MAX_POISON_STRIPS_PER_BATCH`,
default `1`). Past the limit we stop and land the batch with
`allow_errors` in a single pass, skipping the remainder. Cost stays a
fixed handful of inserts no matter how large or poisoned a flush is.
- **Trace events and payloads** (high volume, append-only) recover with
a single `allow_errors` insert: the good rows land in one pass and only
the un-ingestable rows are skipped.

Before falling back, a lightweight sanitizer still repairs what it can
losslessly (lone UTF-16 surrogates, out-of-range integers) so a
repairable row lands in full.

To read the failing-row hint we patch `@clickhouse/client-common`: its
error parser truncates the server response and discards the `(at row N)`
position, so the patch preserves the full text for the recovery path to
read.
2026-08-01 09:17:20 +01:00

165 lines
6.8 KiB
JSON
Raw Blame History

{
"name": "triggerdotdev",
"private": true,
"workspaces": [
"apps/*",
"packages/*",
"integrations/*"
],
"version": "0.0.1",
"scripts": {
"build": "turbo run build",
"build:force": "turbo run build --force",
"build:db:seed": "turbo run build:db:seed",
"db:migrate": "turbo run db:migrate:deploy --filter=!@internal/run-ops-database && turbo run generate",
"db:seed": "turbo run db:seed",
"db:studio": "turbo run db:studio --filter=!@internal/run-ops-database",
"db:populate": "turbo run db:populate",
"dev": "turbo run dev",
"i:dev": "infisical run -- turbo run dev",
"generate": "turbo run generate",
"format": "oxfmt .",
"format:prisma": "pnpm --filter @trigger.dev/database run format:prisma && pnpm --filter @internal/run-ops-database run format:prisma",
"lint": "oxlint",
"lint:fix": "oxlint --fix",
"knip:deps": "knip --production --dependencies",
"docker": "node scripts/docker.mjs -f docker/docker-compose.yml up -d --build --remove-orphans",
"docker:stop": "node scripts/docker.mjs -f docker/docker-compose.yml stop",
"docker:full": "node scripts/docker.mjs -f docker/docker-compose.yml -f docker/docker-compose.extras.yml up -d --build --remove-orphans",
"docker:full:stop": "node scripts/docker.mjs -f docker/docker-compose.yml -f docker/docker-compose.extras.yml stop",
"dev:docker": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml up -d --build --remove-orphans",
"dev:docker:build": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml up -d --build",
"dev:docker:stop": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml stop",
"test": "turbo run test --concurrency=1 -- --run",
"test:webapp": "turbo run test --filter webapp -- --run",
"test:packages": "turbo run test --concurrency=1 --filter \"@trigger.dev/*\" -- --run",
"test:internal": "turbo run test --concurrency=1 --filter \"@internal/*\" -- --run",
"test:dev": "turbo run test:dev",
"start": "turbo run start",
"check-exports": "turbo run check-exports",
"clean": "turbo run clean",
"clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +",
"typecheck": "turbo run typecheck",
"test:e2e": "playwright test",
"test:e2e:ui": "playwright test --ui",
"test:e2e:dev": "turbo run test:e2e:dev",
"test:e2e:ci": "turbo run test:e2e:ci",
"setup": "turbo run generate db:migrate:force db:seed",
"env:pull": "turbo run env:pull",
"changeset:add": "changeset",
"changeset:version": "changeset version && pnpm install --lockfile-only && node scripts/bump-helm-chart.mjs && node scripts/cleanup-server-changes.mjs",
"changeset:release": "pnpm run build --filter \"@trigger.dev/*\" --filter \"trigger.dev\" && changeset publish",
"changeset:v4": "changeset pre enter v4",
"changeset:normal": "changeset pre exit",
"clean:sourcemaps": "turbo run clean:sourcemaps",
"storybook": "turbo run storybook"
},
"devDependencies": {
"@manypkg/cli": "^0.19.2",
"@playwright/test": "^1.36.2",
"@trigger.dev/database": "workspace:*",
"@types/node": "24.13.3",
"@vitest/coverage-v8": "4.1.7",
"autoprefixer": "^10.4.12",
"knip": "6.25.0",
"oxfmt": "^0.54.0",
"oxlint": "^1.69.0",
"pkg-pr-new": "0.0.75",
"pkg-types": "1.1.3",
"tsx": "^3.7.1",
"turbo": "^1.10.3",
"typescript": "catalog:",
"vite-tsconfig-paths": "^4.0.5",
"vitest": "4.1.7"
},
"packageManager": "pnpm@10.33.2",
"dependencies": {
"@changesets/cli": "2.26.2",
"@remix-run/changelog-github": "^0.0.5",
"agentcrumbs": "^0.5.0",
"node-fetch": "2.6.x"
},
"pnpm": {
"patchedDependencies": {
"@changesets/assemble-release-plan@5.2.4": "patches/@changesets__assemble-release-plan@5.2.4.patch",
"engine.io-parser@5.2.2": "patches/engine.io-parser@5.2.2.patch",
"redlock@5.0.0-beta.2": "patches/redlock@5.0.0-beta.2.patch",
"@kubernetes/client-node@1.0.0": "patches/@kubernetes__client-node@1.0.0.patch",
"@sentry/remix@9.46.0": "patches/@sentry__remix@9.46.0.patch",
"@upstash/ratelimit@1.1.3": "patches/@upstash__ratelimit.patch",
"antlr4ts@0.5.0-alpha.4": "patches/antlr4ts@0.5.0-alpha.4.patch",
"@window-splitter/state@1.1.3": "patches/@window-splitter__state@1.1.3.patch",
"streamdown@2.5.0": "patches/streamdown@2.5.0.patch",
"tsup@8.4.0": "patches/tsup@8.4.0.patch",
"@remix-run/router@1.23.3": "patches/@remix-run__router@1.23.3.patch",
"@clickhouse/client-common@1.12.1": "patches/@clickhouse__client-common@1.12.1.patch"
},
"overrides": {
"typescript": "catalog:",
"@types/node": "24.13.3",
"react@^18": "18.3.1",
"react-dom@^18": "18.3.1",
"ai@^6": "6.0.116",
"@ai-sdk/provider-utils@^4": "4.0.29",
"express@^4>body-parser": "1.20.3",
"@remix-run/dev@2.17.5>tar-fs": "2.1.4",
"tar": "7.5.19",
"form-data@^2": "2.5.4",
"form-data@^3": "3.0.5",
"form-data@^4": "4.0.6",
"ws@>=7 <7.5.11": "7.5.11",
"ws@>=8 <8.21.0": "8.21.0",
"hono@>=4 <4.12.25": "4.12.25",
"undici@>=6 <6.27.0": "6.27.0",
"undici@>=7 <7.28.0": "7.28.0",
"js-yaml@>=3.0.0 <3.14.2": "3.14.2",
"js-yaml@>=4.0.0 <4.1.1": "4.1.1",
"jws@<3.2.3": "3.2.3",
"qs@>=6.0.0 <6.15.2": "^6.15.2",
"lodash@>=4.17 <4.18.0": "^4.18.0",
"lodash-es@>=4.17 <4.18.0": "^4.18.0",
"dompurify@>=3 <3.4.0": "^3.4.1",
"vite@>=5.0.0 <6.4.2": "^6.4.2",
"rollup@>=4 <4.59.0": "^4.59.0",
"flatted@>=3 <3.4.2": "^3.4.2",
"picomatch@>=2 <2.3.2": "^2.3.2",
"picomatch@>=4 <4.0.4": "^4.0.4",
"minimatch@>=3 <3.1.3": "^3.1.3",
"protobufjs@>=7 <7.5.6": "^7.5.6",
"fast-xml-parser@>=4 <4.5.5": "^4.5.5",
"fast-xml-parser@>=5 <5.7.0": "^5.7.0",
"path-to-regexp@>=0.1 <0.1.13": "^0.1.13",
"ajv@>=8 <8.18.0": "^8.18.0",
"socket.io-parser@>=4 <4.2.6": "^4.2.6",
"postcss@>=8 <8.5.10": "^8.5.10",
"yaml@>=2 <2.8.3": "^2.8.3",
"semver@>=5 <5.7.2": "^5.7.2",
"defu@>=6 <6.1.5": "^6.1.5",
"fast-uri@<3.1.2": "^3.1.2",
"js-cookie@<3.0.8": "3.0.8",
"tmp@<0.2.7": "0.2.7",
"brace-expansion@<1.1.13": "1.1.13",
"brace-expansion@>=2 <2.0.3": "2.0.3",
"brace-expansion@>=5 <5.0.6": "5.0.6",
"@jsonhero/json-infer-types>ip-address": "^10.2.0",
"@modelcontextprotocol/sdk@>=1.26.0>express-rate-limit": "^8.6.0"
},
"onlyBuiltDependencies": [
"@depot/cli",
"@fingerprintjs/fingerprintjs-pro-react",
"@prisma/client",
"@prisma/engines",
"@sentry/cli",
"@swc/core",
"better-sqlite3",
"cpu-features",
"esbuild",
"prisma",
"protobufjs",
"sharp",
"ssh2",
"turbo"
]
}
}
Reference in New Issue View Git Blame Copy Permalink