Daniel Sutton
bea7e2be90
## Summary Run-graph data (runs, batches, waitpoints, and their related tables) can now live in a database separate from the control plane, with every read and write routed to the correct database by each run's residency. This makes reading and writing run data more reliable once the two are split, and is a no-op for single-database installs. ## Design - Run-graph table access goes through the run-store router, which selects the legacy or the new run-ops store per run instead of assuming one shared client. - The legacy run-ops client is now independently pointable, so legacy run data can be served from its own database (and replica) rather than the control-plane connection. - Run-graph writes go straight to the run-graph database instead of being forwarded through the control plane, and replication targets are split so runs in the new database still replicate to analytics without under-counting. - Read-through slots refuse the control-plane client, so a missing residency fails loudly instead of silently reading the wrong database. - Migration `20260710120000_drop_remaining_run_graph_seam_foreign_keys` drops the foreign keys that still crossed the run-graph / control-plane seam, which is what lets the two live in separate databases. The split stays off unless explicitly enabled and the two databases are confirmed physically distinct; startup fails closed otherwise. Verified by running the full dashboard end-to-end suite against both a single-database configuration and a three-database configuration (control plane, the new database, and a physically separate legacy database), with runs on both residencies. No misrouted reads in either configuration.
95 lines
3.6 KiB
Bash
Executable File
95 lines
3.6 KiB
Bash
Executable File
#!/bin/sh
|
||
set -xe
|
||
|
||
if [ -n "$DATABASE_HOST" ]; then
|
||
scripts/wait-for-it.sh ${DATABASE_HOST} -- echo "database is up"
|
||
fi
|
||
|
||
if [ "$SKIP_POSTGRES_MIGRATIONS" != "1" ]; then
|
||
echo "Running prisma migrations"
|
||
pnpm --filter @trigger.dev/database db:migrate:deploy
|
||
echo "Prisma migrations done"
|
||
else
|
||
echo "SKIP_POSTGRES_MIGRATIONS=1, skipping Postgres migrations."
|
||
fi
|
||
|
||
# Run-ops split: migrate the dedicated NEW run-ops database only when it is configured. Single-DB
|
||
# installs never set the URL, so this is a no-op there.
|
||
if [ -n "$RUN_OPS_DATABASE_URL" ]; then
|
||
if [ "$SKIP_RUN_OPS_MIGRATIONS" != "1" ]; then
|
||
echo "Running run-ops migrations"
|
||
pnpm --filter @internal/run-ops-database db:migrate:deploy
|
||
echo "Run-ops migrations done"
|
||
else
|
||
echo "SKIP_RUN_OPS_MIGRATIONS=1, skipping run-ops migrations."
|
||
fi
|
||
else
|
||
echo "RUN_OPS_DATABASE_URL not set, skipping run-ops migrations."
|
||
fi
|
||
|
||
# Run-ops split: keep the legacy runs DB's schema current by applying the full @trigger.dev/database
|
||
# migrations to it too, pointed at its direct (non-pooled) URL. Only runs when that URL is configured;
|
||
# installs that never set it skip this entirely.
|
||
if [ -n "$RUN_OPS_LEGACY_DIRECT_URL" ]; then
|
||
if [ "$SKIP_RUN_OPS_LEGACY_MIGRATIONS" != "1" ]; then
|
||
echo "Running legacy run-ops migrations"
|
||
# Subshell with tracing off so `set -x` does not print the DSN (with credentials) to the logs.
|
||
(set +x; DATABASE_URL="$RUN_OPS_LEGACY_DIRECT_URL" DIRECT_URL="$RUN_OPS_LEGACY_DIRECT_URL" pnpm --filter @trigger.dev/database db:migrate:deploy)
|
||
echo "Legacy run-ops migrations done"
|
||
else
|
||
echo "SKIP_RUN_OPS_LEGACY_MIGRATIONS=1, skipping legacy run-ops migrations."
|
||
fi
|
||
else
|
||
echo "RUN_OPS_LEGACY_DIRECT_URL not set, skipping legacy run-ops migrations."
|
||
fi
|
||
|
||
if [ "$SKIP_DASHBOARD_AGENT_MIGRATIONS" != "1" ]; then
|
||
echo "Running dashboard agent migrations"
|
||
pnpm --filter @internal/dashboard-agent-db db:migrate:deploy
|
||
echo "Dashboard agent migrations done"
|
||
else
|
||
echo "SKIP_DASHBOARD_AGENT_MIGRATIONS=1, skipping dashboard agent migrations."
|
||
fi
|
||
|
||
if [ -n "$CLICKHOUSE_URL" ] && [ "$SKIP_CLICKHOUSE_MIGRATIONS" != "1" ]; then
|
||
# Run ClickHouse migrations
|
||
echo "Running ClickHouse migrations..."
|
||
export GOOSE_DRIVER=clickhouse
|
||
|
||
# Ensure secure=true is in the connection string
|
||
if echo "$CLICKHOUSE_URL" | grep -q "secure="; then
|
||
# secure parameter already exists, use as is
|
||
export GOOSE_DBSTRING="$CLICKHOUSE_URL"
|
||
elif echo "$CLICKHOUSE_URL" | grep -q "?"; then
|
||
# URL has query parameters, append secure=true
|
||
export GOOSE_DBSTRING="${CLICKHOUSE_URL}&secure=true"
|
||
else
|
||
# URL has no query parameters, add secure=true
|
||
export GOOSE_DBSTRING="${CLICKHOUSE_URL}?secure=true"
|
||
fi
|
||
|
||
export GOOSE_MIGRATION_DIR=/triggerdotdev/internal-packages/clickhouse/schema
|
||
/usr/local/bin/goose up
|
||
echo "ClickHouse migrations complete."
|
||
elif [ "$SKIP_CLICKHOUSE_MIGRATIONS" = "1" ]; then
|
||
echo "SKIP_CLICKHOUSE_MIGRATIONS=1, skipping ClickHouse migrations."
|
||
else
|
||
echo "CLICKHOUSE_URL not set, skipping ClickHouse migrations."
|
||
fi
|
||
|
||
# Copy over required prisma files
|
||
cp internal-packages/database/prisma/schema.prisma apps/webapp/prisma/
|
||
cp node_modules/@prisma/engines/*.node apps/webapp/prisma/
|
||
|
||
cd /triggerdotdev/apps/webapp
|
||
|
||
|
||
# Decide how much old-space memory Node should get.
|
||
# Use $NODE_MAX_OLD_SPACE_SIZE if it’s set; otherwise fall back to 8192.
|
||
MAX_OLD_SPACE_SIZE="${NODE_MAX_OLD_SPACE_SIZE:-8192}"
|
||
|
||
echo "Setting max old space size to ${MAX_OLD_SPACE_SIZE}"
|
||
|
||
NODE_PATH='/triggerdotdev/node_modules/.pnpm/node_modules' exec dumb-init node --max-old-space-size=${MAX_OLD_SPACE_SIZE} ./build/server.js
|
||
|