Eric Allam
f9d57d3bd5
## Summary Adds a second backend for the realtime runs feed (`useRealtimeRun`, `subscribeToRunsWithTag`, `subscribeToBatch`), built to stay healthy when a single busy environment has many subscribers watching many runs at once. It is gated behind a feature flag with the existing backend as the default, so nothing changes for users until it is enabled per environment. ## Design A run change is published once, as a small self-describing record, to a single per-environment channel. Every feed is then a predicate over that one stream rather than owning a channel: - A per-instance router indexes the currently-held feeds by run, tag, and batch. When a run changes it hydrates the affected rows once and serializes them once, then fans the result to every matching feed. One hot shared tag watched by many subscribers costs a single database query and serialize, not one per subscriber. - Feeds that don't match a change are never woken, wake delivery per environment is coalesced on a leading edge (250ms default) so a burst of changes costs one wake, and cold reads coalesce onto a single short-TTL-cached resolve. - An admission gate bounds how many cold ClickHouse resolves run concurrently, so a mass reconnect across many distinct filters queues instead of stampeding the database. - Changes that land while a client is between long-polls are delivered on its next poll instead of waiting for the periodic backstop: each environment buffers its recent change records, subscriptions linger briefly after the last feed closes, and a newly-armed poll replays exactly the connection's gap. - The per-connection replay cursors behind that are shared across instances via Redis (a single timestamp each), so a poll landing on a different instance behind the load balancer still reads the connection's true gap instead of falling back to a cold resolve. Cursor reads have a bounded deadline and degrade to the cold-read path on any Redis trouble. - Tag subscriptions with multiple tags match runs carrying all of the tags, mirroring the existing backend's filter semantics, and live long-polls hold for about 20 seconds to match its cadence. - The per-environment channel supports Redis Cluster sharded pub/sub, so the wake path scales horizontally across shards by environment. - The backend reports its health through OpenTelemetry metrics (delivery lag, poll resolution paths, backstop outcomes, replay and cursor-store activity), with a provisioned Grafana dashboard for local development. Everything is behind the feature flag and tunable via env vars; the existing backend remains the default.
125 lines
4.3 KiB
YAML
125 lines
4.3 KiB
YAML
# Optional services for advanced local-dev workflows. Pair with
|
|
# `docker-compose.yml` via `pnpm run docker:full`.
|
|
#
|
|
# Same `name:` so `docker compose` treats both files as one project — bring
|
|
# them up together and they share the `app_network` and `triggerdotdev-docker`
|
|
# volume namespace. Tear down with `pnpm run docker:full:stop`.
|
|
#
|
|
# Includes:
|
|
# - electric-shard-1: second Electric instance for multi-shard testing
|
|
# - ch-ui: ClickHouse browser UI
|
|
# - toxiproxy: chaos / flake testing
|
|
# - nginx-h2: HTTP/2 reverse proxy
|
|
# - otel-collector + prometheus + grafana: observability stack
|
|
name: triggerdotdev-docker
|
|
|
|
volumes:
|
|
prometheus-data:
|
|
grafana-data:
|
|
|
|
networks:
|
|
app_network:
|
|
external: false
|
|
|
|
services:
|
|
electric-shard-1:
|
|
container_name: ${CONTAINER_PREFIX:-}electric-shard-1
|
|
image: electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36
|
|
restart: always
|
|
environment:
|
|
DATABASE_URL: postgresql://postgres:postgres@database:5432/postgres?sslmode=disable
|
|
ELECTRIC_INSECURE: true
|
|
ELECTRIC_REPLICATION_STREAM_ID: "triggershard1"
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- "${ELECTRIC_SHARD_1_HOST_PORT:-3061}:3000"
|
|
depends_on:
|
|
- database
|
|
|
|
ch-ui:
|
|
image: ghcr.io/caioricciuti/ch-ui:latest@sha256:288abf7103d6e0f45527ee835ee79bd1e0bfb82d55dc514fb3cf43816306b538
|
|
restart: always
|
|
ports:
|
|
- "${CH_UI_HOST_PORT:-5521}:5521"
|
|
environment:
|
|
VITE_CLICKHOUSE_URL: "http://localhost:${CLICKHOUSE_HTTP_HOST_PORT:-8123}"
|
|
VITE_CLICKHOUSE_USER: "default"
|
|
VITE_CLICKHOUSE_PASS: "password"
|
|
networks:
|
|
- app_network
|
|
|
|
toxiproxy:
|
|
container_name: ${CONTAINER_PREFIX:-}toxiproxy
|
|
image: ghcr.io/shopify/toxiproxy:latest@sha256:9378ed52a28bc50edc1350f936f518f31fa95f0d15917d6eb40b8e376d1a214e
|
|
restart: always
|
|
volumes:
|
|
- ./config/toxiproxy.json:/config/toxiproxy.json
|
|
ports:
|
|
- "${TOXIPROXY_PROXY_HOST_PORT:-30303}:30303" # Proxied webapp port
|
|
- "${TOXIPROXY_API_HOST_PORT:-8474}:8474" # Toxiproxy API port
|
|
networks:
|
|
- app_network
|
|
command: ["-host", "0.0.0.0", "-config", "/config/toxiproxy.json"]
|
|
|
|
nginx-h2:
|
|
image: nginx:1.27@sha256:6784fb0834aa7dbbe12e3d7471e69c290df3e6ba810dc38b34ae33d3c1c05f7d
|
|
container_name: ${CONTAINER_PREFIX:-}nginx-h2
|
|
restart: unless-stopped
|
|
ports:
|
|
- "${NGINX_H2_HOST_PORT:-8443}:8443"
|
|
volumes:
|
|
- ./config/nginx.conf:/etc/nginx/nginx.conf:ro
|
|
- ./config/certs:/etc/nginx/certs:ro
|
|
networks:
|
|
- app_network
|
|
|
|
# Observability stack for local development
|
|
otel-collector:
|
|
container_name: ${CONTAINER_PREFIX:-}otel-collector
|
|
image: otel/opentelemetry-collector-contrib:0.96.0@sha256:7ef2a2ff46b9e432321fdd63df104bfeedaf7b4e276950f42c634d0f23521fc4
|
|
restart: always
|
|
command: ["--config", "/etc/otel-collector-config.yaml"]
|
|
volumes:
|
|
- ./config/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
|
|
ports:
|
|
- "${OTEL_GRPC_HOST_PORT:-4317}:4317" # OTLP gRPC receiver
|
|
- "${OTEL_HTTP_HOST_PORT:-4318}:4318" # OTLP HTTP receiver
|
|
- "${OTEL_PROMETHEUS_HOST_PORT:-8889}:8889" # Prometheus exporter
|
|
networks:
|
|
- app_network
|
|
|
|
prometheus:
|
|
container_name: ${CONTAINER_PREFIX:-}prometheus
|
|
image: prom/prometheus:v2.54.1@sha256:f6639335d34a77d9d9db382b92eeb7fc00934be8eae81dbc03b31cfe90411a94
|
|
restart: always
|
|
volumes:
|
|
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
|
- prometheus-data:/prometheus
|
|
ports:
|
|
- "${PROMETHEUS_HOST_PORT:-9090}:9090"
|
|
networks:
|
|
- app_network
|
|
command:
|
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
|
- "--storage.tsdb.path=/prometheus"
|
|
- "--web.enable-lifecycle"
|
|
|
|
grafana:
|
|
container_name: ${CONTAINER_PREFIX:-}grafana
|
|
image: grafana/grafana:11.3.0@sha256:a0f881232a6fb71a0554a47d0fe2203b6888fe77f4cefb7ea62bed7eb54e13c3
|
|
restart: always
|
|
volumes:
|
|
- grafana-data:/var/lib/grafana
|
|
- ./config/grafana/provisioning:/etc/grafana/provisioning:ro
|
|
ports:
|
|
- "${GRAFANA_HOST_PORT:-4001}:3000"
|
|
environment:
|
|
GF_SECURITY_ADMIN_USER: admin
|
|
GF_SECURITY_ADMIN_PASSWORD: admin
|
|
GF_USERS_ALLOW_SIGN_UP: false
|
|
networks:
|
|
- app_network
|
|
depends_on:
|
|
- prometheus
|