68e88d0d71
This allows seamless migration to different object storage. Existing runs that have offloaded payloads/outputs will continue to use the default object store (configured using `OBJECT_STORE_*` env vars). You can add additional stores by setting new env vars: - `OBJECT_STORE_DEFAULT_PROTOCOL` this determines where new run large payloads will get stored. - If you set that you need to set new env vars for that protocol. Example: ``` OBJECT_STORE_DEFAULT_PROTOCOL=“s3" OBJECT_STORE_S3_BASE_URL=https://s3.us-east-1.amazonaws.com OBJECT_STORE_S3_ACCESS_KEY_ID=<val> OBJECT_STORE_S3_SECRET_ACCESS_KEY=<val> OBJECT_STORE_S3_REGION=us-east-1 OBJECT_STORE_S3_SERVICE=s3 ``` --------- Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>
273 lines
6.5 KiB
YAML
273 lines
6.5 KiB
YAML
version: "3"
|
|
|
|
volumes:
|
|
database-data:
|
|
database-data-alt:
|
|
redis-data:
|
|
minio-data:
|
|
clickhouse-data:
|
|
clickhouse-logs:
|
|
prometheus-data:
|
|
grafana-data:
|
|
|
|
networks:
|
|
app_network:
|
|
external: false
|
|
|
|
services:
|
|
database:
|
|
container_name: database
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.postgres
|
|
restart: always
|
|
volumes:
|
|
- ${DB_VOLUME:-database-data}:/var/lib/postgresql/data/
|
|
environment:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: postgres
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- 5432:5432
|
|
command:
|
|
- -c
|
|
- listen_addresses=*
|
|
- -c
|
|
- wal_level=logical
|
|
- -c
|
|
- shared_preload_libraries=pg_partman_bgw
|
|
|
|
redis:
|
|
container_name: redis
|
|
image: redis:7
|
|
restart: always
|
|
volumes:
|
|
- redis-data:/data
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- 6379:6379
|
|
|
|
# S3-compatible API for local object store (large payloads / packet offload).
|
|
# Host :9005 = S3 API, :9006 = web console (ClickHouse uses host :9000).
|
|
minio:
|
|
container_name: minio
|
|
image: minio/minio:latest
|
|
restart: always
|
|
command: server /data --console-address ":9001"
|
|
environment:
|
|
MINIO_ROOT_USER: minioadmin
|
|
MINIO_ROOT_PASSWORD: minioadmin
|
|
volumes:
|
|
- minio-data:/data
|
|
ports:
|
|
- "9005:9000"
|
|
- "9006:9001"
|
|
networks:
|
|
- app_network
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 5s
|
|
|
|
minio-init:
|
|
image: minio/mc:latest
|
|
depends_on:
|
|
minio:
|
|
condition: service_healthy
|
|
networks:
|
|
- app_network
|
|
entrypoint: /bin/sh
|
|
command:
|
|
- -c
|
|
- |
|
|
mc alias set local http://minio:9000 minioadmin minioadmin
|
|
mc mb -p local/packets || true
|
|
restart: "no"
|
|
|
|
electric:
|
|
container_name: electric
|
|
image: electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36
|
|
restart: always
|
|
environment:
|
|
DATABASE_URL: postgresql://postgres:postgres@database:5432/postgres?sslmode=disable
|
|
ELECTRIC_INSECURE: true
|
|
ELECTRIC_ENABLE_INTEGRATION_TESTING: true
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- "3060:3000"
|
|
depends_on:
|
|
- database
|
|
|
|
electric-shard-1:
|
|
container_name: electric-shard-1
|
|
image: electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36
|
|
restart: always
|
|
environment:
|
|
DATABASE_URL: postgresql://postgres:postgres@database:5432/postgres?sslmode=disable
|
|
ELECTRIC_INSECURE: true
|
|
ELECTRIC_REPLICATION_STREAM_ID: "triggershard1"
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- "3061:3000"
|
|
depends_on:
|
|
- database
|
|
|
|
clickhouse:
|
|
image: clickhouse/clickhouse-server:25.6.2
|
|
restart: always
|
|
container_name: clickhouse
|
|
ulimits:
|
|
nofile:
|
|
soft: 262144
|
|
hard: 262144
|
|
environment:
|
|
CLICKHOUSE_USER: default
|
|
CLICKHOUSE_PASSWORD: password
|
|
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
|
ports:
|
|
- "8123:8123"
|
|
- "9000:9000"
|
|
volumes:
|
|
- clickhouse-data:/var/lib/clickhouse
|
|
- clickhouse-logs:/var/log/clickhouse-server
|
|
networks:
|
|
- app_network
|
|
healthcheck:
|
|
test:
|
|
[
|
|
"CMD",
|
|
"clickhouse-client",
|
|
"--host",
|
|
"localhost",
|
|
"--port",
|
|
"9000",
|
|
"--user",
|
|
"default",
|
|
"--password",
|
|
"password",
|
|
"--query",
|
|
"SELECT 1",
|
|
]
|
|
interval: "3s"
|
|
timeout: "5s"
|
|
retries: "5"
|
|
start_period: "10s"
|
|
|
|
clickhouse_migrator:
|
|
build:
|
|
context: ../internal-packages/clickhouse
|
|
dockerfile: ./Dockerfile
|
|
depends_on:
|
|
clickhouse:
|
|
condition: service_healthy
|
|
networks:
|
|
- app_network
|
|
command: ["goose", "${GOOSE_COMMAND:-up}"]
|
|
|
|
ch-ui:
|
|
image: ghcr.io/caioricciuti/ch-ui:latest
|
|
restart: always
|
|
ports:
|
|
- "5521:5521"
|
|
environment:
|
|
VITE_CLICKHOUSE_URL: "http://localhost:8123"
|
|
VITE_CLICKHOUSE_USER: "default"
|
|
VITE_CLICKHOUSE_PASS: "password"
|
|
networks:
|
|
- app_network
|
|
|
|
s2:
|
|
image: ghcr.io/s2-streamstore/s2
|
|
command: ["lite", "--init-file", "/s2-spec.json"]
|
|
volumes:
|
|
- ./config/s2-spec.json:/s2-spec.json:ro
|
|
ports:
|
|
- "4566:80"
|
|
networks:
|
|
- app_network
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "wget -qO- http://localhost:80/v1/basins?limit=1 || exit 1"]
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 5
|
|
start_period: 3s
|
|
|
|
toxiproxy:
|
|
container_name: toxiproxy
|
|
image: ghcr.io/shopify/toxiproxy:latest
|
|
restart: always
|
|
volumes:
|
|
- ./config/toxiproxy.json:/config/toxiproxy.json
|
|
ports:
|
|
- "30303:30303" # Proxied webapp port
|
|
- "8474:8474" # Toxiproxy API port
|
|
networks:
|
|
- app_network
|
|
command: ["-host", "0.0.0.0", "-config", "/config/toxiproxy.json"]
|
|
|
|
nginx-h2:
|
|
image: nginx:1.27
|
|
container_name: nginx-h2
|
|
restart: unless-stopped
|
|
ports:
|
|
- "8443:8443"
|
|
volumes:
|
|
- ./config/nginx.conf:/etc/nginx/nginx.conf:ro
|
|
- ./config/certs:/etc/nginx/certs:ro
|
|
|
|
# Observability stack for local development
|
|
otel-collector:
|
|
container_name: otel-collector
|
|
image: otel/opentelemetry-collector-contrib:0.96.0
|
|
restart: always
|
|
command: ["--config", "/etc/otel-collector-config.yaml"]
|
|
volumes:
|
|
- ./config/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
|
|
ports:
|
|
- "4317:4317" # OTLP gRPC receiver
|
|
- "4318:4318" # OTLP HTTP receiver
|
|
- "8889:8889" # Prometheus exporter
|
|
networks:
|
|
- app_network
|
|
|
|
prometheus:
|
|
container_name: prometheus
|
|
image: prom/prometheus:v2.54.1
|
|
restart: always
|
|
volumes:
|
|
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
|
- prometheus-data:/prometheus
|
|
ports:
|
|
- "9090:9090"
|
|
networks:
|
|
- app_network
|
|
command:
|
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
|
- "--storage.tsdb.path=/prometheus"
|
|
- "--web.enable-lifecycle"
|
|
|
|
grafana:
|
|
container_name: grafana
|
|
image: grafana/grafana:11.3.0
|
|
restart: always
|
|
volumes:
|
|
- grafana-data:/var/lib/grafana
|
|
- ./config/grafana/provisioning:/etc/grafana/provisioning:ro
|
|
ports:
|
|
- "3001:3000"
|
|
environment:
|
|
GF_SECURITY_ADMIN_USER: admin
|
|
GF_SECURITY_ADMIN_PASSWORD: admin
|
|
GF_USERS_ALLOW_SIGN_UP: false
|
|
networks:
|
|
- app_network
|
|
depends_on:
|
|
- prometheus
|