Matt Aitken
f38d35e9f6
* Add retry logic for insert operations Add a generic retry mechanism for task run and payload inserts to handle transient connection errors. The new #insertWithRetry method retries up to three times with exponential backoff and jitter on retryable connection errors such as connection resets or timeouts. Errors are logged and recorded in tracing spans to improve observability and robustness of the replication service. * Replication settings are configurable * Log out the runIds for failed batches * Detecting bad JSON in run replication and ignoring it * Reproduced split unicode error * Move output file * Massively improved the performance * Minor performance improvements * Unskip tests * Remove unused test in CH package * Fix for the ClickHouse UI explorer * RunReplication keepAlive defaults to false * Add concurrency_key and bulk_action_group_ids to ClickHouse task runs * ClickHouse package doesn't need to be built anymore for the webapp * Set the concurrency_key from the run replication service
132 lines
2.9 KiB
YAML
132 lines
2.9 KiB
YAML
version: "3"
|
|
|
|
volumes:
|
|
database-data:
|
|
database-data-alt:
|
|
redis-data:
|
|
clickhouse:
|
|
|
|
networks:
|
|
app_network:
|
|
external: false
|
|
|
|
services:
|
|
database:
|
|
container_name: database
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.postgres
|
|
restart: always
|
|
volumes:
|
|
- ${DB_VOLUME:-database-data}:/var/lib/postgresql/data/
|
|
environment:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: postgres
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- 5432:5432
|
|
command:
|
|
- -c
|
|
- listen_addresses=*
|
|
- -c
|
|
- wal_level=logical
|
|
- -c
|
|
- shared_preload_libraries=pg_partman_bgw
|
|
|
|
redis:
|
|
container_name: redis
|
|
image: redis:7
|
|
restart: always
|
|
volumes:
|
|
- redis-data:/data
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- 6379:6379
|
|
|
|
electric:
|
|
container_name: electric
|
|
image: electricsql/electric:1.0.13@sha256:4e69c4a6ec3e976efbdd8b7e6de427e771aeacdbc0c8c7ca22eb0ca6ab1611ff
|
|
restart: always
|
|
environment:
|
|
DATABASE_URL: postgresql://postgres:postgres@database:5432/postgres?sslmode=disable
|
|
ELECTRIC_INSECURE: true
|
|
networks:
|
|
- app_network
|
|
ports:
|
|
- "3060:3000"
|
|
depends_on:
|
|
- database
|
|
|
|
clickhouse:
|
|
image: bitnami/clickhouse:latest
|
|
container_name: clickhouse
|
|
environment:
|
|
CLICKHOUSE_ADMIN_USER: default
|
|
CLICKHOUSE_ADMIN_PASSWORD: password
|
|
ports:
|
|
- "8123:8123"
|
|
- "9000:9000"
|
|
volumes:
|
|
- clickhouse:/bitnami/clickhouse
|
|
networks:
|
|
- app_network
|
|
healthcheck:
|
|
test:
|
|
[
|
|
"CMD",
|
|
"clickhouse-client",
|
|
"--host",
|
|
"localhost",
|
|
"--port",
|
|
"9000",
|
|
"--user",
|
|
"default",
|
|
"--password",
|
|
"password",
|
|
"--query",
|
|
"SELECT 1",
|
|
]
|
|
interval: 3s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 10s
|
|
|
|
clickhouse_migrator:
|
|
build:
|
|
context: ../internal-packages/clickhouse
|
|
dockerfile: ./Dockerfile
|
|
depends_on:
|
|
clickhouse:
|
|
condition: service_healthy
|
|
networks:
|
|
- app_network
|
|
command: ["goose", "${GOOSE_COMMAND:-up}"]
|
|
|
|
ch-ui:
|
|
image: ghcr.io/caioricciuti/ch-ui:latest
|
|
restart: always
|
|
ports:
|
|
- "5521:5521"
|
|
environment:
|
|
VITE_CLICKHOUSE_URL: "http://localhost:8123"
|
|
VITE_CLICKHOUSE_USER: "default"
|
|
VITE_CLICKHOUSE_PASS: "password"
|
|
networks:
|
|
- app_network
|
|
|
|
# otel-collector:
|
|
# container_name: otel-collector
|
|
# image: otel/opentelemetry-collector-contrib:latest
|
|
# restart: always
|
|
# command: ["--config", "/etc/otel-collector-config.yaml"]
|
|
# volumes:
|
|
# - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
|
|
# ports:
|
|
# - "55680:55680"
|
|
# - "55681:55681"
|
|
# - "4317:4317" # OTLP gRPC receiver
|
|
# - "4318:4318" # OTLP http receiver
|