Eric Allam
b38405cb88
* Add createdAt filter to realtime subscribing with tags * Filter realtime colums and expose ability to skip some columns * Add sharding support for electric * Use unkey cache for the created at filter caching * Remove 2 unused indexes on TaskRun * Run list now filters by a single runtime environment * Remove project ID indexes * Use clickhouse in task list aggregation queries instead of pg (keep pg for self-hosters) * WIP clickhouse powered runs list stuff * Improve the query to get the latest tasks for the task list presenter * Update the usage task list to use clickhouse * Implement next runs list powered by clickhouse * Add new index for TaskRun for the runs list, by environment ID * Add runTags gin index * Handle possibly malicious inputs * Ignore claude settings * Better handling not finding an environment on the schedule page * Use ms since epoch in test, not seconds * Remove unused function * Fix test * Use an env var for the realtime maximum createdAt filter duration (defaults to 1 day) * Fixed the query builder to correct the group by / order by order * Make sure runs.list still works * Create small-birds-arrive.md
55 lines
1.4 KiB
TypeScript
55 lines
1.4 KiB
TypeScript
import { ClickHouse } from "@internal/clickhouse";
|
|
import { RedisOptions } from "@internal/redis";
|
|
import { PrismaClient } from "~/db.server";
|
|
import { RunsReplicationService } from "~/services/runsReplicationService.server";
|
|
import { afterEach } from "vitest";
|
|
|
|
export async function setupClickhouseReplication({
|
|
prisma,
|
|
databaseUrl,
|
|
clickhouseUrl,
|
|
redisOptions,
|
|
}: {
|
|
prisma: PrismaClient;
|
|
databaseUrl: string;
|
|
clickhouseUrl: string;
|
|
redisOptions: RedisOptions;
|
|
}) {
|
|
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
|
|
|
const clickhouse = new ClickHouse({
|
|
url: clickhouseUrl,
|
|
name: "runs-replication",
|
|
compression: {
|
|
request: true,
|
|
},
|
|
});
|
|
|
|
const runsReplicationService = new RunsReplicationService({
|
|
clickhouse,
|
|
pgConnectionUrl: databaseUrl,
|
|
serviceName: "runs-replication",
|
|
slotName: "task_runs_to_clickhouse_v1",
|
|
publicationName: "task_runs_to_clickhouse_v1_publication",
|
|
redisOptions,
|
|
maxFlushConcurrency: 1,
|
|
flushIntervalMs: 100,
|
|
flushBatchSize: 1,
|
|
leaderLockTimeoutMs: 5000,
|
|
leaderLockExtendIntervalMs: 1000,
|
|
ackIntervalSeconds: 5,
|
|
});
|
|
|
|
await runsReplicationService.start();
|
|
|
|
// Runs after each test in the current context
|
|
afterEach(async () => {
|
|
// Clean up resources here
|
|
await runsReplicationService.stop();
|
|
});
|
|
|
|
return {
|
|
clickhouse,
|
|
};
|
|
}
|