f2babbf637
* Some notes on the new run engine * lockfile with setup for the run engine * Documenting where TaskRun is currently mutated, to try figure out the shape of the new system * Added notes about how triggering currently works * Details about when triggering happens * Lots of notes about waitpoints * Started scaffolding the RunEngine * Sketch of Prisma waitpoint schema while it’s fresh in my mind * Got Prisma working with testcontainers * Use beforeEach/afterEach * Simple Prisma and Redis test * Return Redis options instead of a client * Simplified things * A very simple FIFO pull-based queue to check the tests working properly * Use vitest extend * Separate redis, postgres and combined tests for faster testing * Some fixes and test improvements * Pass a logger into the queue * A queue processor that processes items from the given queue as fast as it can * Test for retrying an item that wasn’t processed * First draft of waitpoints in the Prisma schema * Remove the custom logger from the test * Added a completedAt to Waitpoint * Notes on the flow for an execution starting * Added redlock, moved some files around * Starting point for the TaskRunExecutionSnapshot table * Added relationships to TaskRunExecutionSnapshot * Change some tsconfig * Moved some things around * Added some packages * WIP on the RunQueue * Fix for some imports * Key producer with some tests * Removed the nv type from the keys… it’s not useful to do global queries * Passing unit tests for all the public key producer functions * Some basic tests passing for the RunQueue * Simple enqueue test working * Enqueue and dequeue for dev is working * Don’t log everything during the tests * Enqueuing/dequeuing from the shared queue is working * Tests for getting a shared queue * The key producer sharedQueue can now be named, to allow multiple separate queues * The key producer uses the name of the queue as the input * Extra info in the Prisma schema * Dequeuing a message gets the payload and sets the task concurrency all in one Lua script * Adding more keys so we can read the concurrency from the queue * Setting the concurrency with dequeue and enquque is working * Improved the tests and fixed some bugs * Acking is resetting the concurrencies * Check the key has been removed after acking * Nacking is working * Changed the package to CommonJS + Node10 so it works with Redlock * Moved the database, otel and emails packages to be in internal-packages * Moved some Prisma code to the database package * Started using the RunEngine for triggering * Progress on run engine triggering, first waitpoint code * Create a delay waitpoint * Moved ZodWorker to an internal package so it can be used in the run engine as well as the webapp * Web app now uses the zod worker package * Added parseNaturalLanguageDuration to core/apps * internal-packages/zod-worker in the lockfile * Pass in the master queue, remove old rebalance workers code * Add masterQueue to TaskRun * Fixed the tests * Moved waitpoint code into the run engine, also the zod worker * Completing waitpoints * An experiment to create a new test container with environment * More changes to triggering * Started testing triggering * Test for a run getting triggered and being enqueued * Removed dequeueMessageInEnv * Update dev queue tests to use the shared queue function * Schema changes for TaskRunExecutionSnapshot * First execution snapshot when the run is created. Dequeue run function added to the engine * Separate internal package for testcontainers so they can be used elsewhere * Remove the simple queue and testcontainers from the run-engine. They’re going to be separate * Fix for the wrong path to the Prisma schem,a * Added the testcontainers package to the run-engine * redis-worker package, just a copy of the simple queue for now * The queue now uses Lua to enqueue dequeue * The queue now has a catalog and an invisible period after dequeuing * Added a visibility timeout and acking, with tests * Added more Redis connection logging, deleted todos * Visibility timeouts are now defined on the catalog and can be overridden when enqueuing * Dequeue multiple items at once * Test for dequeuing multiple items * Export some types to be used elsewhere * Partial refactor of the processor * First stab at a worker with concurrency and NodeWorkers * Don’t have a default visibility timeout in the queue * Worker setup and processing items in a simple test * Process jobs in parallel with retrying * Get the attempt when dequeuing * Workers do exponential backoff * Moved todos * DLQ functionality * DLQ tests * Same cluster for all keys in the same queue * Added DLQ tests * Whitespace * Redis pubsub to redrive from the worker * Fixed database paths * Fix for path to zod-worker * Fixes for typecheck errors, mostly with TS versions and module resolution * Redlock required a patch * Moved the new DB migrations to the new database package folder * Remove the run-engine package * Remove the RunEngine prisma schema changes * Delete triggerTaskV2 * Remove zodworker test script (no tests) * Update test-containers readme * Generate the client first * Use a specific version of the prisma package * Generate the prisma client before running the unit tests
198 lines
4.5 KiB
TypeScript
198 lines
4.5 KiB
TypeScript
import {
|
|
Prisma,
|
|
PrismaClient,
|
|
PrismaClientOrTransaction,
|
|
PrismaReplicaClient,
|
|
PrismaTransactionClient,
|
|
PrismaTransactionOptions,
|
|
} from "@trigger.dev/database";
|
|
import invariant from "tiny-invariant";
|
|
import { z } from "zod";
|
|
import { env } from "./env.server";
|
|
import { logger } from "./services/logger.server";
|
|
import { isValidDatabaseUrl } from "./utils/db";
|
|
import { singleton } from "./utils/singleton";
|
|
import { $transaction as transac } from "@trigger.dev/database";
|
|
|
|
export type {
|
|
PrismaTransactionClient,
|
|
PrismaClientOrTransaction,
|
|
PrismaTransactionOptions,
|
|
PrismaReplicaClient,
|
|
};
|
|
|
|
export async function $transaction<R>(
|
|
prisma: PrismaClientOrTransaction,
|
|
fn: (prisma: PrismaTransactionClient) => Promise<R>,
|
|
options?: PrismaTransactionOptions
|
|
): Promise<R | undefined> {
|
|
return transac(
|
|
prisma,
|
|
fn,
|
|
(error) => {
|
|
logger.error("prisma.$transaction error", {
|
|
code: error.code,
|
|
meta: error.meta,
|
|
stack: error.stack,
|
|
message: error.message,
|
|
name: error.name,
|
|
});
|
|
},
|
|
options
|
|
);
|
|
}
|
|
|
|
export { Prisma };
|
|
|
|
export const prisma = singleton("prisma", getClient);
|
|
|
|
export const $replica: PrismaReplicaClient = singleton(
|
|
"replica",
|
|
() => getReplicaClient() ?? prisma
|
|
);
|
|
|
|
function getClient() {
|
|
const { DATABASE_URL } = process.env;
|
|
invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
|
|
|
|
const databaseUrl = extendQueryParams(DATABASE_URL, {
|
|
connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
|
|
pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
|
|
});
|
|
|
|
console.log(`🔌 setting up prisma client to ${redactUrlSecrets(databaseUrl)}`);
|
|
|
|
const client = new PrismaClient({
|
|
datasources: {
|
|
db: {
|
|
url: databaseUrl.href,
|
|
},
|
|
},
|
|
// @ts-expect-error
|
|
log: [
|
|
{
|
|
emit: "stdout",
|
|
level: "error",
|
|
},
|
|
{
|
|
emit: "stdout",
|
|
level: "info",
|
|
},
|
|
{
|
|
emit: "stdout",
|
|
level: "warn",
|
|
},
|
|
].concat(
|
|
process.env.VERBOSE_PRISMA_LOGS === "1"
|
|
? [
|
|
{ emit: "event", level: "query" },
|
|
{ emit: "stdout", level: "query" },
|
|
]
|
|
: []
|
|
),
|
|
});
|
|
|
|
// connect eagerly
|
|
client.$connect();
|
|
|
|
console.log(`🔌 prisma client connected`);
|
|
|
|
return client;
|
|
}
|
|
|
|
function getReplicaClient() {
|
|
if (!env.DATABASE_READ_REPLICA_URL) {
|
|
console.log(`🔌 No database replica, using the regular client`);
|
|
return;
|
|
}
|
|
|
|
const replicaUrl = extendQueryParams(env.DATABASE_READ_REPLICA_URL, {
|
|
connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
|
|
pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
|
|
});
|
|
|
|
console.log(`🔌 setting up read replica connection to ${redactUrlSecrets(replicaUrl)}`);
|
|
|
|
const replicaClient = new PrismaClient({
|
|
datasources: {
|
|
db: {
|
|
url: replicaUrl.href,
|
|
},
|
|
},
|
|
// @ts-expect-error
|
|
log: [
|
|
{
|
|
emit: "stdout",
|
|
level: "error",
|
|
},
|
|
{
|
|
emit: "stdout",
|
|
level: "info",
|
|
},
|
|
{
|
|
emit: "stdout",
|
|
level: "warn",
|
|
},
|
|
].concat(
|
|
process.env.VERBOSE_PRISMA_LOGS === "1"
|
|
? [
|
|
{ emit: "event", level: "query" },
|
|
{ emit: "stdout", level: "query" },
|
|
]
|
|
: []
|
|
),
|
|
});
|
|
|
|
// connect eagerly
|
|
replicaClient.$connect();
|
|
|
|
console.log(`🔌 read replica connected`);
|
|
|
|
return replicaClient;
|
|
}
|
|
|
|
function extendQueryParams(hrefOrUrl: string | URL, queryParams: Record<string, string>) {
|
|
const url = new URL(hrefOrUrl);
|
|
const query = url.searchParams;
|
|
|
|
for (const [key, val] of Object.entries(queryParams)) {
|
|
query.set(key, val);
|
|
}
|
|
|
|
url.search = query.toString();
|
|
|
|
return url;
|
|
}
|
|
|
|
function redactUrlSecrets(hrefOrUrl: string | URL) {
|
|
const url = new URL(hrefOrUrl);
|
|
url.password = "";
|
|
return url.href;
|
|
}
|
|
|
|
export type { PrismaClient } from "@trigger.dev/database";
|
|
|
|
export const PrismaErrorSchema = z.object({
|
|
code: z.string(),
|
|
});
|
|
|
|
function getDatabaseSchema() {
|
|
if (!isValidDatabaseUrl(env.DATABASE_URL)) {
|
|
throw new Error("Invalid Database URL");
|
|
}
|
|
|
|
const databaseUrl = new URL(env.DATABASE_URL);
|
|
const schemaFromSearchParam = databaseUrl.searchParams.get("schema");
|
|
|
|
if (!schemaFromSearchParam) {
|
|
console.debug("❗ database schema unspecified, will default to `public` schema");
|
|
return "public";
|
|
}
|
|
|
|
return schemaFromSearchParam;
|
|
}
|
|
|
|
export const DATABASE_SCHEMA = singleton("DATABASE_SCHEMA", getDatabaseSchema);
|
|
|
|
export const sqlDatabaseSchema = Prisma.sql([`${DATABASE_SCHEMA}`]);
|