Files
triggerdotdev--trigger.dev/apps/webapp/app/db.server.ts
T
Matt Aitken f2babbf637 Internal packages (testcontainers, redis-worker and zod-worker) (#1392)
* Some notes on the new run engine

* lockfile with setup for the run engine

* Documenting where TaskRun is currently mutated, to try figure out the shape of the new system

* Added notes about how triggering currently works

* Details about when triggering happens

* Lots of notes about waitpoints

* Started scaffolding the RunEngine

* Sketch of Prisma waitpoint schema while it’s fresh in my mind

* Got Prisma working with testcontainers

* Use beforeEach/afterEach

* Simple Prisma and Redis test

* Return Redis options instead of a client

* Simplified things

* A very simple FIFO pull-based queue to check the tests working properly

* Use vitest extend

* Separate redis, postgres and combined tests for faster testing

* Some fixes and test improvements

* Pass a logger into the queue

* A queue processor that processes items from the given queue as fast as it can

* Test for retrying an item that wasn’t processed

* First draft of waitpoints in the Prisma schema

* Remove the custom logger from the test

* Added a completedAt to Waitpoint

* Notes on the flow for an execution starting

* Added redlock, moved some files around

* Starting point for the TaskRunExecutionSnapshot table

* Added relationships to TaskRunExecutionSnapshot

* Change some tsconfig

* Moved some things around

* Added some packages

* WIP on the RunQueue

* Fix for some imports

* Key producer with some tests

* Removed the nv type from the keys… it’s not useful to do global queries

* Passing unit tests for all the public key producer functions

* Some basic tests passing for the RunQueue

* Simple enqueue test working

* Enqueue and dequeue for dev is working

* Don’t log everything during the tests

* Enqueuing/dequeuing from the shared queue is working

* Tests for getting a shared queue

* The key producer sharedQueue can now be named, to allow multiple separate queues

* The key producer uses the name of the queue as the input

* Extra info in the Prisma schema

* Dequeuing a message gets the payload and sets the task concurrency all in one Lua script

* Adding more keys so we can read the concurrency from the queue

* Setting the concurrency with dequeue and enquque is working

* Improved the tests and fixed some bugs

* Acking is resetting the concurrencies

* Check the key has been removed after acking

* Nacking is working

* Changed the package to CommonJS + Node10 so it works with Redlock

* Moved the database, otel and emails packages to be in internal-packages

* Moved some Prisma code to the database package

* Started using the RunEngine for triggering

* Progress on run engine triggering, first waitpoint code

* Create a delay waitpoint

* Moved ZodWorker to an internal package so it can be used in the run engine as well as the webapp

* Web app now uses the zod worker package

* Added parseNaturalLanguageDuration to core/apps

* internal-packages/zod-worker in the lockfile

* Pass in the master queue, remove old rebalance workers code

* Add masterQueue to TaskRun

* Fixed the tests

* Moved waitpoint code into the run engine, also the zod worker

* Completing waitpoints

* An experiment to create a new test container with environment

* More changes to triggering

* Started testing triggering

* Test for a run getting triggered and being enqueued

* Removed dequeueMessageInEnv

* Update dev queue tests to use the shared queue function

* Schema changes for TaskRunExecutionSnapshot

* First execution snapshot when the run is created. Dequeue run function added to the engine

* Separate internal package for testcontainers so they can be used elsewhere

* Remove the simple queue and testcontainers from the run-engine. They’re going to be separate

* Fix for the wrong path to the Prisma schem,a

* Added the testcontainers package to the run-engine

* redis-worker package, just a copy of the simple queue for now

* The queue now uses Lua to enqueue dequeue

* The queue now has a catalog and an invisible period after dequeuing

* Added a visibility timeout and acking, with tests

* Added more Redis connection logging, deleted todos

* Visibility timeouts are now defined on the catalog and can be overridden when enqueuing

* Dequeue multiple items at once

* Test for dequeuing multiple items

* Export some types to be used elsewhere

* Partial refactor of the processor

* First stab at a worker with concurrency and NodeWorkers

* Don’t have a default visibility timeout in the queue

* Worker setup and processing items in a simple test

* Process jobs in parallel with retrying

* Get the attempt when dequeuing

* Workers do exponential backoff

* Moved todos

* DLQ functionality

* DLQ tests

* Same cluster for all keys in the same queue

* Added DLQ tests

* Whitespace

* Redis pubsub to redrive from the worker

* Fixed database paths

* Fix for path to zod-worker

* Fixes for typecheck errors, mostly with TS versions and module resolution

* Redlock required a patch

* Moved the new DB migrations to the new database package folder

* Remove the run-engine package

* Remove the RunEngine prisma schema changes

* Delete triggerTaskV2

* Remove zodworker test script (no tests)

* Update test-containers readme

* Generate the client first

* Use a specific version of the prisma package

* Generate the prisma client before running the unit tests
2024-10-08 17:41:22 +01:00

198 lines
4.5 KiB
TypeScript

import {
Prisma,
PrismaClient,
PrismaClientOrTransaction,
PrismaReplicaClient,
PrismaTransactionClient,
PrismaTransactionOptions,
} from "@trigger.dev/database";
import invariant from "tiny-invariant";
import { z } from "zod";
import { env } from "./env.server";
import { logger } from "./services/logger.server";
import { isValidDatabaseUrl } from "./utils/db";
import { singleton } from "./utils/singleton";
import { $transaction as transac } from "@trigger.dev/database";
export type {
PrismaTransactionClient,
PrismaClientOrTransaction,
PrismaTransactionOptions,
PrismaReplicaClient,
};
export async function $transaction<R>(
prisma: PrismaClientOrTransaction,
fn: (prisma: PrismaTransactionClient) => Promise<R>,
options?: PrismaTransactionOptions
): Promise<R | undefined> {
return transac(
prisma,
fn,
(error) => {
logger.error("prisma.$transaction error", {
code: error.code,
meta: error.meta,
stack: error.stack,
message: error.message,
name: error.name,
});
},
options
);
}
export { Prisma };
export const prisma = singleton("prisma", getClient);
export const $replica: PrismaReplicaClient = singleton(
"replica",
() => getReplicaClient() ?? prisma
);
function getClient() {
const { DATABASE_URL } = process.env;
invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
const databaseUrl = extendQueryParams(DATABASE_URL, {
connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
});
console.log(`🔌 setting up prisma client to ${redactUrlSecrets(databaseUrl)}`);
const client = new PrismaClient({
datasources: {
db: {
url: databaseUrl.href,
},
},
// @ts-expect-error
log: [
{
emit: "stdout",
level: "error",
},
{
emit: "stdout",
level: "info",
},
{
emit: "stdout",
level: "warn",
},
].concat(
process.env.VERBOSE_PRISMA_LOGS === "1"
? [
{ emit: "event", level: "query" },
{ emit: "stdout", level: "query" },
]
: []
),
});
// connect eagerly
client.$connect();
console.log(`🔌 prisma client connected`);
return client;
}
function getReplicaClient() {
if (!env.DATABASE_READ_REPLICA_URL) {
console.log(`🔌 No database replica, using the regular client`);
return;
}
const replicaUrl = extendQueryParams(env.DATABASE_READ_REPLICA_URL, {
connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
});
console.log(`🔌 setting up read replica connection to ${redactUrlSecrets(replicaUrl)}`);
const replicaClient = new PrismaClient({
datasources: {
db: {
url: replicaUrl.href,
},
},
// @ts-expect-error
log: [
{
emit: "stdout",
level: "error",
},
{
emit: "stdout",
level: "info",
},
{
emit: "stdout",
level: "warn",
},
].concat(
process.env.VERBOSE_PRISMA_LOGS === "1"
? [
{ emit: "event", level: "query" },
{ emit: "stdout", level: "query" },
]
: []
),
});
// connect eagerly
replicaClient.$connect();
console.log(`🔌 read replica connected`);
return replicaClient;
}
function extendQueryParams(hrefOrUrl: string | URL, queryParams: Record<string, string>) {
const url = new URL(hrefOrUrl);
const query = url.searchParams;
for (const [key, val] of Object.entries(queryParams)) {
query.set(key, val);
}
url.search = query.toString();
return url;
}
function redactUrlSecrets(hrefOrUrl: string | URL) {
const url = new URL(hrefOrUrl);
url.password = "";
return url.href;
}
export type { PrismaClient } from "@trigger.dev/database";
export const PrismaErrorSchema = z.object({
code: z.string(),
});
function getDatabaseSchema() {
if (!isValidDatabaseUrl(env.DATABASE_URL)) {
throw new Error("Invalid Database URL");
}
const databaseUrl = new URL(env.DATABASE_URL);
const schemaFromSearchParam = databaseUrl.searchParams.get("schema");
if (!schemaFromSearchParam) {
console.debug("❗ database schema unspecified, will default to `public` schema");
return "public";
}
return schemaFromSearchParam;
}
export const DATABASE_SCHEMA = singleton("DATABASE_SCHEMA", getDatabaseSchema);
export const sqlDatabaseSchema = Prisma.sql([`${DATABASE_SCHEMA}`]);