From 5bef810e75a6418920906cb72221d85c55196c1d Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 13 May 2024 11:46:52 -0400 Subject: [PATCH] WIP --- pnpm-lock.yaml | 88 ++- {experiments => v3-next}/context.ts | 0 v3-next/package.json | 12 + v3-next/v3.ts | 667 ++++++++++++++++++ .../with-generic-parser.ts | 0 .../with-separate-zod-task.ts | 0 6 files changed, 756 insertions(+), 11 deletions(-) rename {experiments => v3-next}/context.ts (100%) create mode 100644 v3-next/package.json create mode 100644 v3-next/v3.ts rename {experiments => v3-next}/with-generic-parser.ts (100%) rename {experiments => v3-next}/with-separate-zod-task.ts (100%) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e9777ab6..1c27a2286 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,17 +4,6 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -patchedDependencies: - '@changesets/assemble-release-plan@5.2.4': - hash: 3wuhjtl4hjck4itk3w32z4cd5u - path: patches/@changesets__assemble-release-plan@5.2.4.patch - engine.io-parser@5.2.2: - hash: e6nctogrhpxoivwiwy37ersfu4 - path: patches/engine.io-parser@5.2.2.patch - tsup@8.0.1: - hash: a5ztaafw5l4qfghy2hjjuynb34 - path: patches/tsup@8.0.1.patch - importers: .: @@ -3185,6 +3174,24 @@ importers: specifier: ^3.18.0 version: 3.18.0 + v3-next: + dependencies: + '@t3-oss/env-nextjs': + specifier: ^0.10.1 + version: 0.10.1(zod@3.22.3) + '@trpc/server': + specifier: ^10.45.2 + version: 10.45.2 + '@typeschema/valibot': + specifier: ^0.13.4 + version: 0.13.4(valibot@0.30.0) + valibot: + specifier: ^0.30.0 + version: 0.30.0 + zod: + specifier: 3.22.3 + version: 3.22.3 + packages: /@aashutoshrathi/word-wrap@1.2.6: @@ -14690,6 +14697,31 @@ packages: defer-to-connect: 2.0.1 dev: false + /@t3-oss/env-core@0.10.1(zod@3.22.3): + resolution: {integrity: sha512-GcKZiCfWks5CTxhezn9k5zWX3sMDIYf6Kaxy2Gx9YEQftFcz8hDRN56hcbylyAO3t4jQnQ5ifLawINsNgCDpOg==} + peerDependencies: + typescript: '>=5.0.0' + zod: ^3.0.0 + peerDependenciesMeta: + typescript: + optional: true + dependencies: + zod: 3.22.3 + dev: false + + /@t3-oss/env-nextjs@0.10.1(zod@3.22.3): + resolution: {integrity: sha512-iy2qqJLnFh1RjEWno2ZeyTu0ufomkXruUsOZludzDIroUabVvHsrSjtkHqwHp1/pgPUzN3yBRHMILW162X7x2Q==} + peerDependencies: + typescript: '>=5.0.0' + zod: ^3.0.0 + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@t3-oss/env-core': 0.10.1(zod@3.22.3) + zod: 3.22.3 + dev: false + /@tabler/icons-react@2.40.0(react@18.2.0): resolution: {integrity: sha512-C+dDOZowFbwI3LGQP0fdua+hOPkGkW7XeMcRXTSdEKc5fD75W6zRO5nXnWivIMRKsi/Y26EDmnQo15N8JX378w==} peerDependencies: @@ -14898,6 +14930,10 @@ packages: - utf-8-validate dev: false + /@trpc/server@10.45.2: + resolution: {integrity: sha512-wOrSThNNE4HUnuhJG6PfDRp4L2009KDVxsd+2VYH8ro6o/7/jwYZ8Uu5j+VaW+mOmc8EHerHzGcdbGNQSAUPgg==} + dev: false + /@tsconfig/node10@1.0.9: resolution: {integrity: sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==} @@ -15697,6 +15733,32 @@ packages: dev: false optional: true + /@typeschema/core@0.13.2: + resolution: {integrity: sha512-pAt0MK249/9szYaoPuvzhSfOd3smrLhhwCCpUNB4onX32mRx5F3lzDIveIYGQkLYRq58xOX5sjoW+n72f/MLLw==} + peerDependencies: + '@types/json-schema': ^7.0.15 + peerDependenciesMeta: + '@types/json-schema': + optional: true + dev: false + + /@typeschema/valibot@0.13.4(valibot@0.30.0): + resolution: {integrity: sha512-DU095eQ3gy5AS4HGTzWWOmGNMZuAMkV9QQe+TtA6a4rdGzBS7c2KPG4IKcs14yIJ/PsSHHrEdODpRms6qF1vJQ==} + peerDependencies: + '@gcornut/valibot-json-schema': ^0.0.25 + valibot: ^0.29.0 + peerDependenciesMeta: + '@gcornut/valibot-json-schema': + optional: true + valibot: + optional: true + dependencies: + '@typeschema/core': 0.13.2 + valibot: 0.30.0 + transitivePeerDependencies: + - '@types/json-schema' + dev: false + /@typescript-eslint/eslint-plugin@5.59.6(@typescript-eslint/parser@5.59.6)(eslint@8.31.0)(typescript@5.2.2): resolution: {integrity: sha512-sXtOgJNEuRU5RLwPUb1jxtToZbgvq3M6FPpY4QENxoOggK+UpTxUBpj6tD8+Qh2g46Pi9We87E+eHnUw8YcGsw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -34322,6 +34384,10 @@ packages: convert-source-map: 1.9.0 dev: true + /valibot@0.30.0: + resolution: {integrity: sha512-5POBdbSkM+3nvJ6ZlyQHsggisfRtyT4tVTo1EIIShs6qCdXJnyWU5TJ68vr8iTg5zpOLjXLRiBqNx+9zwZz/rA==} + dev: false + /validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} dependencies: diff --git a/experiments/context.ts b/v3-next/context.ts similarity index 100% rename from experiments/context.ts rename to v3-next/context.ts diff --git a/v3-next/package.json b/v3-next/package.json new file mode 100644 index 000000000..172d87cd2 --- /dev/null +++ b/v3-next/package.json @@ -0,0 +1,12 @@ +{ + "name": "@trigger.dev/v3-next", + "description": "Next.js project for Trigger.dev", + "version": "0.1.0", + "dependencies": { + "@t3-oss/env-nextjs": "^0.10.1", + "@trpc/server": "^10.45.2", + "@typeschema/valibot": "^0.13.4", + "valibot": "^0.30.0", + "zod": "3.22.3" + } +} \ No newline at end of file diff --git a/v3-next/v3.ts b/v3-next/v3.ts new file mode 100644 index 000000000..1b236915d --- /dev/null +++ b/v3-next/v3.ts @@ -0,0 +1,667 @@ +import { z } from "zod"; +import * as v from "valibot"; +import { wrap } from "@typeschema/valibot"; + +// https://github.com/t3-oss/t3-env +import { createEnv } from "@t3-oss/env-nextjs"; + +export const env = createEnv({ + /* + * Serverside Environment variables, not available on the client. + * Will throw if you access these variables on the client. + */ + server: { + DATABASE_URL: z.string().url(), + OPEN_AI_API_KEY: z.string().min(1), + }, + /* + * Environment variables available on the client (and server). + * + * 💡 You'll get type errors if these are not prefixed with NEXT_PUBLIC_. + */ + client: { + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: z.string().min(1), + }, + /* + * Due to how Next.js bundles environment variables on Edge and Client, + * we need to manually destructure them to make sure all are included in bundle. + * + * 💡 You'll get type errors if not all variables from `server` & `client` are included here. + */ + runtimeEnv: { + DATABASE_URL: process.env.DATABASE_URL, + OPEN_AI_API_KEY: process.env.OPEN_AI_API_KEY, + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, + }, +}); + +// borrowed from https://github.com/trpc/trpc/blob/main/packages/server/src/core/parser.ts +export type ParserZodEsque = { + _input: TInput; + _output: TParsedInput; +}; + +export type ParserMyZodEsque = { + parse: (input: any) => TInput; +}; + +export type ParserSuperstructEsque = { + create: (input: unknown) => TInput; +}; + +export type ParserCustomValidatorEsque = (input: unknown) => Promise | TInput; + +export type ParserYupEsque = { + validateSync: (input: unknown) => TInput; +}; + +export type ParserScaleEsque = { + assert(value: unknown): asserts value is TInput; +}; + +export type ParserWithoutInput = + | ParserCustomValidatorEsque + | ParserMyZodEsque + | ParserScaleEsque + | ParserSuperstructEsque + | ParserYupEsque; + +export type ParserWithInputOutput = ParserZodEsque; + +export type Parser = ParserWithInputOutput | ParserWithoutInput; + +export type inferParser = TParser extends ParserWithInputOutput< + infer $TIn, + infer $TOut +> + ? { + in: $TIn; + out: $TOut; + } + : TParser extends ParserWithoutInput + ? { + in: $InOut; + out: $InOut; + } + : never; + +export type Simplify = TType extends any[] | Date ? TType : { [K in keyof TType]: TType[K] }; + +export type TriggerResult = { + id: string; +}; + +export type TaskRunResult = + | { + ok: true; + id: string; + output: TOutput; + } + | { + ok: false; + id: string; + error: unknown; + }; + +export type RunMetadata = { + run: string; +}; + +export type inferContext = + TContextBuilder extends ContextBuilder + ? TContext extends UnsetMarker + ? unknown + : TContextOverrides extends UnsetMarker + ? Simplify + : Simplify> + : never; + +export type RunFnParams = { + /** Metadata about the task, run, attempt, queue, environment, organization, project and batch. */ + meta: RunMetadata; + + /** Context added by task middleware */ + ctx: inferContext; + + payload: TPayload; +}; + +/** + * See https://github.com/microsoft/TypeScript/issues/41966#issuecomment-758187996 + * Fixes issues with iterating over keys of objects with index signatures. + * Without this, iterations over keys of objects with index signatures will lose + * type information about the keys and only the index signature will remain. + * @internal + */ +export type WithoutIndexSignature = { + [K in keyof TObj as string extends K ? never : number extends K ? never : K]: TObj[K]; +}; + +/** + * @internal + * Overwrite properties in `TType` with properties in `TWith` + * Only overwrites properties when the type to be overwritten + * is an object. Otherwise it will just use the type from `TWith`. + */ +export type Overwrite = TWith extends any + ? TType extends object + ? { + [K in + | keyof WithoutIndexSignature + | keyof WithoutIndexSignature]: K extends keyof TWith // Exclude index signature from keys + ? TWith[K] + : K extends keyof TType + ? TType[K] + : never; + } & (string extends keyof TWith // Handle cases with an index signature + ? { [key: string]: TWith[string] } + : number extends keyof TWith + ? { [key: number]: TWith[number] } + : // eslint-disable-next-line @typescript-eslint/ban-types + {}) + : TWith + : never; + +/** @internal */ +export const contextMiddlewareMarker = "contextMiddlewareMarker" as "contextMiddlewareMarker" & { + __brand: "contextMiddlewareMarker"; +}; +type ContextMiddlewareMarker = typeof contextMiddlewareMarker; + +interface ContextMiddlewareResultBase { + /** + * All middlewares should pass through their `next()`'s output. + * Requiring this marker makes sure that can't be forgotten at compile-time. + */ + readonly marker: ContextMiddlewareMarker; +} + +interface ContextMiddlewareOKResult<_TContextOverride> extends ContextMiddlewareResultBase { + ok: true; + data: unknown; +} + +interface ContextMiddlewareErrorResult<_TContextOverride> extends ContextMiddlewareResultBase { + ok: false; + error: Error; // should be our error +} + +/** + * @internal + */ +export type ContextMiddlewareResult<_TContextOverride> = + | ContextMiddlewareErrorResult<_TContextOverride> + | ContextMiddlewareOKResult<_TContextOverride>; + +export type ContextMiddlewareFunction< + TContext, + TContextOverridesIn, + $ContextOverridesOut, + TPayloadOut, +> = { + (opts: { + payload: TPayloadOut; + ctx: Simplify>; + meta: RunMetadata; + next: { + (): Promise>; + <$ContextOverride>(ctx: $ContextOverride): Promise>; + }; + }): Promise>; +}; + +export const unsetMarker = Symbol("unsetMarker"); +export type UnsetMarker = typeof unsetMarker; + +export interface ContextBuilder { + use<$ContextOverridesOut>( + fn: ContextMiddlewareFunction + ): ContextBuilder, TPayloadOut>; +} + +export type AnyContextBuilder = ContextBuilder; + +export function createContext(options?: { + ctx?: TContext; + payload?: TParser; +}): ContextBuilder> { + const builder: AnyContextBuilder = { + use(middlewareFn) { + return {} as AnyContextBuilder; + }, + }; + + return builder; +} + +export type TaskOptions< + TOutput, + TContext extends AnyContextBuilder, + TIdentifier extends string, + TParser extends Parser | undefined = undefined, +> = { + /** An id for your task. This must be unique inside your project and not change between versions. */ + id: TIdentifier; + + schema?: TParser; + + context?: TContext; + + /** This gets called when a task is triggered. It's where you put the code you want to execute. + * + * @param payload - The payload that is passed to your task when it's triggered. This must be JSON serializable. + * @param params - Metadata about the run. + */ + run: (params: Simplify, TContext>>) => Promise; +}; + +export interface Task< + TOutput, + TIdentifier extends string, + TParser extends Parser | undefined = undefined, +> { + /** + * The id of the task. + */ + id: TIdentifier; + /** + * Trigger a task with the given payload, and continue without waiting for the result. If you want to wait for the result, use `triggerAndWait`. Returns the id of the triggered task run. + * @param payload + * @param options + * @returns TriggerResult + * - `id` - The id of the triggered task run. + */ + trigger: ( + payload: Simplify>, + options?: TriggerTaskOptions + ) => Promise; + + /** + * Trigger a task with the given payload, and wait for the result. Returns the result of the task run + * @param payload + * @param options - Options for the task run + * @returns TaskRunResult + * @example + * ``` + * const result = await task.triggerAndWait({ foo: "bar" }); + * + * if (result.ok) { + * console.log(result.output); + * } else { + * console.error(result.error); + * } + * ``` + */ + triggerAndWait: ( + payload: Simplify>, + options?: TriggerTaskOptions + ) => Promise>; +} + +export type AnyTask = Task; + +type inferParserIn = TParser extends Parser + ? inferParser["in"] + : TDefault; +type inferParserOut = TParser extends Parser + ? inferParser["out"] + : TDefault; + +export type TaskPayloadIn = TTask extends Task + ? inferParserIn + : never; + +export type TaskPayloadOut = TTask extends Task + ? inferParserOut + : never; + +export type TaskOutput = TTask extends Task + ? TOutput + : never; + +export type TaskIdentifier = TTask extends Task + ? TIdentifier + : never; + +export type TaskTypes = TTask extends Task< + infer TOutput, + infer TIdentifier, + infer TParser +> + ? { + id: TIdentifier; + payloadIn: inferParserIn; + payloadOut: inferParserOut; + output: TOutput; + } + : never; + +export type TriggerTaskOptions = { + idempotencyKey?: string; + maxAttempts?: number; + startAt?: Date; + startAfter?: number; + concurrencyKey?: string; +}; + +export type Prettify = { + [K in keyof T]: T[K]; +} & {}; + +export function task< + TOutput, + TContext extends AnyContextBuilder, + TIdentifier extends string, + TParser extends Parser | undefined = undefined, +>( + options: TaskOptions +): Task { + return createTask(options); +} + +export function createTask< + TOutput, + TContext extends AnyContextBuilder, + TIndentifier extends string, + TParser extends Parser | undefined = undefined, +>( + params: TaskOptions +): Task { + const task: Task = { + id: params.id, + trigger: async (payload, options) => { + return { + id: "run_1234", + }; + }, + triggerAndWait: async (payload, options) => { + const output = await params.run({ + meta: { run: "run_1234" }, + payload: payload as unknown as inferParserOut, // Actually do the parsing + ctx: {} as inferContext, + }); + + return { + ok: true, + id: "run_1234", + output, + }; + }, + }; + + return task; +} + +export interface TaskLibraryRecord { + [key: string]: AnyTask | TaskLibraryRecord; +} + +export interface TaskLibrary { + _def: { record: TRecord }; +} + +export type AnyTaskLibrary = TaskLibrary; + +export type CreateTaskLibraryOptions = { + [key: string]: AnyTask | AnyTaskLibrary | CreateTaskLibraryOptions; +}; + +export type DecorateCreateTaskLibraryOptions = + { + [K in keyof TTaskLibraryOptions]: TTaskLibraryOptions[K] extends infer $Value + ? $Value extends AnyTask + ? $Value + : $Value extends TaskLibrary + ? TRecord + : $Value extends CreateTaskLibraryOptions + ? DecorateCreateTaskLibraryOptions<$Value> + : never + : never; + }; + +function taskLibrary( + input: TInput +): TaskLibrary>; +function taskLibrary(input: TInput): TaskLibrary; +function taskLibrary(input: TaskLibraryRecord | CreateTaskLibraryOptions) { + // TODO: reserved words + + return { + _def: { + record: input, + }, + }; +} + +// ======== client side +type DecorateTask = { + trigger: (id: TaskIdentifier, payload: TaskPayloadIn) => Promise<{ id: string }>; +}; + +type DecoratedTaskLibraryRecord< + TTaskLibrary extends AnyTaskLibrary, + TRecord extends TaskLibraryRecord, +> = { + [TKey in keyof TRecord]: TRecord[TKey] extends infer $Value + ? $Value extends TaskLibraryRecord + ? DecoratedTaskLibraryRecord + : $Value extends AnyTask + ? DecorateTask<$Value> + : never + : never; +}; + +export type inferTaskLibraryClient = + DecoratedTaskLibraryRecord; + +export type CreateTriggerClient = { + lib: inferTaskLibraryClient; + runs: { + retrieve: (id: string) => Promise<{ status: boolean }>; + }; +}; + +export type CreateTriggerClientOptions = { + secretKey?: string; +}; + +export function createTriggerClient( + options?: CreateTriggerClientOptions +): CreateTriggerClient { + return {} as CreateTriggerClient; +} + +// trigger/my-tasks.ts +// TODO: Support https://trpc.io/docs/server/middlewares#experimental-standalone-middlewares +const contextBuilder = createContext({ ctx: { foo: "bar" } }); +const context = contextBuilder + .use((opts) => { + return opts.next({ + baz: "whatever", + }); + }) + .use((opts) => { + return opts.next({ + db: { + find: async (id: string) => { + return "hello"; + }, + }, + }); + }); + +const contextBuilder2 = createContext(); + +const contextBuilder3 = createContext({ ctx: { bar: "baz" } }); + +const contextBuilder4 = createContext().use((opts) => { + return opts.next({ + hello: "world", + }); +}); + +const taskOne = task({ + id: "task-1", + context, + run: async () => { + const handle = await taskTwo.trigger({ url: "https://trigger.dev" }); + const result = await taskTwo.triggerAndWait({ url: "https://trigger.dev" }); + + return "foo-bar"; + }, +}); + +const taskTwo = task({ + id: "task-2", + async run(params) { + return { + hello: "world", + // @ts-expect-error + payload: params.payload.other, + }; + }, +}); + +const userTaskOne = task({ + id: "user/task-1", + context: contextBuilder4, + run: async (params) => { + return "foo-bar"; + }, +}); + +const userTaskTwo = task({ + id: "user/task-2", + context: contextBuilder3, + run: async (params) => { + return "foo-bar"; + }, +}); + +const zodTaskOne = task({ + id: "zod/task-1", + context: contextBuilder, + schema: z.object({ foo: z.string() }), + run: async (params) => {}, +}); + +const zodTaskTwo = task({ + id: "zod/task-2", + schema: z.object({ foo: z.string(), isAdmin: z.boolean().default(false) }), + context: contextBuilder2, + run: async (params) => { + console.log(params.payload.foo, params.meta.run); + }, +}); + +const contextWithPayloadSchema = createContext({ payload: z.object({ foo: z.string() }) }).use( + (opts) => { + return opts.next({ + bar: "baz", + }); + } +); + +const zodTaskThreeWithContext = task({ + id: "zod/task-2", + schema: z.object({ foo: z.string(), isAdmin: z.boolean().default(false) }), + context: contextWithPayloadSchema, + run: async (params) => { + console.log(params.payload.foo, params.meta.run); + }, +}); + +const zodTaskThreeWithoutMatchingPayload = task({ + id: "zod/task-2", + schema: z.object({ bar: z.string() }), + context: contextWithPayloadSchema, + run: async (params) => { + console.log(params.ctx.bar); + console.log(params.payload.bar, params.meta.run); + }, +}); + +const valibotTaskOne = task({ + id: "valibot/task-1", + schema: wrap( + v.object({ + foo: v.string(), + }) + ), + run: async (params) => { + await zodTaskOne.trigger({ foo: "bar" }); + await zodTaskTwo.trigger({ foo: "bar" }); + + await valibotTaskTwo.trigger({ foo: "bar" }); + }, +}); + +const valibotTaskTwo = task({ + id: "valibot/task-2", + schema: wrap( + v.object({ + foo: v.string(), + isAdmin: v.optional(v.boolean(), true), + }) + ), + run: async (params) => { + await valibotTaskOne.trigger({ foo: "bar" }); + }, +}); + +// in trigger/lib.ts +const myTaskLibrary = taskLibrary({ + myTasks: { taskOne, taskTwo }, +}); + +const userTaskLibrary = taskLibrary({ + userTaskOne, + userTaskTwo, +}); + +const zodTaskLibrary = taskLibrary({ + zodTaskOne, + zodTaskTwo, +}); + +const valibotTaskLibrary = taskLibrary({ + valibotTaskOne, + valibotTaskTwo, +}); + +export const library = taskLibrary({ + foo: myTaskLibrary, + bar: userTaskLibrary, + zod: zodTaskLibrary, + valibot: valibotTaskLibrary, +}); + +// Export the library type +export type Library = typeof library; + +// Now on the client +const client = createTriggerClient({ + secretKey: "tr_dev_1234", +}); + +client.runs.retrieve("run_12343"); // Call regular API client calls + +// Tasks are now available under lib +client.lib.foo.myTasks.taskOne.trigger("task-1", { hello: "world" }); +client.lib.bar.userTaskOne.trigger("user/task-1", { userId: "user_123" }); +client.lib.bar.userTaskTwo.trigger("user/task-2", { + userId: "user_123", + isAdmin: true, +}); +client.lib.bar.userTaskTwo.trigger("user/task-2", { + userId: "user_123", + isAdmin: false, +}); +client.lib.zod.zodTaskOne.trigger("zod/task-1", { foo: "bar" }); +client.lib.zod.zodTaskTwo.trigger("zod/task-2", { foo: "bar" }); +client.lib.zod.zodTaskTwo.trigger("zod/task-2", { foo: "bar", isAdmin: false }); +client.lib.valibot.valibotTaskTwo.trigger("valibot/task-2", { foo: "bar" }); +client.lib.valibot.valibotTaskTwo.trigger("valibot/task-2", { + foo: "bar", + isAdmin: true, +}); diff --git a/experiments/with-generic-parser.ts b/v3-next/with-generic-parser.ts similarity index 100% rename from experiments/with-generic-parser.ts rename to v3-next/with-generic-parser.ts diff --git a/experiments/with-separate-zod-task.ts b/v3-next/with-separate-zod-task.ts similarity index 100% rename from experiments/with-separate-zod-task.ts rename to v3-next/with-separate-zod-task.ts