From ba71f959e2953e82d4aaf8dfe0b37c69faa2f722 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 31 May 2024 11:02:28 +0100 Subject: [PATCH] v3: Runs List Management SDK and SDK improvements (#1133) * Improved the existing runs API * WIP next runs API * Improve the returned ApiPromise to add ability to return response * More WIP * WI{ * Added offset/limit pagination stuff like the cursor one, and converted all API methods to use ApiPromise * More run API stuff - Adding schedule output from the retrieveRun endpoint - Ability to filter by schedule and isTest * Remove env from retrieve run in openAPI * prefer duplication over merge * WIP docs * Use spread to DRY up some run API schemas * Finish the overview docs * Adding changeset * Fixed typecheck errors * Typo fix * Re-export zodfetch from core so the v3 CLI can use it * Fixed type errors --------- Co-authored-by: Matt Aitken --- .changeset/stupid-adults-sniff.md | 6 + apps/webapp/app/env.server.ts | 1 + .../app/models/runtimeEnvironment.server.ts | 2 + .../v3/ApiRetrieveRunPresenter.server.ts | 151 +++- .../v3/ApiRunListPresenter.server.ts | 274 +++++++ .../presenters/v3/RunListPresenter.server.ts | 21 +- .../v3/ViewSchedulePresenter.server.ts | 6 +- .../app/presenters/v3/basePresenter.server.ts | 22 + .../route.tsx | 9 +- .../api.v1.projects.$projectRef.runs.ts | 62 ++ apps/webapp/app/routes/api.v1.runs.ts | 52 ++ .../routes/api.v2.runs.$runParam.cancel.ts | 2 +- apps/webapp/app/v3/marqs/index.server.ts | 6 + docs/mint.json | 42 +- docs/v3-openapi.yaml | 609 ++++++++++++++-- docs/v3/management-cancel-run.mdx | 4 - docs/v3/management-get-run.mdx | 6 - docs/v3/management-get-runs.mdx | 6 - docs/v3/management-replay-run.mdx | 4 - docs/v3/management-retrieve-run.mdx | 4 - docs/v3/management-start-run.mdx | 6 - .../envvars/create.mdx} | 0 .../envvars/delete.mdx} | 0 .../envvars/import.mdx} | 0 .../envvars/list.mdx} | 0 .../envvars/retrieve.mdx} | 0 .../envvars/update.mdx} | 0 docs/v3/management/overview.mdx | 220 ++++++ docs/v3/management/projects/runs.mdx | 4 + docs/v3/management/runs/cancel.mdx | 4 + docs/v3/management/runs/list.mdx | 4 + docs/v3/management/runs/replay.mdx | 4 + docs/v3/management/runs/retrieve.mdx | 4 + .../schedules/activate.mdx} | 0 .../schedules/create.mdx} | 0 .../schedules/deactivate.mdx} | 0 .../schedules/delete.mdx} | 0 .../schedules/list.mdx} | 0 .../schedules/retrieve.mdx} | 0 .../schedules/update.mdx} | 0 packages/cli-v3/src/apiClient.ts | 5 +- packages/core/src/v3/apiClient/core.ts | 687 ++++++++++++++++++ .../v3/{apiErrors.ts => apiClient/errors.ts} | 26 +- packages/core/src/v3/apiClient/index.ts | 140 +++- packages/core/src/v3/apiClient/pagination.ts | 163 +++++ packages/core/src/v3/apiClient/types.ts | 20 +- packages/core/src/v3/errors.ts | 10 +- packages/core/src/v3/index.ts | 5 +- packages/core/src/v3/schemas/api.ts | 92 ++- packages/core/src/v3/schemas/schemas.ts | 11 +- packages/core/src/v3/zodfetch.ts | 440 +---------- packages/trigger-sdk/src/v3/envvars.ts | 146 ++-- packages/trigger-sdk/src/v3/index.ts | 2 +- packages/trigger-sdk/src/v3/runs.ts | 43 +- .../trigger-sdk/src/v3/schedules/index.ts | 21 +- references/v3-catalog/src/management.ts | 252 +++++-- 56 files changed, 2797 insertions(+), 801 deletions(-) create mode 100644 .changeset/stupid-adults-sniff.md create mode 100644 apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts create mode 100644 apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts create mode 100644 apps/webapp/app/routes/api.v1.runs.ts delete mode 100644 docs/v3/management-cancel-run.mdx delete mode 100644 docs/v3/management-get-run.mdx delete mode 100644 docs/v3/management-get-runs.mdx delete mode 100644 docs/v3/management-replay-run.mdx delete mode 100644 docs/v3/management-retrieve-run.mdx delete mode 100644 docs/v3/management-start-run.mdx rename docs/v3/{management-envvars-create.mdx => management/envvars/create.mdx} (100%) rename docs/v3/{management-envvars-delete.mdx => management/envvars/delete.mdx} (100%) rename docs/v3/{management-envvars-import.mdx => management/envvars/import.mdx} (100%) rename docs/v3/{management-envvars-list.mdx => management/envvars/list.mdx} (100%) rename docs/v3/{management-envvars-retrieve.mdx => management/envvars/retrieve.mdx} (100%) rename docs/v3/{management-envvars-update.mdx => management/envvars/update.mdx} (100%) create mode 100644 docs/v3/management/overview.mdx create mode 100644 docs/v3/management/projects/runs.mdx create mode 100644 docs/v3/management/runs/cancel.mdx create mode 100644 docs/v3/management/runs/list.mdx create mode 100644 docs/v3/management/runs/replay.mdx create mode 100644 docs/v3/management/runs/retrieve.mdx rename docs/v3/{management-activate-schedule.mdx => management/schedules/activate.mdx} (100%) rename docs/v3/{management-create-schedule.mdx => management/schedules/create.mdx} (100%) rename docs/v3/{management-deactivate-schedule.mdx => management/schedules/deactivate.mdx} (100%) rename docs/v3/{management-delete-schedule.mdx => management/schedules/delete.mdx} (100%) rename docs/v3/{management-list-schedules.mdx => management/schedules/list.mdx} (100%) rename docs/v3/{management-retrieve-schedule.mdx => management/schedules/retrieve.mdx} (100%) rename docs/v3/{management-update-schedule.mdx => management/schedules/update.mdx} (100%) create mode 100644 packages/core/src/v3/apiClient/core.ts rename packages/core/src/v3/{apiErrors.ts => apiClient/errors.ts} (81%) create mode 100644 packages/core/src/v3/apiClient/pagination.ts diff --git a/.changeset/stupid-adults-sniff.md b/.changeset/stupid-adults-sniff.md new file mode 100644 index 000000000..02fefb363 --- /dev/null +++ b/.changeset/stupid-adults-sniff.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/sdk": patch +"@trigger.dev/core": patch +--- + +Management SDK overhaul and adding the runs.list API diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index bf08d82b0..0db31c87a 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -170,6 +170,7 @@ const EnvironmentSchema = z.object({ MAX_SEQUENTIAL_INDEX_FAILURE_COUNT: z.coerce.number().default(96), LOOPS_API_KEY: z.string().optional(), + MARQS_DISABLE_REBALANCING: z.coerce.boolean().default(false), }); export type Environment = z.infer; diff --git a/apps/webapp/app/models/runtimeEnvironment.server.ts b/apps/webapp/app/models/runtimeEnvironment.server.ts index 0cccc1e4f..7cfe6da5c 100644 --- a/apps/webapp/app/models/runtimeEnvironment.server.ts +++ b/apps/webapp/app/models/runtimeEnvironment.server.ts @@ -124,6 +124,7 @@ type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ select: { id: true; type: true; + slug: true; orgMember: { select: { user: { @@ -155,6 +156,7 @@ export function displayableEnvironment( return { id: environment.id, type: environment.type, + slug: environment.slug, userName, }; } diff --git a/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts index ad9f44768..afd7028e8 100644 --- a/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts @@ -1,5 +1,15 @@ -import { AttemptStatus, RetrieveRunResponse, RunStatus, logger } from "@trigger.dev/core/v3"; -import { TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database"; +import { + AttemptStatus, + RetrieveRunResponse, + RunStatus, + SerializedError, + TaskRunError, + conditionallyImportPacket, + createJsonErrorObject, + logger, + parsePacket, +} from "@trigger.dev/core/v3"; +import { Prisma, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database"; import assertNever from "assert-never"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { BasePresenter } from "./basePresenter.server"; @@ -23,6 +33,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter { }, }, lockedToVersion: true, + schedule: true, }, }); @@ -32,14 +43,68 @@ export class ApiRetrieveRunPresenter extends BasePresenter { return undefined; } + let $payload: any; + let $output: any; + + if (showSecretDetails) { + const payloadPacket = await conditionallyImportPacket({ + data: taskRun.payload, + dataType: taskRun.payloadType, + }); + + $payload = + payloadPacket.dataType === "application/json" + ? await parsePacket(payloadPacket) + : payloadPacket.data; + + if (taskRun.status === "COMPLETED_SUCCESSFULLY") { + const completedAttempt = taskRun.attempts.find( + (a) => a.status === "COMPLETED" && typeof a.output !== null + ); + + if (completedAttempt && completedAttempt.output) { + const outputPacket = await conditionallyImportPacket({ + data: completedAttempt.output, + dataType: completedAttempt.outputType, + }); + + $output = await parsePacket(outputPacket); + } + } + } + + const apiStatus = ApiRetrieveRunPresenter.apiStatusFromRunStatus(taskRun.status); + return { id: taskRun.friendlyId, - status: ApiRetrieveRunPresenter.apiStatusFromRunStatus(taskRun.status), + status: apiStatus, taskIdentifier: taskRun.taskIdentifier, idempotencyKey: taskRun.idempotencyKey ?? undefined, version: taskRun.lockedToVersion ? taskRun.lockedToVersion.version : undefined, createdAt: taskRun.createdAt ?? undefined, updatedAt: taskRun.updatedAt ?? undefined, + startedAt: taskRun.lockedAt ?? undefined, + finishedAt: ApiRetrieveRunPresenter.isStatusFinished(apiStatus) + ? taskRun.updatedAt + : undefined, + payload: $payload, + output: $output, + isTest: taskRun.isTest, + schedule: taskRun.schedule + ? { + id: taskRun.schedule.friendlyId, + externalId: taskRun.schedule.externalId ?? undefined, + deduplicationKey: taskRun.schedule.userProvidedDeduplicationKey + ? taskRun.schedule.deduplicationKey + : undefined, + generator: { + type: "CRON", + expression: taskRun.schedule.generatorExpression, + description: taskRun.schedule.generatorDescription, + }, + } + : undefined, + ...ApiRetrieveRunPresenter.apiBooleanHelpersFromRunStatus(apiStatus), attempts: !showSecretDetails ? [] : taskRun.attempts.map((a) => ({ @@ -49,34 +114,68 @@ export class ApiRetrieveRunPresenter extends BasePresenter { updatedAt: a.updatedAt ?? undefined, startedAt: a.startedAt ?? undefined, completedAt: a.completedAt ?? undefined, + error: ApiRetrieveRunPresenter.apiErrorFromError(a.error), })), }; }); } + static apiErrorFromError(error: Prisma.JsonValue): SerializedError | undefined { + if (!error) { + return; + } + + const errorData = TaskRunError.safeParse(error); + + if (errorData.success) { + return createJsonErrorObject(errorData.data); + } + } + + static isStatusFinished(status: RunStatus) { + return ( + status === "COMPLETED" || + status === "FAILED" || + status === "CANCELED" || + status === "INTERRUPTED" || + status === "CRASHED" || + status === "SYSTEM_FAILURE" + ); + } + static apiStatusFromRunStatus(status: TaskRunStatus): RunStatus { switch (status) { - case "WAITING_FOR_DEPLOY": - case "PENDING": { - return "PENDING"; + case "WAITING_FOR_DEPLOY": { + return "WAITING_FOR_DEPLOY"; + } + case "PENDING": { + return "QUEUED"; + } + case "PAUSED": + case "WAITING_TO_RESUME": { + return "FROZEN"; + } + case "RETRYING_AFTER_FAILURE": { + return "REATTEMPTING"; } - case "RETRYING_AFTER_FAILURE": case "EXECUTING": { return "EXECUTING"; } - case "WAITING_TO_RESUME": - case "PAUSED": { - return "PAUSED"; - } case "CANCELED": { return "CANCELED"; } case "COMPLETED_SUCCESSFULLY": { return "COMPLETED"; } - case "SYSTEM_FAILURE": - case "INTERRUPTED": - case "CRASHED": + case "SYSTEM_FAILURE": { + return "SYSTEM_FAILURE"; + } + case "INTERRUPTED": { + return "INTERRUPTED"; + } + case "CRASHED": { + return "CRASHED"; + } case "COMPLETED_WITH_ERRORS": { return "FAILED"; } @@ -86,6 +185,30 @@ export class ApiRetrieveRunPresenter extends BasePresenter { } } + static apiBooleanHelpersFromRunStatus(status: RunStatus) { + const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY"; + const isExecuting = status === "EXECUTING" || status === "REATTEMPTING" || status === "FROZEN"; + const isCompleted = + status === "COMPLETED" || + status === "CANCELED" || + status === "FAILED" || + status === "CRASHED" || + status === "INTERRUPTED" || + status === "SYSTEM_FAILURE"; + const isFailed = isCompleted && status !== "COMPLETED"; + const isSuccess = isCompleted && status === "COMPLETED"; + const isCancelled = status === "CANCELED"; + + return { + isQueued, + isExecuting, + isCompleted, + isFailed, + isSuccess, + isCancelled, + }; + } + static apiStatusFromAttemptStatus(status: TaskRunAttemptStatus): AttemptStatus { switch (status) { case "PENDING": { diff --git a/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts new file mode 100644 index 000000000..933f58218 --- /dev/null +++ b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts @@ -0,0 +1,274 @@ +import { ListRunResponse, ListRunResponseItem, RunStatus } from "@trigger.dev/core/v3"; +import { Project, RuntimeEnvironment, TaskRunStatus } from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { z } from "zod"; +import { fromZodError } from "zod-validation-error"; +import { logger } from "~/services/logger.server"; +import { ApiRetrieveRunPresenter } from "./ApiRetrieveRunPresenter.server"; +import { RunListOptions, RunListPresenter } from "./RunListPresenter.server"; +import { BasePresenter } from "./basePresenter.server"; + +const SearchParamsSchema = z.object({ + "page[size]": z.coerce.number().int().positive().min(1).max(100).optional(), + "page[after]": z.string().optional(), + "page[before]": z.string().optional(), + "filter[status]": z + .string() + .optional() + .transform((value, ctx) => { + if (!value) { + return undefined; + } + + const statuses = value.split(","); + const parsedStatuses = statuses.map((status) => RunStatus.safeParse(status)); + + if (parsedStatuses.some((result) => !result.success)) { + const invalidStatuses: string[] = []; + + for (const [index, result] of parsedStatuses.entries()) { + if (!result.success) { + invalidStatuses.push(statuses[index]); + } + } + + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid status values: ${invalidStatuses.join(", ")}`, + }); + + return z.NEVER; + } + + const $statuses = parsedStatuses + .map((result) => (result.success ? result.data : undefined)) + .filter(Boolean); + + return Array.from(new Set($statuses)); + }), + "filter[env]": z + .string() + .optional() + .transform((value) => { + return value ? value.split(",") : undefined; + }), + "filter[taskIdentifier]": z + .string() + .optional() + .transform((value) => { + return value ? value.split(",") : undefined; + }), + "filter[version]": z + .string() + .optional() + .transform((value) => { + return value ? value.split(",") : undefined; + }), + "filter[bulkAction]": z.string().optional(), + "filter[schedule]": z.string().optional(), + "filter[isTest]": z + .string() + .optional() + .transform((value, ctx) => { + if (!value) { + return undefined; + } + + if (value === "true") { + return true; + } + + if (value === "false") { + return false; + } + + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid value for isTest: ${value}`, + }); + + return z.NEVER; + }), + "filter[createdAt][from]": z.coerce.date().optional(), + "filter[createdAt][to]": z.coerce.date().optional(), + "filter[createdAt][period]": z.string().optional(), +}); + +type SearchParamsSchema = z.infer; + +export class ApiRunListPresenter extends BasePresenter { + public async call( + project: Project, + searchParams: URLSearchParams, + environment?: RuntimeEnvironment + ): Promise { + return this.trace("call", async (span) => { + const rawSearchParams = Object.fromEntries(searchParams.entries()); + const $searchParams = SearchParamsSchema.safeParse(rawSearchParams); + + if (!$searchParams.success) { + logger.error("Invalid search params", { + searchParams: rawSearchParams, + errors: $searchParams.error.errors, + }); + + throw fromZodError($searchParams.error); + } + + logger.debug("Valid search params", { searchParams: $searchParams.data }); + + const options: RunListOptions = { + projectId: project.id, + }; + + // pagination + if ($searchParams.data["page[size]"]) { + options.pageSize = $searchParams.data["page[size]"]; + } + + if ($searchParams.data["page[after]"]) { + options.cursor = $searchParams.data["page[after]"]; + options.direction = "forward"; + } + + if ($searchParams.data["page[before]"]) { + options.cursor = $searchParams.data["page[before]"]; + options.direction = "backward"; + } + + // filters + if (environment) { + options.environments = [environment.id]; + } else { + if ($searchParams.data["filter[env]"]) { + const environments = await this._prisma.runtimeEnvironment.findMany({ + where: { + projectId: project.id, + slug: { + in: $searchParams.data["filter[env]"], + }, + }, + }); + + options.environments = environments.map((env) => env.id); + } + } + + if ($searchParams.data["filter[status]"]) { + options.statuses = $searchParams.data["filter[status]"].flatMap((status) => + ApiRunListPresenter.apiStatusToRunStatuses(status) + ); + } + + if ($searchParams.data["filter[taskIdentifier]"]) { + options.tasks = $searchParams.data["filter[taskIdentifier]"]; + } + + if ($searchParams.data["filter[version]"]) { + options.versions = $searchParams.data["filter[version]"]; + } + + if ($searchParams.data["filter[bulkAction]"]) { + options.bulkId = $searchParams.data["filter[bulkAction]"]; + } + + if ($searchParams.data["filter[schedule]"]) { + options.scheduleId = $searchParams.data["filter[schedule]"]; + } + + if ($searchParams.data["filter[createdAt][from]"]) { + options.from = $searchParams.data["filter[createdAt][from]"].getTime(); + } + + if ($searchParams.data["filter[createdAt][to]"]) { + options.to = $searchParams.data["filter[createdAt][to]"].getTime(); + } + + if ($searchParams.data["filter[createdAt][period]"]) { + options.period = $searchParams.data["filter[createdAt][period]"]; + } + + if (typeof $searchParams.data["filter[isTest]"] === "boolean") { + options.isTest = $searchParams.data["filter[isTest]"]; + } + + const presenter = new RunListPresenter(); + + logger.debug("Calling RunListPresenter", { options }); + + const results = await presenter.call(options); + + const data: ListRunResponseItem[] = results.runs.map((run) => { + return { + id: run.friendlyId, + status: ApiRetrieveRunPresenter.apiStatusFromRunStatus(run.status), + taskIdentifier: run.taskIdentifier, + idempotencyKey: run.idempotencyKey, + version: run.version ?? undefined, + createdAt: new Date(run.createdAt), + updatedAt: new Date(run.updatedAt), + startedAt: run.startedAt ? new Date(run.startedAt) : undefined, + finishedAt: run.finishedAt ? new Date(run.finishedAt) : undefined, + isTest: run.isTest, + env: { + id: run.environment.id, + name: run.environment.slug, + user: run.environment.userName, + }, + ...ApiRetrieveRunPresenter.apiBooleanHelpersFromRunStatus( + ApiRetrieveRunPresenter.apiStatusFromRunStatus(run.status) + ), + }; + }); + + return { + data, + pagination: { + next: results.pagination.next, + previous: results.pagination.previous, + }, + }; + }); + } + + static apiStatusToRunStatuses(status: RunStatus): TaskRunStatus[] | TaskRunStatus { + switch (status) { + case "WAITING_FOR_DEPLOY": { + return "WAITING_FOR_DEPLOY"; + } + case "QUEUED": { + return "PENDING"; + } + case "EXECUTING": { + return "EXECUTING"; + } + case "REATTEMPTING": { + return "RETRYING_AFTER_FAILURE"; + } + case "FROZEN": { + return ["PAUSED", "WAITING_TO_RESUME"]; + } + case "CANCELED": { + return "CANCELED"; + } + case "COMPLETED": { + return "COMPLETED_SUCCESSFULLY"; + } + case "SYSTEM_FAILURE": { + return "SYSTEM_FAILURE"; + } + case "INTERRUPTED": { + return "INTERRUPTED"; + } + case "CRASHED": { + return "CRASHED"; + } + case "FAILED": { + return "COMPLETED_WITH_ERRORS"; + } + default: { + assertNever(status); + } + } + } +} diff --git a/apps/webapp/app/presenters/v3/RunListPresenter.server.ts b/apps/webapp/app/presenters/v3/RunListPresenter.server.ts index b07b9b594..ae57dc533 100644 --- a/apps/webapp/app/presenters/v3/RunListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunListPresenter.server.ts @@ -7,9 +7,9 @@ import { displayableEnvironment } from "~/models/runtimeEnvironment.server"; import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server"; import { BasePresenter } from "./basePresenter.server"; -type RunListOptions = { +export type RunListOptions = { userId?: string; - projectSlug: string; + projectId: string; //filters tasks?: string[]; versions?: string[]; @@ -20,6 +20,7 @@ type RunListOptions = { bulkId?: string; from?: number; to?: number; + isTest?: boolean; //pagination direction?: Direction; cursor?: string; @@ -35,7 +36,7 @@ export type RunListAppliedFilters = RunList["filters"]; export class RunListPresenter extends BasePresenter { public async call({ userId, - projectSlug, + projectId, tasks, versions, statuses, @@ -43,6 +44,7 @@ export class RunListPresenter extends BasePresenter { scheduleId, period, bulkId, + isTest, from, to, direction = "forward", @@ -59,7 +61,9 @@ export class RunListPresenter extends BasePresenter { (period !== undefined && period !== "all") || (bulkId !== undefined && bulkId !== "") || from !== undefined || - to !== undefined; + to !== undefined || + (scheduleId !== undefined && scheduleId !== "") || + typeof isTest === "boolean"; // Find the project scoped to the organization const project = await this._replica.project.findFirstOrThrow({ @@ -85,7 +89,7 @@ export class RunListPresenter extends BasePresenter { }, }, where: { - slug: projectSlug, + id: projectId, }, }); @@ -156,6 +160,7 @@ export class RunListPresenter extends BasePresenter { updatedAt: Date; isTest: boolean; spanId: string; + idempotencyKey: string | null; }[] >` SELECT @@ -170,7 +175,8 @@ export class RunListPresenter extends BasePresenter { tr."lockedAt" AS "lockedAt", tr."updatedAt" AS "updatedAt", tr."isTest" AS "isTest", - tr."spanId" AS "spanId" + tr."spanId" AS "spanId", + tr."idempotencyKey" AS "idempotencyKey" FROM ${sqlDatabaseSchema}."TaskRun" tr LEFT JOIN @@ -210,6 +216,7 @@ export class RunListPresenter extends BasePresenter { : Prisma.empty } ${scheduleId ? Prisma.sql`AND tr."scheduleId" = ${scheduleId}` : Prisma.empty} + ${typeof isTest === "boolean" ? Prisma.sql`AND tr."isTest" = ${isTest}` : Prisma.empty} ${ periodMs ? Prisma.sql`AND tr."createdAt" >= NOW() - INTERVAL '1 millisecond' * ${periodMs}` @@ -270,6 +277,7 @@ export class RunListPresenter extends BasePresenter { friendlyId: run.runFriendlyId, number: Number(run.number), createdAt: run.createdAt.toISOString(), + updatedAt: run.updatedAt.toISOString(), startedAt: run.lockedAt ? run.lockedAt.toISOString() : undefined, hasFinished, finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined, @@ -281,6 +289,7 @@ export class RunListPresenter extends BasePresenter { isReplayable: true, isCancellable: CANCELLABLE_STATUSES.includes(run.status), environment: displayableEnvironment(environment, userId), + idempotencyKey: run.idempotencyKey ? run.idempotencyKey : undefined, }; }), pagination: { diff --git a/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts b/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts index c374de5ac..a990f1e9f 100644 --- a/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts @@ -30,7 +30,7 @@ export class ViewSchedulePresenter { taskIdentifier: true, project: { select: { - slug: true, + id: true, }, }, instances: { @@ -39,6 +39,7 @@ export class ViewSchedulePresenter { select: { id: true, type: true, + slug: true, orgMember: { select: { user: { @@ -71,8 +72,9 @@ export class ViewSchedulePresenter { : []; const runPresenter = new RunListPresenter(this.#prismaClient); + const { runs } = await runPresenter.call({ - projectSlug: schedule.project.slug, + projectId: schedule.project.id, scheduleId: schedule.id, pageSize: 5, }); diff --git a/apps/webapp/app/presenters/v3/basePresenter.server.ts b/apps/webapp/app/presenters/v3/basePresenter.server.ts index 39298c9da..562582ebf 100644 --- a/apps/webapp/app/presenters/v3/basePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/basePresenter.server.ts @@ -34,4 +34,26 @@ export abstract class BasePresenter { } ); } + + protected async trace(trace: string, fn: (span: Span) => Promise): Promise { + return tracer.startActiveSpan( + `${this.constructor.name}.${trace}`, + { kind: SpanKind.SERVER }, + async (span) => { + try { + return await fn(span); + } catch (e) { + if (e instanceof Error) { + span.recordException(e); + } else { + span.recordException(new Error(String(e))); + } + + throw e; + } finally { + span.end(); + } + } + ); + } } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx index eadaf17db..5f086d689 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx @@ -39,6 +39,7 @@ import { cn } from "~/utils/cn"; import { ProjectParamSchema, v3ProjectPath, v3RunsPath, v3TestPath } from "~/utils/pathBuilder"; import { ListPagination } from "../../components/ListPagination"; import { BULK_ACTION_RUN_LIMIT } from "~/consts"; +import { findProjectBySlug } from "~/models/project.server"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); @@ -57,10 +58,16 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const { tasks, versions, statuses, environments, period, bulkId, from, to, cursor, direction } = TaskRunListSearchFilters.parse(s); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + + if (!project) { + throw new Error("Project not found"); + } + const presenter = new RunListPresenter(); const list = presenter.call({ userId, - projectSlug: projectParam, + projectId: project.id, tasks, versions, statuses, diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts new file mode 100644 index 000000000..639338c8d --- /dev/null +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts @@ -0,0 +1,62 @@ +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { ValidationError } from "zod-validation-error"; +import { findProjectByRef } from "~/models/project.server"; +import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { apiCors } from "~/utils/apiCors"; + +const ParamsSchema = z.object({ + projectRef: z.string(), +}); + +export async function loader({ request, params }: LoaderFunctionArgs) { + if (request.method.toUpperCase() === "OPTIONS") { + return apiCors(request, json({})); + } + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 })); + } + + const $params = ParamsSchema.safeParse(params); + + if (!$params.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + const project = await findProjectByRef($params.data.projectRef, authenticationResult.userId); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const url = new URL(request.url); + + const presenter = new ApiRunListPresenter(); + + try { + const result = await presenter.call(project, url.searchParams); + + if (!result) { + return apiCors(request, json({ data: [] })); + } + + return apiCors(request, json(result)); + } catch (error) { + if (error instanceof ValidationError) { + return apiCors( + request, + json({ error: "Query Error", details: error.details }, { status: 400 }) + ); + } else { + return apiCors( + request, + json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 }) + ); + } + } +} diff --git a/apps/webapp/app/routes/api.v1.runs.ts b/apps/webapp/app/routes/api.v1.runs.ts new file mode 100644 index 000000000..b439c7755 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.runs.ts @@ -0,0 +1,52 @@ +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { ValidationError } from "zod-validation-error"; +import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; +import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { apiCors } from "~/utils/apiCors"; + +export async function loader({ request, params }: LoaderFunctionArgs) { + if (request.method.toUpperCase() === "OPTIONS") { + return apiCors(request, json({})); + } + + const authenticationResult = await authenticateApiRequest(request, { + allowPublicKey: false, + }); + + if (!authenticationResult) { + return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 })); + } + + const authenticatedEnv = authenticationResult.environment; + + const url = new URL(request.url); + + const presenter = new ApiRunListPresenter(); + + try { + const result = await presenter.call( + authenticatedEnv.project, + url.searchParams, + authenticatedEnv + ); + + if (!result) { + return apiCors(request, json({ data: [] })); + } + + return apiCors(request, json(result)); + } catch (error) { + if (error instanceof ValidationError) { + return apiCors( + request, + json({ error: "Query Error", details: error.details }, { status: 400 }) + ); + } else { + return apiCors( + request, + json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 }) + ); + } + } +} diff --git a/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts b/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts index 8bfe68d94..8862c8875 100644 --- a/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts +++ b/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts @@ -48,5 +48,5 @@ export async function action({ request, params }: ActionFunctionArgs) { return json({ error: "Internal Server Error" }, { status: 500 }); } - return json({ message: "Run cancelled" }, { status: 200 }); + return json({ id: runParam }, { status: 200 }); } diff --git a/apps/webapp/app/v3/marqs/index.server.ts b/apps/webapp/app/v3/marqs/index.server.ts index 0816bab3b..1e369c685 100644 --- a/apps/webapp/app/v3/marqs/index.server.ts +++ b/apps/webapp/app/v3/marqs/index.server.ts @@ -49,6 +49,7 @@ export type MarQSOptions = { keysProducer: MarQSKeyProducer; queuePriorityStrategy: MarQSQueuePriorityStrategy; envQueuePriorityStrategy: MarQSQueuePriorityStrategy; + enableRebalancing?: boolean; }; /** @@ -707,6 +708,10 @@ export class MarQS { } #startRebalanceWorkers() { + if (!this.options.enableRebalancing) { + return; + } + // Start a new worker to rebalance parent queues periodically for (let i = 0; i < this.options.workers; i++) { const worker = new AsyncWorker(this.#rebalanceParentQueues.bind(this), 60_000); @@ -1597,6 +1602,7 @@ function getMarQSClient() { defaultEnvConcurrency: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT, defaultOrgConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT, visibilityTimeoutInMs: 120 * 1000, // 2 minutes, + enableRebalancing: !env.MARQS_DISABLE_REBALANCING, }); } else { console.warn( diff --git a/docs/mint.json b/docs/mint.json index e25294dc8..dce9bae32 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -5,7 +5,7 @@ "versions": ["v3 (Developer Preview)", "v2"], "api": { "playground": { - "mode": "hide" + "mode": "simple" }, "maintainOrder": true }, @@ -165,35 +165,43 @@ "group": "API reference", "version": "v3 (Developer Preview)", "pages": [ + "v3/management/overview", { "group": "Runs API", "pages": [ - "v3/management-retrieve-run", - "v3/management-replay-run", - "v3/management-cancel-run" + "v3/management/runs/list", + "v3/management/runs/retrieve", + "v3/management/runs/replay", + "v3/management/runs/cancel" ] }, { "group": "Schedules API", "pages": [ - "v3/management-list-schedules", - "v3/management-create-schedule", - "v3/management-retrieve-schedule", - "v3/management-update-schedule", - "v3/management-delete-schedule", - "v3/management-deactivate-schedule", - "v3/management-activate-schedule" + "v3/management/schedules/list", + "v3/management/schedules/create", + "v3/management/schedules/retrieve", + "v3/management/schedules/update", + "v3/management/schedules/delete", + "v3/management/schedules/deactivate", + "v3/management/schedules/activate" ] }, { "group": "Env Vars API", "pages": [ - "v3/management-envvars-list", - "v3/management-envvars-import", - "v3/management-envvars-create", - "v3/management-envvars-retrieve", - "v3/management-envvars-update", - "v3/management-envvars-delete" + "v3/management/envvars/list", + "v3/management/envvars/import", + "v3/management/envvars/create", + "v3/management/envvars/retrieve", + "v3/management/envvars/update", + "v3/management/envvars/delete" + ] + }, + { + "group": "Projects API", + "pages": [ + "v3/management/projects/runs" ] } ] diff --git a/docs/v3-openapi.yaml b/docs/v3-openapi.yaml index c9b2198b1..c65af7807 100644 --- a/docs/v3-openapi.yaml +++ b/docs/v3-openapi.yaml @@ -39,7 +39,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -79,7 +79,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -114,7 +114,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -157,7 +157,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -189,7 +189,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -223,7 +223,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -257,7 +257,7 @@ paths: tags: - schedules security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -265,20 +265,14 @@ paths: const schedule = await schedules.activate(scheduleId); - "/api/v1/runs/{run_id}/replay": + "/api/v1/runs/{runId}/replay": + parameters: + - $ref: "#/components/parameters/runId" post: operationId: replay_run_v1 summary: Replay a run description: Creates a new run with the same payload and options as the original run. - parameters: - - in: path - name: run_id - required: true - schema: - type: string - description: The ID of an existing run. When you trigger a run you will get - an id in the response. responses: "200": description: Successful request @@ -325,9 +319,9 @@ paths: enum: - Run not found tags: - - run + - runs security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -335,19 +329,14 @@ paths: const handle = await runs.replay("run_1234"); - "/api/v1/runs/{run_id}/cancel": + "/api/v2/runs/{runId}/cancel": + parameters: + - $ref: "#/components/parameters/runId" post: operationId: cancel_run_v1 - description: Cancels a run. summary: Cancel a run - parameters: - - in: path - name: run_id - required: true - schema: - type: string - description: The ID of an existing run. When you trigger a run you will get - an id in the response. + description: Cancels an in-progress run. If the run is already completed, this + will have no effect. responses: "200": description: Successful request @@ -356,9 +345,10 @@ paths: schema: type: object properties: - message: + id: type: string - description: Confirmation message that the run was canceled. + description: The ID of the run that was canceled. + example: run_1234 "400": description: Invalid request content: @@ -394,9 +384,9 @@ paths: enum: - Run not found tags: - - run + - runs security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- @@ -404,19 +394,14 @@ paths: await runs.cancel("run_1234"); - "/api/v3/runs/{run_id}": + "/api/v3/runs/{runId}": + parameters: + - $ref: "#/components/parameters/runId" get: operationId: retrieve_run_v1 - description: Retrieve a run summary: Retrieve a run - parameters: - - in: path - name: run_id - required: true - schema: - type: string - description: The ID of an existing run. When you trigger a run you will get - an id in the response. + description: | + Retrieve information about a run, including its status, payload, output, and attempts. If you authenticate with a Public API key, we will omit the payload and output fields for security reasons. responses: "200": description: Successful request @@ -460,13 +445,182 @@ paths: tags: - run security: - - apiKey: [] + - secretKey: [] x-codeSamples: - lang: typescript source: |- import { runs } from "@trigger.dev/sdk/v3"; - await runs.retrieve("run_1234"); + const result = await runs.retrieve("run_1234"); + + // We include boolean helpers to check the status of the run + // (isSuccess, isFailed, isCompleted, etc.) + if (result.isSuccess) { + console.log("Run was successful with output", result.output); + } + + // You also have access to the run status that includes more granular information + console.log("Run status:", result.status); + + // You can access the payload and output + console.log("Payload:", result.payload); + console.log("Output:", result.output); + + // You can also access the attempts, which will give you information about errors (if they exist) + for (const attempt of result.attempts) { + if (attempt.status === "FAILED") { + console.log("Attempt failed with error:", attempt.error); + } + } + + "/api/v1/runs": + get: + operationId: list_runs_v1 + summary: List runs + description: List runs in a specific environment. You can filter the runs by status, created at, task identifier, version, and more. + parameters: + - $ref: "#/components/parameters/cursorPagination" + - $ref: "#/components/parameters/runsFilter" + responses: + "200": + description: Successful request + content: + application/json: + schema: + $ref: "#/components/schemas/ListRunsResult" + "400": + description: Invalid query parameters + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorWithDetailsResponse" + "401": + description: Unauthorized request + tags: + - runs + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + label: List runs + source: |- + import { runs } from "@trigger.dev/sdk/v3"; + + // Get the first page of runs + let page = await runs.list({ limit: 20 }); + + for (const run of page.data) { + console.log(`Run ID: ${run.id}, Status: ${run.status}`); + } + + // Convenience methods are provided for manually paginating: + while (page.hasNextPage()) { + page = await page.getNextPage(); + // Do something with the next page of runs + } + + // Auto-paginate through all runs + const allRuns = []; + + for await (const run of runs.list({ limit: 20 })) { + allRuns.push(run); + } + - lang: typescript + label: Filter runs + source: |- + import { runs } from "@trigger.dev/sdk/v3"; + + const response = await runs.list({ + status: ["QUEUED", "EXECUTING"], + taskIdentifier: ["my-task", "my-other-task"], + from: new Date("2024-04-01T00:00:00Z"), + to: new Date(), + }); + + for (const run of response.data) { + console.log(`Run ID: ${run.id}, Status: ${run.status}`); + } + + "/api/v1/projects/{projectRef}/runs": + parameters: + - $ref: "#/components/parameters/projectRef" + get: + operationId: list_project_runs_v1 + summary: List project runs + description: List runs in a project, across multiple environments, using Personal Access Token auth. You can filter the runs by status, created at, task identifier, version, and more. + parameters: + - $ref: "#/components/parameters/cursorPagination" + - $ref: "#/components/parameters/runsFilterWithEnv" + responses: + "200": + description: Successful request + content: + application/json: + schema: + $ref: "#/components/schemas/ListRunsResult" + "400": + description: Invalid request + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorWithDetailsResponse" + "401": + description: Unauthorized request + tags: + - runs + security: + - personalAccessToken: [] + x-codeSamples: + - lang: typescript + label: List runs + source: |- + import { runs, configure } from "@trigger.dev/sdk/v3"; + + configure({ + secretKey: "tr_pat_1234" // always use an environment variable for this + }); + + // Get the first page of runs + let page = await runs.list("proj_1234", { limit: 20 }); + + for (const run of page.data) { + console.log(`Run ID: ${run.id}, Status: ${run.status}`); + } + + // Convenience methods are provided for manually paginating: + while (page.hasNextPage()) { + page = await page.getNextPage(); + // Do something with the next page of runs + } + + // Auto-paginate through all runs + const allRuns = []; + + for await (const run of runs.list("proj_1234", { limit: 20 })) { + allRuns.push(run); + } + - lang: typescript + label: Filter runs + source: |- + import { runs, configure } from "@trigger.dev/sdk/v3"; + + configure({ + secretKey: "tr_pat_1234" // always use an environment variable for this + }); + + const response = await runs.list("proj_1234", { + env: ["prod", "staging"], + status: ["QUEUED", "EXECUTING"], + taskIdentifier: ["my-task", "my-other-task"], + from: new Date("2024-04-01T00:00:00Z"), + to: new Date(), + }); + + for (const run of response.data) { + console.log(`Run ID: ${run.id}, Status: ${run.status}`); + } + + "/api/v1/projects/{projectRef}/envvars/{env}": parameters: @@ -504,7 +658,7 @@ paths: tags: - envvars security: - - apiKey: [] + - secretKey: [] - personalAccessToken: [] x-codeSamples: - lang: typescript @@ -577,7 +731,7 @@ paths: tags: - envvars security: - - apiKey: [] + - secretKey: [] - personalAccessToken: [] x-codeSamples: - lang: typescript @@ -674,7 +828,7 @@ paths: tags: - envvars security: - - apiKey: [] + - secretKey: [] - personalAccessToken: [] x-codeSamples: - lang: typescript @@ -772,7 +926,7 @@ paths: tags: - envvars security: - - apiKey: [] + - secretKey: [] - personalAccessToken: [] x-codeSamples: - lang: typescript @@ -830,7 +984,7 @@ paths: tags: - envvars security: - - apiKey: [] + - secretKey: [] - personalAccessToken: [] x-codeSamples: - lang: typescript @@ -889,7 +1043,7 @@ paths: tags: - envvars security: - - apiKey: [] + - secretKey: [] - personalAccessToken: [] x-codeSamples: - lang: typescript @@ -918,6 +1072,67 @@ paths: components: parameters: + runsFilterWithEnv: + in: query + name: filter + style: deepObject + explode: true + description: | + Use this parameter to filter the runs. You can filter by created at, environment, status, task identifier, and version. + + For array fields, you can provide multiple values to filter by using a comma-separated list. For example, to get QUEUED and EXECUTING runs, you can use `filter[status]=QUEUED,EXECUTING`. + + For object fields, you should use the "form" encoding style. For example, to filter by the period, you can use `filter[createdAt][period]=1d`. + schema: + allOf: + - $ref: "#/components/schemas/CommonRunsFilter" + - $ref: "#/components/schemas/EnvFilter" + runsFilter: + in: query + name: filter + style: deepObject + explode: true + description: | + Use this parameter to filter the runs. You can filter by created at, status, task identifier, and version. + + For array fields, you can provide multiple values to filter by using a comma-separated list. For example, to get QUEUED and EXECUTING runs, you can use `filter[status]=QUEUED,EXECUTING`. + + For object fields, you should use the "form" encoding style. For example, to filter by the period, you can use `filter[createdAt][period]=1d`. + schema: + $ref: "#/components/schemas/CommonRunsFilter" + cursorPagination: + in: query + name: page + style: deepObject + explode: true + description: | + Use this parameter to paginate the results. You can specify the number of runs per page, and the ID of the run to start the page after or before. + + For object fields like `page`, you should use the "form" encoding style. For example, to get the next page of runs, you can use `page[after]=run_1234`. + schema: + type: object + properties: + size: + type: integer + maximum: 100 + minimum: 10 + default: 25 + description: Number of runs per page. Maximum is 100. + after: + type: string + description: The ID of the run to start the page after. This will set the direction of the pagination to forward. + before: + type: string + description: The ID of the run to start the page before. This will set the direction of the pagination to backward. + runId: + in: path + name: runId + required: true + schema: + type: string + description: | + The ID of an run, starts with `run_`. The run ID will be returned when you trigger a run on a task. + example: run_1234 projectRef: in: path name: projectRef @@ -944,7 +1159,7 @@ components: description: The name of the environment variable. example: SLACK_API_KEY securitySchemes: - apiKey: + secretKey: type: http scheme: bearer description: | @@ -974,6 +1189,173 @@ components: configure({ secretKey: "tr_pat_1234" }); ``` schemas: + EnvFilter: + type: object + properties: + env: + type: array + items: + type: string + description: The environment of the project + enum: + - dev + - staging + - prod + CommonRunsFilter: + type: object + properties: + createdAt: + type: object + properties: + from: + type: string + format: date-time + description: The start date to filter the runs by + to: + type: string + format: date-time + description: The end date to filter the runs by + period: + type: string + description: The period to filter the runs by + example: 1d + status: + type: array + items: + type: string + description: The status of the run + enum: + - WAITING_FOR_DEPLOY + - QUEUED + - EXECUTING + - REATTEMPTING + - FROZEN + - COMPLETED + - CANCELED + - FAILED + - CRASHED + - INTERRUPTED + - SYSTEM_FAILURE + taskIdentifier: + type: array + items: + type: string + description: The identifier of the task that was run + version: + type: array + items: + type: string + description: The version of the worker that executed the run + + bulkAction: + type: string + description: The bulk action ID to filter the runs by + example: bulk_1234 + schedule: + type: string + description: The schedule ID to filter the runs by + example: schedule_1234 + isTest: + type: boolean + description: Whether the run is a test run or not + example: false + ListRunsResult: + type: object + properties: + data: + type: array + items: + "$ref": "#/components/schemas/ListRunItem" + pagination: + type: object + properties: + next: + type: string + description: The run ID to start the next page after. This should be used as the `page[after]` parameter in the next request. + example: run_1234 + previous: + type: string + description: The run ID to start the previous page before. This should be used as the `page[before]` parameter in the next request. + example: run_5678 + ListRunItem: + type: object + required: + - id + - status + - taskIdentifier + - createdAt + - updatedAt + - isTest + - env + properties: + id: + type: string + description: The unique ID of the run, prefixed with `run_` + example: run_1234 + status: + type: string + description: The status of the run + enum: + - WAITING_FOR_DEPLOY + - QUEUED + - EXECUTING + - REATTEMPTING + - FROZEN + - COMPLETED + - CANCELED + - FAILED + - CRASHED + - INTERRUPTED + - SYSTEM_FAILURE + taskIdentifier: + type: string + description: The identifier of the task that was run + example: my-task + version: + type: string + example: 20240523.1 + description: The version of the worker that executed the run + env: + type: object + description: The environment of the run + required: + - id + - name + properties: + id: + type: string + description: The unique ID of the environment + example: cl1234 + name: + type: string + description: The name of the environment + example: dev + user: + type: string + description: If this is a dev environment, the username of the user represented by this environment + example: Anna + idempotencyKey: + type: string + description: The idempotency key used to prevent creating duplicate runs, if provided + example: idempotency_key_1234 + isTest: + type: boolean + description: Whether the run is a test run or not + example: false + createdAt: + type: string + format: date-time + updatedAt: + type: string + format: date-time + startedAt: + type: string + format: date-time + description: The time the run started + finishedAt: + type: string + format: date-time + description: The time the run finished InvalidEnvVarsRequestResponse: type: object properties: @@ -999,6 +1381,35 @@ components: error: type: string required: ["error"] + ErrorWithDetailsResponse: + type: object + properties: + error: + type: string + example: Query Error + details: + type: array + items: + type: object + required: + - code + - message + properties: + code: + type: string + description: The error code + example: custom + message: + type: string + description: The error message + example: "Invalid status values: FOOBAR" + path: + type: array + items: + type: string + description: The relevant path in the request + example: ["filter[status]"] + required: ["error"] ListEnvironmentVariablesResponse: type: array items: @@ -1032,27 +1443,95 @@ components: properties: id: type: string + description: The unique ID of the run, prefixed with `run_` + example: run_1234 status: type: string + description: The status of the run enum: - - PENDING + - WAITING_FOR_DEPLOY + - QUEUED - EXECUTING - - PAUSED + - REATTEMPTING + - FROZEN - COMPLETED - - FAILED - CANCELED + - FAILED + - CRASHED + - INTERRUPTED + - SYSTEM_FAILURE taskIdentifier: type: string - idempotencyKey: - type: string + description: The identifier of the task that was run + example: my-task version: type: string + example: 20240523.1 + description: The version of the worker that executed the run + payload: + type: object + description: The payload that was sent to the task. Will be omitted if the request was made with a Public API key + example: {"foo": "bar"} + output: + type: object + description: The output of the run. Will be omitted if the request was made with a Public API key + example: {"foo": "bar"} + idempotencyKey: + type: string + description: The idempotency key used to prevent creating duplicate runs, if provided + example: idempotency_key_1234 createdAt: type: string format: date-time updatedAt: type: string format: date-time + isTest: + type: boolean + description: Whether the run is a test run or not + example: false + startedAt: + type: string + format: date-time + description: The time the run started + finishedAt: + type: string + format: date-time + description: The time the run finished + schedule: + type: object + description: The schedule that triggered the run. Will be omitted if the run was not triggered by a schedule + required: + - id + - generator + properties: + id: + type: string + description: The unique ID of the schedule, prefixed with `sched_` + example: sched_1234 + externalId: + type: string + description: The external ID of the schedule. Can be anything that is useful to you (e.g., user ID, org ID, etc.) + example: user_1234 + deduplicationKey: + type: string + description: The deduplication key used to prevent creating duplicate schedules + example: dedup_key_1234 + generator: + type: object + properties: + type: + type: string + enum: + - CRON + expression: + type: string + description: The cron expression used to generate the schedule + example: 0 0 * * * + description: + type: string + description: The description of the generator in plain english + example: Every day at midnight attempts: type: array items: @@ -1065,6 +1544,8 @@ components: properties: id: type: string + description: The unique ID of the attempt, prefixed with `attempt_` + example: attempt_1234 status: type: string enum: @@ -1074,6 +1555,8 @@ components: - COMPLETED - FAILED - CANCELED + error: + $ref: "#/components/schemas/SerializedError" createdAt: type: string format: date-time @@ -1174,3 +1657,17 @@ components: type: string userName: type: string + SerializedError: + type: object + required: + - message + properties: + message: + type: string + example: Something went wrong + name: + type: string + example: Error + stackTrace: + type: string + example: "Error: Something went wrong" diff --git a/docs/v3/management-cancel-run.mdx b/docs/v3/management-cancel-run.mdx deleted file mode 100644 index d17ca453d..000000000 --- a/docs/v3/management-cancel-run.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Cancel run" -openapi: "v3-openapi POST /api/v1/runs/{run_id}/cancel" ---- diff --git a/docs/v3/management-get-run.mdx b/docs/v3/management-get-run.mdx deleted file mode 100644 index c74c5ecb5..000000000 --- a/docs/v3/management-get-run.mdx +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Get run" -description: "Get a run using the Task id." ---- - - diff --git a/docs/v3/management-get-runs.mdx b/docs/v3/management-get-runs.mdx deleted file mode 100644 index 887567ac4..000000000 --- a/docs/v3/management-get-runs.mdx +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Get runs" -description: "Get runs using a Task id." ---- - - diff --git a/docs/v3/management-replay-run.mdx b/docs/v3/management-replay-run.mdx deleted file mode 100644 index 8de9a1921..000000000 --- a/docs/v3/management-replay-run.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Replay run" -openapi: "v3-openapi POST /api/v1/runs/{run_id}/replay" ---- diff --git a/docs/v3/management-retrieve-run.mdx b/docs/v3/management-retrieve-run.mdx deleted file mode 100644 index d03f7c77f..000000000 --- a/docs/v3/management-retrieve-run.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Retrieve run" -openapi: "v3-openapi GET /api/v3/runs/{run_id}" ---- diff --git a/docs/v3/management-start-run.mdx b/docs/v3/management-start-run.mdx deleted file mode 100644 index 3cc8f7fe0..000000000 --- a/docs/v3/management-start-run.mdx +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Start run" -description: "Start a run using the Task id, payload and options." ---- - - diff --git a/docs/v3/management-envvars-create.mdx b/docs/v3/management/envvars/create.mdx similarity index 100% rename from docs/v3/management-envvars-create.mdx rename to docs/v3/management/envvars/create.mdx diff --git a/docs/v3/management-envvars-delete.mdx b/docs/v3/management/envvars/delete.mdx similarity index 100% rename from docs/v3/management-envvars-delete.mdx rename to docs/v3/management/envvars/delete.mdx diff --git a/docs/v3/management-envvars-import.mdx b/docs/v3/management/envvars/import.mdx similarity index 100% rename from docs/v3/management-envvars-import.mdx rename to docs/v3/management/envvars/import.mdx diff --git a/docs/v3/management-envvars-list.mdx b/docs/v3/management/envvars/list.mdx similarity index 100% rename from docs/v3/management-envvars-list.mdx rename to docs/v3/management/envvars/list.mdx diff --git a/docs/v3/management-envvars-retrieve.mdx b/docs/v3/management/envvars/retrieve.mdx similarity index 100% rename from docs/v3/management-envvars-retrieve.mdx rename to docs/v3/management/envvars/retrieve.mdx diff --git a/docs/v3/management-envvars-update.mdx b/docs/v3/management/envvars/update.mdx similarity index 100% rename from docs/v3/management-envvars-update.mdx rename to docs/v3/management/envvars/update.mdx diff --git a/docs/v3/management/overview.mdx b/docs/v3/management/overview.mdx new file mode 100644 index 000000000..80773ed44 --- /dev/null +++ b/docs/v3/management/overview.mdx @@ -0,0 +1,220 @@ +--- +title: Overview & Authentication +sidebarTitle: Overview & Authentication +description: Using the Trigger.dev v3 management API +--- + +## Installation + +The management API is available through the same `@trigger.dev/sdk` package used in defining and triggering tasks. If you have already installed the package in your project, you can skip this step. + +Make sure you use the `beta` tag when installing, as v3 is still in Developer Preview. + + + +```bash npm +npm i @trigger.dev/sdk@beta +``` + +```bash pnpm +pnpm add @trigger.dev/sdk@beta +``` + +```bash yarn +yarn add @trigger.dev/sdk@beta +``` + + + +## Usage + +All `v3` functionality is provided through the `@trigger.dev/sdk/v3` module. You can import the entire module or individual resources as needed. + +```ts +import { configure, runs } from "@trigger.dev/sdk/v3"; + +configure({ + // this is the default and if the `TRIGGER_SECRET_KEY` environment variable is set, can omit calling configure + secretKey: process.env["TRIGGER_SECRET_KEY"], +}); + +async function main() { + const runs = await runs.list({ + limit: 10, + status: ["COMPLETED"], + }); +} + +main().catch(console.error); +``` + +## Authentication + +There are two methods of authenticating with the management API: using a secret key associated with a specific environment in a project (`secretKey`), or using a personal access token (`personalAccessToken`). Both methods should only be used in a backend server, as they provide full access to the project. + + + Support for client-side authentication is coming soon to v3 but is not available at the time of + writing. + + +Certain API functions work with both authentication methods, but require different arguments depending on the method used. For example, the `runs.list` function can be called using either a `secretKey` or a `personalAccessToken`, but the `projectRef` argument is required when using a `personalAccessToken`: + +```ts +import { configure, runs } from "@trigger.dev/sdk/v3"; + +// Using secretKey authentication +configure({ + secretKey: process.env["TRIGGER_SECRET_KEY"], // starts with tr_dev_ or tr_prod_ +}); + +function secretKeyExample() { + return runs.list({ + limit: 10, + status: ["COMPLETED"], + }); +} + +// Using personalAccessToken authentication +configure({ + secretKey: process.env["TRIGGER_ACCESS_TOKEN"], // starts with tr_pat_ +}); + +function personalAccessTokenExample() { + // Notice the projectRef argument is required when using a personalAccessToken + return runs.list("prof_1234", { + limit: 10, + status: ["COMPLETED"], + projectRef: "tr_proj_1234567890", + }); +} +``` + + + Consult the following table to see which endpoints support each authentication method. + +| Endpoint | Secret key | Personal Access Token | +| ---------------------- | ---------- | --------------------- | +| `runs.list` | ✅ | ✅ | +| `runs.retrieve` | ✅ | | +| `runs.cancel` | ✅ | | +| `runs.replay` | ✅ | | +| `envvars.list` | ✅ | ✅ | +| `envvars.retrieve` | ✅ | ✅ | +| `envvars.upload` | ✅ | ✅ | +| `envvars.create` | ✅ | ✅ | +| `envvars.update` | ✅ | ✅ | +| `envvars.del` | ✅ | ✅ | +| `schedules.list` | ✅ | | +| `schedules.create` | ✅ | | +| `schedules.retrieve` | ✅ | | +| `schedules.update` | ✅ | | +| `schedules.activate` | ✅ | | +| `schedules.deactivate` | ✅ | | +| `schedules.del` | ✅ | | + + + +### Secret key + +Secret key authentication scopes the API access to a specific environment in a project, and works with certain endpoints. You can read our [API Keys guide](/v3/apikeys) for more information. + +### Personal Access Token (PAT) + +A PAT is a token associated with a specific user, and gives access to all the orgs, projects, and environments that the user has access to. You can identify a PAT by the `tr_pat_` prefix. Because a PAT does not scope access to a specific environment, you must provide the `projectRef` argument when using a PAT (and sometimes the environment as well). + +For example, when uploading environment variables using a PAT, you must provide the `projectRef` and `environment` arguments: + +```ts +import { configure, envvars } from "@trigger.dev/sdk/v3"; + +configure({ + secretKey: process.env["TRIGGER_ACCESS_TOKEN"], // starts with tr_pat_ +}); + +await envvars.upload("proj_1234", "dev", { + variables: { + MY_ENV_VAR: "MY_ENV_VAR_VALUE", + }, + override: true, +}); +``` + +## Handling errors + +When the SDK method is unable to connect to the API server, or the API server returns a non-successful response, the SDK will throw an `ApiError` that you can catch and handle: + +```ts +import { runs, APIError } from "@trigger.dev/sdk/v3"; + +async function main() { + try { + const run = await runs.retrieve("run_1234"); + } catch (error) { + if (error instanceof ApiError) { + console.error(`API error: ${error.status}, ${error.headers}, ${error.body}`); + } else { + console.error(`Unknown error: ${error.message}`); + } + } +} +``` + +## Auto-pagination + +All list endpoints in the management API support auto-pagination. +You can use `for await … of` syntax to iterate through items across all pages: + +```ts +import { runs } from "@trigger.dev/sdk/v3"; + +async function fetchAllRuns() { + const runs = []; + + for await (const run of runs.list({ limit: 10 })) { + runs.push(run); + } + + return runs; +} +``` + +You can also use helpers on the return value from any `list` method to get the next/previous page of results: + +```ts +import { runs } from "@trigger.dev/sdk/v3"; + +async function main() { + let page = await runs.list({ limit: 10 }); + + for (const run of page.data) { + console.log(run); + } + + while (page.hasNextPage()) { + page = await page.getNextPage(); + // ... do something with the next page + } +} +``` + +## Advanced usage + +### Accessing raw HTTP responses + +All API methods return a `Promise` subclass `ApiPromise` that includes helpers for accessing the underlying HTTP response: + +```ts +import { runs } from "@trigger.dev/sdk/v3"; + +async function main() { + const { data: run, response: raw } = await runs.retrieve("run_1234").withResponse(); + + console.log(raw.status); + console.log(raw.headers); + + const response = await runs.retrieve("run_1234").asResponse(); // Returns a Response object + + console.log(response.status); + console.log(response.headers); +} +``` diff --git a/docs/v3/management/projects/runs.mdx b/docs/v3/management/projects/runs.mdx new file mode 100644 index 000000000..ad5a30468 --- /dev/null +++ b/docs/v3/management/projects/runs.mdx @@ -0,0 +1,4 @@ +--- +title: "List runs" +openapi: "v3-openapi GET /api/v1/projects/{projectRef}/runs" +--- diff --git a/docs/v3/management/runs/cancel.mdx b/docs/v3/management/runs/cancel.mdx new file mode 100644 index 000000000..f99d7ff64 --- /dev/null +++ b/docs/v3/management/runs/cancel.mdx @@ -0,0 +1,4 @@ +--- +title: "Cancel run" +openapi: "v3-openapi POST /api/v2/runs/{runId}/cancel" +--- diff --git a/docs/v3/management/runs/list.mdx b/docs/v3/management/runs/list.mdx new file mode 100644 index 000000000..d674e35fd --- /dev/null +++ b/docs/v3/management/runs/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List runs" +openapi: "v3-openapi GET /api/v1/runs" +--- diff --git a/docs/v3/management/runs/replay.mdx b/docs/v3/management/runs/replay.mdx new file mode 100644 index 000000000..7028e1f9e --- /dev/null +++ b/docs/v3/management/runs/replay.mdx @@ -0,0 +1,4 @@ +--- +title: "Replay run" +openapi: "v3-openapi POST /api/v1/runs/{runId}/replay" +--- diff --git a/docs/v3/management/runs/retrieve.mdx b/docs/v3/management/runs/retrieve.mdx new file mode 100644 index 000000000..c6b0e0267 --- /dev/null +++ b/docs/v3/management/runs/retrieve.mdx @@ -0,0 +1,4 @@ +--- +title: "Retrieve run" +openapi: "v3-openapi GET /api/v3/runs/{runId}" +--- diff --git a/docs/v3/management-activate-schedule.mdx b/docs/v3/management/schedules/activate.mdx similarity index 100% rename from docs/v3/management-activate-schedule.mdx rename to docs/v3/management/schedules/activate.mdx diff --git a/docs/v3/management-create-schedule.mdx b/docs/v3/management/schedules/create.mdx similarity index 100% rename from docs/v3/management-create-schedule.mdx rename to docs/v3/management/schedules/create.mdx diff --git a/docs/v3/management-deactivate-schedule.mdx b/docs/v3/management/schedules/deactivate.mdx similarity index 100% rename from docs/v3/management-deactivate-schedule.mdx rename to docs/v3/management/schedules/deactivate.mdx diff --git a/docs/v3/management-delete-schedule.mdx b/docs/v3/management/schedules/delete.mdx similarity index 100% rename from docs/v3/management-delete-schedule.mdx rename to docs/v3/management/schedules/delete.mdx diff --git a/docs/v3/management-list-schedules.mdx b/docs/v3/management/schedules/list.mdx similarity index 100% rename from docs/v3/management-list-schedules.mdx rename to docs/v3/management/schedules/list.mdx diff --git a/docs/v3/management-retrieve-schedule.mdx b/docs/v3/management/schedules/retrieve.mdx similarity index 100% rename from docs/v3/management-retrieve-schedule.mdx rename to docs/v3/management/schedules/retrieve.mdx diff --git a/docs/v3/management-update-schedule.mdx b/docs/v3/management/schedules/update.mdx similarity index 100% rename from docs/v3/management-update-schedule.mdx rename to docs/v3/management/schedules/update.mdx diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 6b6fbcacb..3b65f4582 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -17,9 +17,8 @@ import { ImportEnvironmentVariablesRequestBody, EnvironmentVariableResponseBody, TaskRunExecution, - APIError, } from "@trigger.dev/core/v3"; -import { zodfetch } from "@trigger.dev/core/v3/zodfetch"; +import { zodfetch, ApiError } from "@trigger.dev/core/v3/zodfetch"; export class CliApiClient { private readonly apiURL: string; @@ -265,7 +264,7 @@ async function wrapZodFetch( data: response, }; } catch (error) { - if (error instanceof APIError) { + if (error instanceof ApiError) { return { success: false, error: error.message, diff --git a/packages/core/src/v3/apiClient/core.ts b/packages/core/src/v3/apiClient/core.ts new file mode 100644 index 000000000..ef77bbe83 --- /dev/null +++ b/packages/core/src/v3/apiClient/core.ts @@ -0,0 +1,687 @@ +import { z } from "zod"; +import { fromZodError } from "zod-validation-error"; +import { ApiConnectionError, ApiError } from "./errors"; +import { RetryOptions } from "../schemas"; +import { calculateNextRetryDelay } from "../utils/retries"; +import { FormDataEncoder } from "form-data-encoder"; +import { Readable } from "node:stream"; +import { + CursorPage, + CursorPageParams, + CursorPageResponse, + OffsetLimitPage, + OffsetLimitPageParams, + OffsetLimitPageResponse, +} from "./pagination"; + +export const defaultRetryOptions = { + maxAttempts: 3, + factor: 2, + minTimeoutInMs: 1000, + maxTimeoutInMs: 60000, + randomize: false, +} satisfies RetryOptions; + +export type ZodFetchOptions = { + retry?: RetryOptions; +}; + +interface FetchCursorPageParams extends CursorPageParams { + query?: URLSearchParams; +} + +interface FetchOffsetLimitPageParams extends OffsetLimitPageParams { + query?: URLSearchParams; +} + +export function zodfetch( + schema: TResponseBodySchema, + url: string, + requestInit?: RequestInit, + options?: ZodFetchOptions +): ApiPromise> { + return new ApiPromise(_doZodFetch(schema, url, requestInit, options)); +} + +export function zodfetchCursorPage( + schema: TItemSchema, + url: string, + params: FetchCursorPageParams, + requestInit?: RequestInit, + options?: ZodFetchOptions +) { + const query = new URLSearchParams(params.query); + + if (params.limit) { + query.set("page[size]", String(params.limit)); + } + + if (params.after) { + query.set("page[after]", params.after); + } + + if (params.before) { + query.set("page[before]", params.before); + } + + const cursorPageSchema = z.object({ + data: z.array(schema), + pagination: z.object({ + next: z.string().optional(), + previous: z.string().optional(), + }), + }); + + const $url = new URL(url); + $url.search = query.toString(); + + const fetchResult = _doZodFetch(cursorPageSchema, $url.href, requestInit, options); + + return new CursorPagePromise(fetchResult, schema, url, params, requestInit, options); +} + +export function zodfetchOffsetLimitPage( + schema: TItemSchema, + url: string, + params: FetchOffsetLimitPageParams, + requestInit?: RequestInit, + options?: ZodFetchOptions +) { + const query = new URLSearchParams(params.query); + + if (params.limit) { + query.set("perPage", String(params.limit)); + } + + if (params.page) { + query.set("page", String(params.page)); + } + + const offsetLimitPageSchema = z.object({ + data: z.array(schema), + pagination: z.object({ + currentPage: z.coerce.number(), + totalPages: z.coerce.number(), + count: z.coerce.number(), + }), + }); + + const $url = new URL(url); + $url.search = query.toString(); + + const fetchResult = _doZodFetch(offsetLimitPageSchema, $url.href, requestInit, options); + + return new OffsetLimitPagePromise(fetchResult, schema, url, params, requestInit, options); +} + +export function zodupload< + TResponseBodySchema extends z.ZodTypeAny, + TBody = Record, +>( + schema: TResponseBodySchema, + url: string, + body: TBody, + requestInit?: RequestInit, + options?: ZodFetchOptions +): ApiPromise> { + const finalRequestInit = createMultipartFormRequestInit(body, requestInit); + + return new ApiPromise(_doZodFetch(schema, url, finalRequestInit, options)); +} + +async function createMultipartFormRequestInit>( + body: TBody, + requestInit?: RequestInit +): Promise { + const form = await createForm(body); + const encoder = new FormDataEncoder(form); + + const finalHeaders: Record = {}; + + for (const [key, value] of Object.entries(requestInit?.headers || {})) { + finalHeaders[key] = value as string; + } + + for (const [key, value] of Object.entries(encoder.headers)) { + finalHeaders[key] = value; + } + + finalHeaders["Content-Length"] = String(encoder.contentLength); + + const finalRequestInit: RequestInit = { + ...requestInit, + headers: finalHeaders, + body: Readable.from(encoder) as any, + // @ts-expect-error + duplex: "half", + }; + + return finalRequestInit; +} + +const createForm = async >(body: T | undefined): Promise => { + const form = new FormData(); + await Promise.all( + Object.entries(body || {}).map(([key, value]) => addFormValue(form, key, value)) + ); + return form; +}; + +type ZodFetchResult = { + data: T; + response: Response; +}; + +type PromiseOrValue = T | Promise; + +async function _doZodFetch( + schema: TResponseBodySchema, + url: string, + requestInit?: PromiseOrValue, + options?: ZodFetchOptions, + attempt = 1 +): Promise>> { + try { + const $requestInit = await requestInit; + + const response = await fetch(url, requestInitWithCache($requestInit)); + + const responseHeaders = createResponseHeaders(response.headers); + + if (!response.ok) { + const retryResult = shouldRetry(response, attempt, options?.retry); + + if (retryResult.retry) { + await new Promise((resolve) => setTimeout(resolve, retryResult.delay)); + + return await _doZodFetch(schema, url, requestInit, options, attempt + 1); + } else { + const errText = await response.text().catch((e) => castToError(e).message); + const errJSON = safeJsonParse(errText); + const errMessage = errJSON ? undefined : errText; + + throw ApiError.generate(response.status, errJSON, errMessage, responseHeaders); + } + } + + const jsonBody = await response.json(); + const parsedResult = schema.safeParse(jsonBody); + + if (parsedResult.success) { + return { data: parsedResult.data, response }; + } + + throw fromZodError(parsedResult.error); + } catch (error) { + if (error instanceof ApiError) { + throw error; + } + + if (options?.retry) { + const retry = { ...defaultRetryOptions, ...options.retry }; + + const delay = calculateNextRetryDelay(retry, attempt); + + if (delay) { + await new Promise((resolve) => setTimeout(resolve, delay)); + + return await _doZodFetch(schema, url, requestInit, options, attempt + 1); + } + } + + throw new ApiConnectionError({ cause: castToError(error) }); + } +} + +function castToError(err: any): Error { + if (err instanceof Error) return err; + return new Error(err); +} + +type ShouldRetryResult = + | { + retry: false; + } + | { + retry: true; + delay: number; + }; + +function shouldRetry( + response: Response, + attempt: number, + retryOptions?: RetryOptions +): ShouldRetryResult { + function shouldRetryForOptions(): ShouldRetryResult { + const retry = { ...defaultRetryOptions, ...retryOptions }; + + const delay = calculateNextRetryDelay(retry, attempt); + + if (delay) { + return { retry: true, delay }; + } else { + return { retry: false }; + } + } + + // Note this is not a standard header. + const shouldRetryHeader = response.headers.get("x-should-retry"); + + // If the server explicitly says whether or not to retry, obey. + if (shouldRetryHeader === "true") return shouldRetryForOptions(); + if (shouldRetryHeader === "false") return { retry: false }; + + // Retry on request timeouts. + if (response.status === 408) return shouldRetryForOptions(); + + // Retry on lock timeouts. + if (response.status === 409) return shouldRetryForOptions(); + + // Retry on rate limits. + if (response.status === 429) return shouldRetryForOptions(); + + // Retry internal errors. + if (response.status >= 500) return shouldRetryForOptions(); + + return { retry: false }; +} + +function safeJsonParse(text: string): any { + try { + return JSON.parse(text); + } catch (e) { + return undefined; + } +} + +function createResponseHeaders(headers: Response["headers"]): Record { + return new Proxy( + Object.fromEntries( + // @ts-ignore + headers.entries() + ), + { + get(target, name) { + const key = name.toString(); + return target[key.toLowerCase()] || target[key]; + }, + } + ); +} + +function requestInitWithCache(requestInit?: RequestInit): RequestInit { + try { + const withCache: RequestInit = { + ...requestInit, + cache: "no-cache", + }; + + const _ = new Request("http://localhost", withCache); + + return withCache; + } catch (error) { + return requestInit ?? {}; + } +} + +const addFormValue = async (form: FormData, key: string, value: unknown): Promise => { + if (value === undefined) return; + if (value == null) { + throw new TypeError( + `Received null for "${key}"; to pass null in FormData, you must use the string 'null'` + ); + } + + // TODO: make nested formats configurable + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + form.append(key, String(value)); + } else if ( + isUploadable(value) || + isBlobLike(value) || + value instanceof Buffer || + value instanceof ArrayBuffer + ) { + const file = await toFile(value); + form.append(key, file as File); + } else if (Array.isArray(value)) { + await Promise.all(value.map((entry) => addFormValue(form, key + "[]", entry))); + } else if (typeof value === "object") { + await Promise.all( + Object.entries(value).map(([name, prop]) => addFormValue(form, `${key}[${name}]`, prop)) + ); + } else { + throw new TypeError( + `Invalid value given to form, expected a string, number, boolean, object, Array, File or Blob but got ${value} instead` + ); + } +}; + +export type ToFileInput = Uploadable | Exclude | AsyncIterable; + +/** + * Helper for creating a {@link File} to pass to an SDK upload method from a variety of different data formats + * @param value the raw content of the file. Can be an {@link Uploadable}, {@link BlobLikePart}, or {@link AsyncIterable} of {@link BlobLikePart}s + * @param {string=} name the name of the file. If omitted, toFile will try to determine a file name from bits if possible + * @param {Object=} options additional properties + * @param {string=} options.type the MIME type of the content + * @param {number=} options.lastModified the last modified timestamp + * @returns a {@link File} with the given properties + */ +export async function toFile( + value: ToFileInput | PromiseLike, + name?: string | null | undefined, + options?: FilePropertyBag | undefined +): Promise { + // If it's a promise, resolve it. + value = await value; + + // Use the file's options if there isn't one provided + options ??= isFileLike(value) ? { lastModified: value.lastModified, type: value.type } : {}; + + if (isResponseLike(value)) { + const blob = await value.blob(); + name ||= new URL(value.url).pathname.split(/[\\/]/).pop() ?? "unknown_file"; + + return new File([blob as any], name, options); + } + + const bits = await getBytes(value); + + name ||= getName(value) ?? "unknown_file"; + + if (!options.type) { + const type = (bits[0] as any)?.type; + if (typeof type === "string") { + options = { ...options, type }; + } + } + + return new File(bits, name, options); +} + +function getName(value: any): string | undefined { + return ( + getStringFromMaybeBuffer(value.name) || + getStringFromMaybeBuffer(value.filename) || + // For fs.ReadStream + getStringFromMaybeBuffer(value.path)?.split(/[\\/]/).pop() + ); +} + +const getStringFromMaybeBuffer = (x: string | Buffer | unknown): string | undefined => { + if (typeof x === "string") return x; + if (typeof Buffer !== "undefined" && x instanceof Buffer) return String(x); + return undefined; +}; + +async function getBytes(value: ToFileInput): Promise> { + let parts: Array = []; + if ( + typeof value === "string" || + ArrayBuffer.isView(value) || // includes Uint8Array, Buffer, etc. + value instanceof ArrayBuffer + ) { + parts.push(value); + } else if (isBlobLike(value)) { + parts.push(await value.arrayBuffer()); + } else if ( + isAsyncIterableIterator(value) // includes Readable, ReadableStream, etc. + ) { + for await (const chunk of value) { + parts.push(chunk as BlobPart); // TODO, consider validating? + } + } else { + throw new Error( + `Unexpected data type: ${typeof value}; constructor: ${value?.constructor + ?.name}; props: ${propsForError(value)}` + ); + } + + return parts; +} + +function propsForError(value: any): string { + const props = Object.getOwnPropertyNames(value); + return `[${props.map((p) => `"${p}"`).join(", ")}]`; +} + +const isAsyncIterableIterator = (value: any): value is AsyncIterableIterator => + value != null && typeof value === "object" && typeof value[Symbol.asyncIterator] === "function"; + +/** + * Intended to match web.Blob, node.Blob, node-fetch.Blob, etc. + */ +export interface BlobLike { + /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) */ + readonly size: number; + /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) */ + readonly type: string; + /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) */ + text(): Promise; + /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) */ + slice(start?: number, end?: number): BlobLike; + // unfortunately @types/node-fetch@^2.6.4 doesn't type the arrayBuffer method +} + +/** + * Intended to match web.File, node.File, node-fetch.File, etc. + */ +export interface FileLike extends BlobLike { + /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) */ + readonly lastModified: number; + /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) */ + readonly name: string; +} + +/** + * Intended to match web.Response, node.Response, node-fetch.Response, etc. + */ +export interface ResponseLike { + url: string; + blob(): Promise; +} + +export type Uploadable = FileLike | ResponseLike | Readable; + +export const isResponseLike = (value: any): value is ResponseLike => + value != null && + typeof value === "object" && + typeof value.url === "string" && + typeof value.blob === "function"; + +export const isFileLike = (value: any): value is FileLike => + value != null && + typeof value === "object" && + typeof value.name === "string" && + typeof value.lastModified === "number" && + isBlobLike(value); + +/** + * The BlobLike type omits arrayBuffer() because @types/node-fetch@^2.6.4 lacks it; but this check + * adds the arrayBuffer() method type because it is available and used at runtime + */ +export const isBlobLike = ( + value: any +): value is BlobLike & { arrayBuffer(): Promise } => + value != null && + typeof value === "object" && + typeof value.size === "number" && + typeof value.type === "string" && + typeof value.text === "function" && + typeof value.slice === "function" && + typeof value.arrayBuffer === "function"; + +export const isFsReadStream = (value: any): value is Readable => value instanceof Readable; + +export const isUploadable = (value: any): value is Uploadable => { + return isFileLike(value) || isResponseLike(value) || isFsReadStream(value); +}; + +export type BlobLikePart = + | string + | ArrayBuffer + | ArrayBufferView + | BlobLike + | Uint8Array + | DataView; + +export const isRecordLike = (value: any): value is Record => + value != null && + typeof value === "object" && + !Array.isArray(value) && + Object.keys(value).length > 0 && + Object.keys(value).every((key) => typeof key === "string" && typeof value[key] === "string"); + +/** + * A subclass of `Promise` providing additional helper methods + * for interacting with the SDK. + */ +export class ApiPromise extends Promise { + constructor(private responsePromise: Promise>) { + super((resolve) => { + // this is maybe a bit weird but this has to be a no-op to not implicitly + // parse the response body; instead .then, .catch, .finally are overridden + // to parse the response + resolve(null as any); + }); + } + + /** + * Gets the raw `Response` instance instead of parsing the response + * data. + * + * If you want to parse the response body but still get the `Response` + * instance, you can use {@link withResponse()}. + */ + asResponse(): Promise { + return this.responsePromise.then((p) => p.response); + } + + /** + * Gets the parsed response data and the raw `Response` instance. + * + * If you just want to get the raw `Response` instance without parsing it, + * you can use {@link asResponse()}. + */ + async withResponse(): Promise<{ data: T; response: Response }> { + const [data, response] = await Promise.all([this.parse(), this.asResponse()]); + return { data, response }; + } + + private parse(): Promise { + return this.responsePromise.then((result) => result.data); + } + + override then( + onfulfilled?: ((value: T) => TResult1 | PromiseLike) | undefined | null, + onrejected?: ((reason: any) => TResult2 | PromiseLike) | undefined | null + ): Promise { + return this.parse().then(onfulfilled, onrejected); + } + + override catch( + onrejected?: ((reason: any) => TResult | PromiseLike) | undefined | null + ): Promise { + return this.parse().catch(onrejected); + } + + override finally(onfinally?: (() => void) | undefined | null): Promise { + return this.parse().finally(onfinally); + } +} + +export class CursorPagePromise + extends ApiPromise>> + implements AsyncIterable> +{ + constructor( + result: Promise>>>, + private schema: TItemSchema, + private url: string, + private params: FetchCursorPageParams, + private requestInit?: RequestInit, + private options?: ZodFetchOptions + ) { + super( + result.then((result) => ({ + data: new CursorPage(result.data.data, result.data.pagination, this.#fetchPage.bind(this)), + response: result.response, + })) + ); + } + + #fetchPage(params: Omit): Promise>> { + return zodfetchCursorPage( + this.schema, + this.url, + { ...this.params, ...params }, + this.requestInit, + this.options + ); + } + + /** + * Allow auto-paginating iteration on an unawaited list call, eg: + * + * for await (const item of client.items.list()) { + * console.log(item) + * } + */ + async *[Symbol.asyncIterator]() { + const page = await this; + for await (const item of page) { + yield item; + } + } +} + +export class OffsetLimitPagePromise + extends ApiPromise>> + implements AsyncIterable> +{ + constructor( + result: Promise>>>, + private schema: TItemSchema, + private url: string, + private params: FetchOffsetLimitPageParams, + private requestInit?: RequestInit, + private options?: ZodFetchOptions + ) { + super( + result.then((result) => ({ + data: new OffsetLimitPage( + result.data.data, + result.data.pagination, + this.#fetchPage.bind(this) + ), + response: result.response, + })) + ); + } + + #fetchPage( + params: Omit + ): Promise>> { + return zodfetchOffsetLimitPage( + this.schema, + this.url, + { ...this.params, ...params }, + this.requestInit, + this.options + ); + } + + /** + * Allow auto-paginating iteration on an unawaited list call, eg: + * + * for await (const item of client.items.list()) { + * console.log(item) + * } + */ + async *[Symbol.asyncIterator]() { + const page = await this; + for await (const item of page) { + yield item; + } + } +} diff --git a/packages/core/src/v3/apiErrors.ts b/packages/core/src/v3/apiClient/errors.ts similarity index 81% rename from packages/core/src/v3/apiErrors.ts rename to packages/core/src/v3/apiClient/errors.ts index 5097bd4e2..a0384375b 100644 --- a/packages/core/src/v3/apiErrors.ts +++ b/packages/core/src/v3/apiClient/errors.ts @@ -1,6 +1,6 @@ export type APIHeaders = Record; -export class APIError extends Error { +export class ApiError extends Error { readonly status: number | undefined; readonly headers: APIHeaders | undefined; readonly error: Object | undefined; @@ -15,7 +15,7 @@ export class APIError extends Error { message: string | undefined, headers: APIHeaders | undefined ) { - super(`${APIError.makeMessage(status, error, message)}`); + super(`${ApiError.makeMessage(status, error, message)}`); this.status = status; this.headers = headers; @@ -54,7 +54,7 @@ export class APIError extends Error { headers: APIHeaders | undefined ) { if (!status) { - return new APIConnectionError({ cause: castToError(errorResponse) }); + return new ApiConnectionError({ cause: castToError(errorResponse) }); } const error = (errorResponse as Record)?.["error"]; @@ -91,11 +91,11 @@ export class APIError extends Error { return new InternalServerError(status, error, message, headers); } - return new APIError(status, error, message, headers); + return new ApiError(status, error, message, headers); } } -export class APIConnectionError extends APIError { +export class ApiConnectionError extends ApiError { override readonly status: undefined = undefined; constructor({ message, cause }: { message?: string; cause?: Error | undefined }) { @@ -106,35 +106,35 @@ export class APIConnectionError extends APIError { } } -export class BadRequestError extends APIError { +export class BadRequestError extends ApiError { override readonly status: 400 = 400; } -export class AuthenticationError extends APIError { +export class AuthenticationError extends ApiError { override readonly status: 401 = 401; } -export class PermissionDeniedError extends APIError { +export class PermissionDeniedError extends ApiError { override readonly status: 403 = 403; } -export class NotFoundError extends APIError { +export class NotFoundError extends ApiError { override readonly status: 404 = 404; } -export class ConflictError extends APIError { +export class ConflictError extends ApiError { override readonly status: 409 = 409; } -export class UnprocessableEntityError extends APIError { +export class UnprocessableEntityError extends ApiError { override readonly status: 422 = 422; } -export class RateLimitError extends APIError { +export class RateLimitError extends ApiError { override readonly status: 429 = 429; } -export class InternalServerError extends APIError {} +export class InternalServerError extends ApiError {} function castToError(err: any): Error { if (err instanceof Error) return err; diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index 00449eb40..166ec7409 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1,6 +1,5 @@ import { context, propagation } from "@opentelemetry/api"; import { version } from "../../../package.json"; -import { APIError } from "../apiErrors"; import { BatchTaskRunExecutionResult, BatchTriggerTaskRequestBody, @@ -13,6 +12,7 @@ import { EnvironmentVariableResponseBody, EnvironmentVariableValue, EnvironmentVariables, + ListRunResponseItem, ListScheduleOptions, ListSchedulesResult, ReplayRunResponse, @@ -25,16 +25,27 @@ import { UpdateScheduleOptions, } from "../schemas"; import { taskContext } from "../task-context-api"; -import { ZodFetchOptions, isRecordLike, zodfetch, zodupload } from "../zodfetch"; import { - ImportEnvironmentVariablesParams, + CursorPagePromise, + ZodFetchOptions, + isRecordLike, + zodfetch, + zodfetchCursorPage, + zodfetchOffsetLimitPage, + zodupload, +} from "./core"; +import { ApiError } from "./errors"; +import { CreateEnvironmentVariableParams, + ImportEnvironmentVariablesParams, + ListProjectRunsQueryParams, + ListRunsQueryParams, UpdateEnvironmentVariableParams, } from "./types"; export type { - ImportEnvironmentVariablesParams, CreateEnvironmentVariableParams, + ImportEnvironmentVariablesParams, UpdateEnvironmentVariableParams, }; @@ -77,7 +88,7 @@ export class ApiClient { zodFetchOptions ); } catch (error) { - if (error instanceof APIError) { + if (error instanceof ApiError) { if (error.status === 404) { return undefined; } @@ -161,6 +172,56 @@ export class ApiClient { ); } + listRuns(query?: ListRunsQueryParams): CursorPagePromise { + const searchParams = createSearchQueryForListRuns(query); + + return zodfetchCursorPage( + ListRunResponseItem, + `${this.baseUrl}/api/v1/runs`, + { + query: searchParams, + limit: query?.limit, + after: query?.after, + before: query?.before, + }, + { + method: "GET", + headers: this.#getHeaders(false), + }, + zodFetchOptions + ); + } + + listProjectRuns( + projectRef: string, + query?: ListProjectRunsQueryParams + ): CursorPagePromise { + const searchParams = createSearchQueryForListRuns(query); + + if (query?.env) { + searchParams.append( + "filter[env]", + Array.isArray(query.env) ? query.env.join(",") : query.env + ); + } + + return zodfetchCursorPage( + ListRunResponseItem, + `${this.baseUrl}/api/v1/projects/${projectRef}/runs`, + { + query: searchParams, + limit: query?.limit, + after: query?.after, + before: query?.before, + }, + { + method: "GET", + headers: this.#getHeaders(false), + }, + zodFetchOptions + ); + } + replayRun(runId: string) { return zodfetch( ReplayRunResponse, @@ -204,9 +265,13 @@ export class ApiClient { searchParams.append("perPage", options.perPage.toString()); } - return zodfetch( - ListSchedulesResult, - `${this.baseUrl}/api/v1/schedules${searchParams.size > 0 ? `?${searchParams}` : ""}`, + return zodfetchOffsetLimitPage( + ScheduleObject, + `${this.baseUrl}/api/v1/schedules`, + { + page: options?.page, + limit: options?.perPage, + }, { method: "GET", headers: this.#getHeaders(false), @@ -356,3 +421,62 @@ export class ApiClient { return headers; } } + +function createSearchQueryForListRuns(query?: ListRunsQueryParams): URLSearchParams { + const searchParams = new URLSearchParams(); + + if (query) { + if (query.status) { + searchParams.append( + "filter[status]", + Array.isArray(query.status) ? query.status.join(",") : query.status + ); + } + + if (query.taskIdentifier) { + searchParams.append( + "filter[taskIdentifier]", + Array.isArray(query.taskIdentifier) ? query.taskIdentifier.join(",") : query.taskIdentifier + ); + } + + if (query.version) { + searchParams.append( + "filter[version]", + Array.isArray(query.version) ? query.version.join(",") : query.version + ); + } + + if (query.bulkAction) { + searchParams.append("filter[bulkAction]", query.bulkAction); + } + + if (query.schedule) { + searchParams.append("filter[schedule]", query.schedule); + } + + if (typeof query.isTest === "boolean") { + searchParams.append("filter[isTest]", String(query.isTest)); + } + + if (query.from) { + searchParams.append( + "filter[createdAt][from]", + query.from instanceof Date ? query.from.getTime().toString() : query.from.toString() + ); + } + + if (query.to) { + searchParams.append( + "filter[createdAt][to]", + query.to instanceof Date ? query.to.getTime().toString() : query.to.toString() + ); + } + + if (query.period) { + searchParams.append("filter[createdAt][period]", query.period); + } + } + + return searchParams; +} diff --git a/packages/core/src/v3/apiClient/pagination.ts b/packages/core/src/v3/apiClient/pagination.ts new file mode 100644 index 000000000..7613651ba --- /dev/null +++ b/packages/core/src/v3/apiClient/pagination.ts @@ -0,0 +1,163 @@ +export interface CursorPageParams { + limit?: number; + after?: string; + before?: string; +} + +export interface OffsetLimitPageParams { + limit?: number; + page?: number; +} + +export interface PageResponse { + data: Array; +} + +export interface CursorPageResponse extends PageResponse { + pagination: { + next?: string; + previous?: string; + }; +} + +export interface OffsetLimitPageResponse extends PageResponse { + pagination: { + currentPage: number; + totalPages: number; + count: number; + }; +} + +export interface Page { + getPaginatedItems(): Item[]; + hasNextPage(): boolean; + hasPreviousPage(): boolean; +} + +export class CursorPage implements CursorPageResponse, Page, AsyncIterable { + data: Array; + pagination: { next?: string; previous?: string }; + + constructor( + data: Array, + pagination: { next?: string; previous?: string }, + private pageFetcher: (params: Omit) => Promise> + ) { + this.data = data; + this.pagination = pagination; + } + + getPaginatedItems(): Item[] { + return this.data ?? []; + } + + hasNextPage(): boolean { + return !!this.pagination.next; + } + + hasPreviousPage(): boolean { + return !!this.pagination.previous; + } + + getNextPage(): Promise> { + if (!this.pagination.next) { + throw new Error("No next page available"); + } + + return this.pageFetcher({ after: this.pagination.next }); + } + + getPreviousPage(): Promise> { + if (!this.pagination.previous) { + throw new Error("No previous page available"); + } + + return this.pageFetcher({ before: this.pagination.previous }); + } + + async *iterPages() { + // eslint-disable-next-line @typescript-eslint/no-this-alias + let page: CursorPage = this; + yield page; + while (page.hasNextPage()) { + page = await page.getNextPage(); + yield page; + } + } + + async *[Symbol.asyncIterator]() { + for await (const page of this.iterPages()) { + for (const item of page.getPaginatedItems()) { + yield item; + } + } + } +} + +export class OffsetLimitPage + implements OffsetLimitPageResponse, Page, AsyncIterable +{ + data: Array; + pagination: { currentPage: number; totalPages: number; count: number }; + + constructor( + data: Array, + pagination: { currentPage: number; totalPages: number; count: number }, + private pageFetcher: ( + params: Omit + ) => Promise> + ) { + this.data = data; + this.pagination = pagination; + } + + getPaginatedItems(): Item[] { + return this.data ?? []; + } + + hasNextPage(): boolean { + return this.pagination.currentPage < this.pagination.totalPages; + } + + hasPreviousPage(): boolean { + return this.pagination.currentPage > 1; + } + + getNextPage(): Promise> { + if (!this.hasNextPage()) { + throw new Error("No next page available"); + } + + return this.pageFetcher({ + page: this.pagination.currentPage + 1, + }); + } + + getPreviousPage(): Promise> { + if (!this.hasPreviousPage()) { + throw new Error("No previous page available"); + } + + return this.pageFetcher({ + page: this.pagination.currentPage - 1, + }); + } + + async *iterPages() { + // eslint-disable-next-line @typescript-eslint/no-this-alias + let page: OffsetLimitPage = this; + yield page; + while (page.hasNextPage()) { + page = await page.getNextPage(); + yield page; + } + } + + async *[Symbol.asyncIterator]() { + for await (const page of this.iterPages()) { + for (const item of page.getPaginatedItems()) { + yield item; + } + } + } +} diff --git a/packages/core/src/v3/apiClient/types.ts b/packages/core/src/v3/apiClient/types.ts index 602c0079e..eb3394ec9 100644 --- a/packages/core/src/v3/apiClient/types.ts +++ b/packages/core/src/v3/apiClient/types.ts @@ -1,4 +1,6 @@ -import { BlobLikePart, Uploadable } from "../zodfetch"; +import { RunStatus } from "../schemas"; +import { BlobLikePart, Uploadable } from "./core"; +import { CursorPageParams } from "./pagination"; export interface ImportEnvironmentVariablesParams { /** @@ -22,3 +24,19 @@ export interface CreateEnvironmentVariableParams { export interface UpdateEnvironmentVariableParams { value: string; } + +export interface ListRunsQueryParams extends CursorPageParams { + status?: Array | RunStatus; + taskIdentifier?: Array | string; + version?: Array | string; + from?: Date | number; + to?: Date | number; + period?: string; + bulkAction?: string; + schedule?: string; + isTest?: boolean; +} + +export interface ListProjectRunsQueryParams extends CursorPageParams, ListRunsQueryParams { + env?: Array<"dev" | "staging" | "prod"> | "dev" | "staging" | "prod"; +} diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index bc12e75d7..9a7c985a5 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -54,7 +54,15 @@ export function createErrorTaskError(error: TaskRunError): any { } } -export function createJsonErrorObject(error: TaskRunError) { +export const SerializedError = z.object({ + message: z.string(), + name: z.string().optional(), + stackTrace: z.string().optional(), +}); + +export type SerializedError = z.infer; + +export function createJsonErrorObject(error: TaskRunError): SerializedError { switch (error.type) { case "BUILT_IN_ERROR": { return { diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 8135153b4..7673d00ed 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -1,5 +1,8 @@ export * from "./apiClient"; -export * from "./apiErrors"; +export * from "./apiClient/types"; +export * from "./apiClient/pagination"; +export type { ApiPromise } from "./apiClient/core"; +export * from "./apiClient/errors"; export * from "./clock-api"; export * from "./errors"; export * from "./limits"; diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 0a14bb2f7..e7ee977a1 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { BackgroundWorkerMetadata, ImageDetailsMetadata } from "./resources"; import { QueueOptions } from "./schemas"; +import { SerializedError } from "../errors"; export const WhoAmIResponseSchema = z.object({ userId: z.string(), @@ -210,7 +211,7 @@ export const ReplayRunResponse = z.object({ export type ReplayRunResponse = z.infer; export const CanceledRunResponse = z.object({ - message: z.string(), + id: z.string(), }); export type CanceledRunResponse = z.infer; @@ -272,17 +273,21 @@ export const UpdateScheduleOptions = CreateScheduleOptions; export type UpdateScheduleOptions = z.infer; +export const ScheduleGenerator = z.object({ + type: z.literal("CRON"), + expression: z.string(), + description: z.string(), +}); + +export type ScheduleGenerator = z.infer; + export const ScheduleObject = z.object({ id: z.string(), task: z.string(), active: z.boolean(), deduplicationKey: z.string().nullish(), externalId: z.string().nullish(), - generator: z.object({ - type: z.literal("CRON"), - expression: z.string(), - description: z.string(), - }), + generator: ScheduleGenerator, nextRun: z.coerce.date().nullish(), environments: z.array( z.object({ @@ -320,12 +325,28 @@ export const ListScheduleOptions = z.object({ export type ListScheduleOptions = z.infer; export const RunStatus = z.enum([ - "PENDING", + /// Task hasn't been deployed yet but is waiting to be executed + "WAITING_FOR_DEPLOY", + /// Task is waiting to be executed by a worker + "QUEUED", + /// Task is currently being executed by a worker "EXECUTING", - "PAUSED", + /// Task has failed and is waiting to be retried + "REATTEMPTING", + /// Task has been paused by the system, and will be resumed by the system + "FROZEN", + /// Task has been completed successfully "COMPLETED", - "FAILED", + /// Task has been canceled by the user "CANCELED", + /// Task has been completed with errors + "FAILED", + /// Task has crashed and won't be retried, most likely the worker ran out of resources, e.g. memory or storage + "CRASHED", + /// Task was interrupted during execution, mostly this happens in development environments + "INTERRUPTED", + /// Task has failed to complete, due to an error in the system + "SYSTEM_FAILURE", ]); export type RunStatus = z.infer; @@ -341,14 +362,47 @@ export const AttemptStatus = z.enum([ export type AttemptStatus = z.infer; -export const RetrieveRunResponse = z.object({ +export const RunEnvironmentDetails = z.object({ + id: z.string(), + name: z.string(), + user: z.string().optional(), +}); + +export type RunEnvironmentDetails = z.infer; + +export const RunScheduleDetails = z.object({ + id: z.string(), + externalId: z.string().optional(), + deduplicationKey: z.string().optional(), + generator: ScheduleGenerator, +}); + +export type RunScheduleDetails = z.infer; + +const CommonRunFields = { id: z.string(), status: RunStatus, taskIdentifier: z.string(), idempotencyKey: z.string().optional(), version: z.string().optional(), + isQueued: z.boolean(), + isExecuting: z.boolean(), + isCompleted: z.boolean(), + isSuccess: z.boolean(), + isFailed: z.boolean(), + isCancelled: z.boolean(), + isTest: z.boolean(), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), +}; + +export const RetrieveRunResponse = z.object({ + ...CommonRunFields, + payload: z.any().optional(), + output: z.any().optional(), + schedule: RunScheduleDetails.optional(), attempts: z.array( z .object({ @@ -358,6 +412,7 @@ export const RetrieveRunResponse = z.object({ updatedAt: z.coerce.date(), startedAt: z.coerce.date().optional(), completedAt: z.coerce.date().optional(), + error: SerializedError.optional(), }) .optional() ), @@ -365,6 +420,23 @@ export const RetrieveRunResponse = z.object({ export type RetrieveRunResponse = z.infer; +export const ListRunResponseItem = z.object({ + ...CommonRunFields, + env: RunEnvironmentDetails, +}); + +export type ListRunResponseItem = z.infer; + +export const ListRunResponse = z.object({ + data: z.array(ListRunResponseItem), + pagination: z.object({ + next: z.string().optional(), + previous: z.string().optional(), + }), +}); + +export type ListRunResponse = z.infer; + export const CreateEnvironmentVariableRequestBody = z.object({ name: z.string(), value: z.string(), diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index 2d73b74dd..a13d71f23 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -176,7 +176,16 @@ export const TaskFileMetadata = z.object({ export type TaskFileMetadata = z.infer; -export const TaskMetadataWithFilePath = TaskMetadata.merge(TaskFileMetadata); +export const TaskMetadataWithFilePath = z.object({ + id: z.string(), + packageVersion: z.string(), + queue: QueueOptions.optional(), + retry: RetryOptions.optional(), + machine: Machine.partial().optional(), + triggerSource: z.string().optional(), + filePath: z.string(), + exportName: z.string(), +}); export type TaskMetadataWithFilePath = z.infer; diff --git a/packages/core/src/v3/zodfetch.ts b/packages/core/src/v3/zodfetch.ts index ac15f38e3..a16193df4 100644 --- a/packages/core/src/v3/zodfetch.ts +++ b/packages/core/src/v3/zodfetch.ts @@ -1,437 +1,3 @@ -import { z } from "zod"; -import { fromZodError } from "zod-validation-error"; -import { APIConnectionError, APIError } from "./apiErrors"; -import { RetryOptions } from "./schemas"; -import { calculateNextRetryDelay } from "./utils/retries"; -import { FormDataEncoder } from "form-data-encoder"; -import { Readable } from "node:stream"; - -export const defaultRetryOptions = { - maxAttempts: 3, - factor: 2, - minTimeoutInMs: 1000, - maxTimeoutInMs: 60000, - randomize: false, -} satisfies RetryOptions; - -export type ZodFetchOptions = { - retry?: RetryOptions; -}; - -export async function zodfetch( - schema: TResponseBodySchema, - url: string, - requestInit?: RequestInit, - options?: ZodFetchOptions -): Promise> { - return await _doZodFetch(schema, url, requestInit, options); -} - -export class MultipartBody { - constructor(public body: any) {} - get [Symbol.toStringTag](): string { - return "MultipartBody"; - } -} - -export async function zodupload< - TResponseBodySchema extends z.ZodTypeAny, - TBody = Record, ->( - schema: TResponseBodySchema, - url: string, - body: TBody, - requestInit?: RequestInit, - options?: ZodFetchOptions -): Promise> { - const form = await createForm(body); - const encoder = new FormDataEncoder(form); - - const finalHeaders: Record = {}; - - for (const [key, value] of Object.entries(requestInit?.headers || {})) { - finalHeaders[key] = value as string; - } - - for (const [key, value] of Object.entries(encoder.headers)) { - finalHeaders[key] = value; - } - - finalHeaders["Content-Length"] = String(encoder.contentLength); - - const finalRequestInit: RequestInit = { - ...requestInit, - headers: finalHeaders, - body: Readable.from(encoder) as any, - // @ts-expect-error - duplex: "half", - }; - - return await _doZodFetch(schema, url, finalRequestInit, options); -} - -export const createForm = async >( - body: T | undefined -): Promise => { - const form = new FormData(); - await Promise.all( - Object.entries(body || {}).map(([key, value]) => addFormValue(form, key, value)) - ); - return form; -}; - -async function _doZodFetch( - schema: TResponseBodySchema, - url: string, - requestInit?: RequestInit, - options?: ZodFetchOptions, - attempt = 1 -): Promise> { - try { - const response = await fetch(url, requestInitWithCache(requestInit)); - - const responseHeaders = createResponseHeaders(response.headers); - - if (!response.ok) { - const retryResult = shouldRetry(response, attempt, options?.retry); - - if (retryResult.retry) { - await new Promise((resolve) => setTimeout(resolve, retryResult.delay)); - - return await _doZodFetch(schema, url, requestInit, options, attempt + 1); - } else { - const errText = await response.text().catch((e) => castToError(e).message); - const errJSON = safeJsonParse(errText); - const errMessage = errJSON ? undefined : errText; - - throw APIError.generate(response.status, errJSON, errMessage, responseHeaders); - } - } - - const jsonBody = await response.json(); - const parsedResult = schema.safeParse(jsonBody); - - if (parsedResult.success) { - return parsedResult.data; - } - - throw fromZodError(parsedResult.error); - } catch (error) { - if (error instanceof APIError) { - throw error; - } - - if (options?.retry) { - const retry = { ...defaultRetryOptions, ...options.retry }; - - const delay = calculateNextRetryDelay(retry, attempt); - - if (delay) { - await new Promise((resolve) => setTimeout(resolve, delay)); - - return await _doZodFetch(schema, url, requestInit, options, attempt + 1); - } - } - - throw new APIConnectionError({ cause: castToError(error) }); - } -} - -function castToError(err: any): Error { - if (err instanceof Error) return err; - return new Error(err); -} - -type ShouldRetryResult = - | { - retry: false; - } - | { - retry: true; - delay: number; - }; - -function shouldRetry( - response: Response, - attempt: number, - retryOptions?: RetryOptions -): ShouldRetryResult { - function shouldRetryForOptions(): ShouldRetryResult { - const retry = { ...defaultRetryOptions, ...retryOptions }; - - const delay = calculateNextRetryDelay(retry, attempt); - - if (delay) { - return { retry: true, delay }; - } else { - return { retry: false }; - } - } - - // Note this is not a standard header. - const shouldRetryHeader = response.headers.get("x-should-retry"); - - // If the server explicitly says whether or not to retry, obey. - if (shouldRetryHeader === "true") return shouldRetryForOptions(); - if (shouldRetryHeader === "false") return { retry: false }; - - // Retry on request timeouts. - if (response.status === 408) return shouldRetryForOptions(); - - // Retry on lock timeouts. - if (response.status === 409) return shouldRetryForOptions(); - - // Retry on rate limits. - if (response.status === 429) return shouldRetryForOptions(); - - // Retry internal errors. - if (response.status >= 500) return shouldRetryForOptions(); - - return { retry: false }; -} - -function safeJsonParse(text: string): any { - try { - return JSON.parse(text); - } catch (e) { - return undefined; - } -} - -function createResponseHeaders(headers: Response["headers"]): Record { - return new Proxy( - Object.fromEntries( - // @ts-ignore - headers.entries() - ), - { - get(target, name) { - const key = name.toString(); - return target[key.toLowerCase()] || target[key]; - }, - } - ); -} - -function requestInitWithCache(requestInit?: RequestInit): RequestInit { - try { - const withCache: RequestInit = { - ...requestInit, - cache: "no-cache", - }; - - const _ = new Request("http://localhost", withCache); - - return withCache; - } catch (error) { - return requestInit ?? {}; - } -} - -const addFormValue = async (form: FormData, key: string, value: unknown): Promise => { - if (value === undefined) return; - if (value == null) { - throw new TypeError( - `Received null for "${key}"; to pass null in FormData, you must use the string 'null'` - ); - } - - // TODO: make nested formats configurable - if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { - form.append(key, String(value)); - } else if ( - isUploadable(value) || - isBlobLike(value) || - value instanceof Buffer || - value instanceof ArrayBuffer - ) { - const file = await toFile(value); - form.append(key, file as File); - } else if (Array.isArray(value)) { - await Promise.all(value.map((entry) => addFormValue(form, key + "[]", entry))); - } else if (typeof value === "object") { - await Promise.all( - Object.entries(value).map(([name, prop]) => addFormValue(form, `${key}[${name}]`, prop)) - ); - } else { - throw new TypeError( - `Invalid value given to form, expected a string, number, boolean, object, Array, File or Blob but got ${value} instead` - ); - } -}; - -export type ToFileInput = Uploadable | Exclude | AsyncIterable; - -/** - * Helper for creating a {@link File} to pass to an SDK upload method from a variety of different data formats - * @param value the raw content of the file. Can be an {@link Uploadable}, {@link BlobLikePart}, or {@link AsyncIterable} of {@link BlobLikePart}s - * @param {string=} name the name of the file. If omitted, toFile will try to determine a file name from bits if possible - * @param {Object=} options additional properties - * @param {string=} options.type the MIME type of the content - * @param {number=} options.lastModified the last modified timestamp - * @returns a {@link File} with the given properties - */ -export async function toFile( - value: ToFileInput | PromiseLike, - name?: string | null | undefined, - options?: FilePropertyBag | undefined -): Promise { - // If it's a promise, resolve it. - value = await value; - - // Use the file's options if there isn't one provided - options ??= isFileLike(value) ? { lastModified: value.lastModified, type: value.type } : {}; - - if (isResponseLike(value)) { - const blob = await value.blob(); - name ||= new URL(value.url).pathname.split(/[\\/]/).pop() ?? "unknown_file"; - - return new File([blob as any], name, options); - } - - const bits = await getBytes(value); - - name ||= getName(value) ?? "unknown_file"; - - if (!options.type) { - const type = (bits[0] as any)?.type; - if (typeof type === "string") { - options = { ...options, type }; - } - } - - return new File(bits, name, options); -} - -function getName(value: any): string | undefined { - return ( - getStringFromMaybeBuffer(value.name) || - getStringFromMaybeBuffer(value.filename) || - // For fs.ReadStream - getStringFromMaybeBuffer(value.path)?.split(/[\\/]/).pop() - ); -} - -const getStringFromMaybeBuffer = (x: string | Buffer | unknown): string | undefined => { - if (typeof x === "string") return x; - if (typeof Buffer !== "undefined" && x instanceof Buffer) return String(x); - return undefined; -}; - -async function getBytes(value: ToFileInput): Promise> { - let parts: Array = []; - if ( - typeof value === "string" || - ArrayBuffer.isView(value) || // includes Uint8Array, Buffer, etc. - value instanceof ArrayBuffer - ) { - parts.push(value); - } else if (isBlobLike(value)) { - parts.push(await value.arrayBuffer()); - } else if ( - isAsyncIterableIterator(value) // includes Readable, ReadableStream, etc. - ) { - for await (const chunk of value) { - parts.push(chunk as BlobPart); // TODO, consider validating? - } - } else { - throw new Error( - `Unexpected data type: ${typeof value}; constructor: ${value?.constructor - ?.name}; props: ${propsForError(value)}` - ); - } - - return parts; -} - -function propsForError(value: any): string { - const props = Object.getOwnPropertyNames(value); - return `[${props.map((p) => `"${p}"`).join(", ")}]`; -} - -const isAsyncIterableIterator = (value: any): value is AsyncIterableIterator => - value != null && typeof value === "object" && typeof value[Symbol.asyncIterator] === "function"; - -/** - * Intended to match web.Blob, node.Blob, node-fetch.Blob, etc. - */ -export interface BlobLike { - /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) */ - readonly size: number; - /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) */ - readonly type: string; - /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) */ - text(): Promise; - /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) */ - slice(start?: number, end?: number): BlobLike; - // unfortunately @types/node-fetch@^2.6.4 doesn't type the arrayBuffer method -} - -/** - * Intended to match web.File, node.File, node-fetch.File, etc. - */ -export interface FileLike extends BlobLike { - /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) */ - readonly lastModified: number; - /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) */ - readonly name: string; -} - -/** - * Intended to match web.Response, node.Response, node-fetch.Response, etc. - */ -export interface ResponseLike { - url: string; - blob(): Promise; -} - -export type Uploadable = FileLike | ResponseLike | Readable; - -export const isResponseLike = (value: any): value is ResponseLike => - value != null && - typeof value === "object" && - typeof value.url === "string" && - typeof value.blob === "function"; - -export const isFileLike = (value: any): value is FileLike => - value != null && - typeof value === "object" && - typeof value.name === "string" && - typeof value.lastModified === "number" && - isBlobLike(value); - -/** - * The BlobLike type omits arrayBuffer() because @types/node-fetch@^2.6.4 lacks it; but this check - * adds the arrayBuffer() method type because it is available and used at runtime - */ -export const isBlobLike = ( - value: any -): value is BlobLike & { arrayBuffer(): Promise } => - value != null && - typeof value === "object" && - typeof value.size === "number" && - typeof value.type === "string" && - typeof value.text === "function" && - typeof value.slice === "function" && - typeof value.arrayBuffer === "function"; - -export const isFsReadStream = (value: any): value is Readable => value instanceof Readable; - -export const isUploadable = (value: any): value is Uploadable => { - return isFileLike(value) || isResponseLike(value) || isFsReadStream(value); -}; - -export type BlobLikePart = - | string - | ArrayBuffer - | ArrayBufferView - | BlobLike - | Uint8Array - | DataView; - -export const isRecordLike = (value: any): value is Record => - value != null && - typeof value === "object" && - !Array.isArray(value) && - Object.keys(value).length > 0 && - Object.keys(value).every((key) => typeof key === "string" && typeof value[key] === "string"); +export * from "./apiClient/core"; +export * from "./apiClient/errors"; +export * from "./apiClient/pagination"; diff --git a/packages/trigger-sdk/src/v3/envvars.ts b/packages/trigger-sdk/src/v3/envvars.ts index cc00b12f2..03c0ea748 100644 --- a/packages/trigger-sdk/src/v3/envvars.ts +++ b/packages/trigger-sdk/src/v3/envvars.ts @@ -1,30 +1,30 @@ import type { - ImportEnvironmentVariablesParams, - EnvironmentVariableResponseBody, - EnvironmentVariables, + ApiPromise, CreateEnvironmentVariableParams, + EnvironmentVariableResponseBody, EnvironmentVariableValue, + EnvironmentVariables, + ImportEnvironmentVariablesParams, UpdateEnvironmentVariableParams, } from "@trigger.dev/core/v3"; -import { SemanticInternalAttributes, apiClientManager, taskContext } from "@trigger.dev/core/v3"; +import { apiClientManager, taskContext } from "@trigger.dev/core/v3"; import { apiClientMissingError } from "./shared"; -import { tracer } from "./tracer"; -export type { ImportEnvironmentVariablesParams, CreateEnvironmentVariableParams }; +export type { CreateEnvironmentVariableParams, ImportEnvironmentVariablesParams }; -export async function upload( +export function upload( projectRef: string, slug: string, params: ImportEnvironmentVariablesParams -): Promise; -export async function upload( +): ApiPromise; +export function upload( params: ImportEnvironmentVariablesParams -): Promise; -export async function upload( +): ApiPromise; +export function upload( projectRefOrParams: string | ImportEnvironmentVariablesParams, slug?: string, params?: ImportEnvironmentVariablesParams -): Promise { +): ApiPromise { let $projectRef: string; let $params: ImportEnvironmentVariablesParams; let $slug: string; @@ -68,22 +68,12 @@ export async function upload( throw apiClientMissingError(); } - return await tracer.startActiveSpan( - "envvars.upload", - async (span) => { - return await apiClient.importEnvVars($projectRef, $slug, $params); - }, - { - attributes: { - [SemanticInternalAttributes.STYLE_ICON]: "file-upload", - }, - } - ); + return apiClient.importEnvVars($projectRef, $slug, $params); } -export async function list(projectRef: string, slug: string): Promise; -export async function list(): Promise; -export async function list(projectRef?: string, slug?: string): Promise { +export function list(projectRef: string, slug: string): ApiPromise; +export function list(): ApiPromise; +export function list(projectRef?: string, slug?: string): ApiPromise { const $projectRef = projectRef ?? taskContext.ctx?.project.ref; const $slug = slug ?? taskContext.ctx?.environment.slug; @@ -101,32 +91,22 @@ export async function list(projectRef?: string, slug?: string): Promise { - return await apiClient.listEnvVars($projectRef, $slug); - }, - { - attributes: { - [SemanticInternalAttributes.STYLE_ICON]: "id", - }, - } - ); + return apiClient.listEnvVars($projectRef, $slug); } -export async function create( +export function create( projectRef: string, slug: string, params: CreateEnvironmentVariableParams -): Promise; -export async function create( +): ApiPromise; +export function create( params: CreateEnvironmentVariableParams -): Promise; -export async function create( +): ApiPromise; +export function create( projectRefOrParams: string | CreateEnvironmentVariableParams, slug?: string, params?: CreateEnvironmentVariableParams -): Promise { +): ApiPromise { let $projectRef: string; let $slug: string; let $params: CreateEnvironmentVariableParams; @@ -170,30 +150,20 @@ export async function create( throw apiClientMissingError(); } - return await tracer.startActiveSpan( - "envvars.create", - async (span) => { - return await apiClient.createEnvVar($projectRef, $slug, $params); - }, - { - attributes: { - [SemanticInternalAttributes.STYLE_ICON]: "id", - }, - } - ); + return apiClient.createEnvVar($projectRef, $slug, $params); } -export async function retrieve( +export function retrieve( projectRef: string, slug: string, name: string -): Promise; -export async function retrieve(name: string): Promise; -export async function retrieve( +): ApiPromise; +export function retrieve(name: string): ApiPromise; +export function retrieve( projectRefOrName: string, slug?: string, name?: string -): Promise { +): ApiPromise { let $projectRef: string; let $slug: string; let $name: string; @@ -222,30 +192,20 @@ export async function retrieve( throw apiClientMissingError(); } - return await tracer.startActiveSpan( - "envvars.retrieve", - async (span) => { - return await apiClient.retrieveEnvVar($projectRef, $slug, $name); - }, - { - attributes: { - [SemanticInternalAttributes.STYLE_ICON]: "id", - }, - } - ); + return apiClient.retrieveEnvVar($projectRef, $slug, $name); } -export async function del( +export function del( projectRef: string, slug: string, name: string -): Promise; -export async function del(name: string): Promise; -export async function del( +): ApiPromise; +export function del(name: string): ApiPromise; +export function del( projectRefOrName: string, slug?: string, name?: string -): Promise { +): ApiPromise { let $projectRef: string; let $slug: string; let $name: string; @@ -274,35 +234,25 @@ export async function del( throw apiClientMissingError(); } - return await tracer.startActiveSpan( - "envvars.delete", - async (span) => { - return await apiClient.deleteEnvVar($projectRef, $slug, $name); - }, - { - attributes: { - [SemanticInternalAttributes.STYLE_ICON]: "id", - }, - } - ); + return apiClient.deleteEnvVar($projectRef, $slug, $name); } -export async function update( +export function update( projectRef: string, slug: string, name: string, params: UpdateEnvironmentVariableParams -): Promise; -export async function update( +): ApiPromise; +export function update( name: string, params: UpdateEnvironmentVariableParams -): Promise; -export async function update( +): ApiPromise; +export function update( projectRefOrName: string, slugOrParams: string | UpdateEnvironmentVariableParams, name?: string, params?: UpdateEnvironmentVariableParams -): Promise { +): ApiPromise { let $projectRef: string; let $slug: string; let $name: string; @@ -350,15 +300,5 @@ export async function update( throw apiClientMissingError(); } - return await tracer.startActiveSpan( - "envvars.update", - async (span) => { - return await apiClient.updateEnvVar($projectRef, $slug, $name, $params); - }, - { - attributes: { - [SemanticInternalAttributes.STYLE_ICON]: "id", - }, - } - ); + return apiClient.updateEnvVar($projectRef, $slug, $name, $params); } diff --git a/packages/trigger-sdk/src/v3/index.ts b/packages/trigger-sdk/src/v3/index.ts index 760a9cd63..bb164e0c0 100644 --- a/packages/trigger-sdk/src/v3/index.ts +++ b/packages/trigger-sdk/src/v3/index.ts @@ -14,7 +14,7 @@ import { apiClientManager } from "@trigger.dev/core/v3"; export type { ApiClientConfiguration }; export { - APIError, + ApiError, AuthenticationError, BadRequestError, ConflictError, diff --git a/packages/trigger-sdk/src/v3/runs.ts b/packages/trigger-sdk/src/v3/runs.ts index c3eab96af..38b9ede58 100644 --- a/packages/trigger-sdk/src/v3/runs.ts +++ b/packages/trigger-sdk/src/v3/runs.ts @@ -1,43 +1,74 @@ import { + ApiPromise, CanceledRunResponse, + ListRunResponseItem, ReplayRunResponse, RetrieveRunResponse, apiClientManager, } from "@trigger.dev/core/v3"; +import type { ListProjectRunsQueryParams, ListRunsQueryParams } from "@trigger.dev/core/v3"; import { apiClientMissingError } from "./shared"; +import { CursorPagePromise } from "@trigger.dev/core/v3/apiClient/core"; + +export type RetrieveRunResult = RetrieveRunResponse; export const runs = { replay: replayRun, cancel: cancelRun, retrieve: retrieveRun, + list: listRuns, }; -async function retrieveRun(runId: string): Promise { +export type ListRunsItem = ListRunResponseItem; + +function listRuns( + projectRef: string, + params?: ListProjectRunsQueryParams +): CursorPagePromise; +function listRuns(params?: ListRunsQueryParams): CursorPagePromise; +function listRuns( + paramsOrProjectRef?: ListRunsQueryParams | string, + params?: ListRunsQueryParams | ListProjectRunsQueryParams +): CursorPagePromise { const apiClient = apiClientManager.client; if (!apiClient) { throw apiClientMissingError(); } - return await apiClient.retrieveRun(runId); + if (typeof paramsOrProjectRef === "string") { + return apiClient.listProjectRuns(paramsOrProjectRef, params); + } + + return apiClient.listRuns(params); } -async function replayRun(runId: string): Promise { +function retrieveRun(runId: string): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { throw apiClientMissingError(); } - return await apiClient.replayRun(runId); + return apiClient.retrieveRun(runId); } -async function cancelRun(runId: string): Promise { +function replayRun(runId: string): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { throw apiClientMissingError(); } - return await apiClient.cancelRun(runId); + return apiClient.replayRun(runId); +} + +function cancelRun(runId: string): ApiPromise { + const apiClient = apiClientManager.client; + + if (!apiClient) { + throw apiClientMissingError(); + } + + return apiClient.cancelRun(runId); } diff --git a/packages/trigger-sdk/src/v3/schedules/index.ts b/packages/trigger-sdk/src/v3/schedules/index.ts index 01d423595..0acd41940 100644 --- a/packages/trigger-sdk/src/v3/schedules/index.ts +++ b/packages/trigger-sdk/src/v3/schedules/index.ts @@ -1,13 +1,14 @@ import { + ApiPromise, DeletedScheduleObject, InitOutput, - ListSchedulesResult, ScheduleObject, apiClientManager, taskCatalog, } from "@trigger.dev/core/v3"; import { Task, TaskOptions, apiClientMissingError, createTask } from "../shared"; import * as SchedulesAPI from "./api"; +import { OffsetLimitPagePromise } from "@trigger.dev/core/v3/apiClient/core"; export function task( params: TaskOptions @@ -30,7 +31,7 @@ export function task( * @param options.deduplicationKey - An optional deduplication key for the schedule * @returns The created schedule */ -export async function create(options: SchedulesAPI.CreateScheduleOptions): Promise { +export function create(options: SchedulesAPI.CreateScheduleOptions): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { @@ -45,7 +46,7 @@ export async function create(options: SchedulesAPI.CreateScheduleOptions): Promi * @param scheduleId - The ID of the schedule to retrieve * @returns The retrieved schedule */ -export async function retrieve(scheduleId: string): Promise { +export function retrieve(scheduleId: string): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { @@ -64,10 +65,10 @@ export async function retrieve(scheduleId: string): Promise { * @param options.externalId - An optional external identifier for the schedule * @returns The updated schedule */ -export async function update( +export function update( scheduleId: string, options: SchedulesAPI.UpdateScheduleOptions -): Promise { +): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { @@ -81,7 +82,7 @@ export async function update( * Deletes a schedule * @param scheduleId - The ID of the schedule to delete */ -export async function del(scheduleId: string): Promise { +export function del(scheduleId: string): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { @@ -95,7 +96,7 @@ export async function del(scheduleId: string): Promise { * Deactivates a schedule * @param scheduleId - The ID of the schedule to deactivate */ -export async function deactivate(scheduleId: string): Promise { +export function deactivate(scheduleId: string): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { @@ -109,7 +110,7 @@ export async function deactivate(scheduleId: string): Promise { * Activates a schedule * @param scheduleId - The ID of the schedule to activate */ -export async function activate(scheduleId: string): Promise { +export function activate(scheduleId: string): ApiPromise { const apiClient = apiClientManager.client; if (!apiClient) { @@ -126,9 +127,9 @@ export async function activate(scheduleId: string): Promise { * @param options.perPage - The number of schedules per page * @returns The list of schedules */ -export async function list( +export function list( options?: SchedulesAPI.ListScheduleOptions -): Promise { +): OffsetLimitPagePromise { const apiClient = apiClientManager.client; if (!apiClient) { diff --git a/references/v3-catalog/src/management.ts b/references/v3-catalog/src/management.ts index 90ee05291..f5146b967 100644 --- a/references/v3-catalog/src/management.ts +++ b/references/v3-catalog/src/management.ts @@ -1,13 +1,13 @@ -import { tracer } from "./tracer"; -import { APIError, configure, runs, schedules, envvars } from "@trigger.dev/sdk/v3"; -import { simpleChildTask } from "./trigger/subtasks"; +import { configure, envvars, runs, schedules, ApiError } from "@trigger.dev/sdk/v3"; import dotenv from "dotenv"; -import { firstScheduledTask } from "./trigger/scheduled"; import { createReadStream } from "node:fs"; +import { firstScheduledTask } from "./trigger/scheduled"; +import { simpleChildTask } from "./trigger/subtasks"; +import { taskThatErrors } from "./trigger/retries"; dotenv.config(); -async function uploadEnvVars() { +async function doEnvVars() { configure({ secretKey: process.env.TRIGGER_ACCESS_TOKEN, }); @@ -88,87 +88,181 @@ async function uploadEnvVars() { console.log("response6", response6); } -export async function run() { - await tracer.startActiveSpan("run", async (span) => { - try { - const run = await simpleChildTask.trigger({ message: "Hello, World!" }); +async function doRuns() { + const run = await simpleChildTask.trigger({ message: "Hello, World!" }); - const retrievedRun = await runs.retrieve(run.id); - console.log("retrieved run", retrievedRun); + const retrievedRun = await runs.retrieve(run.id); + console.log("retrieved run", retrievedRun); - const canceled = await runs.cancel(run.id); - console.log("canceled run", canceled); + const completedRun = await waitForRunToComplete(run.id); + console.log("completed run", completedRun); - const replayed = await runs.replay(run.id); - console.log("replayed run", replayed); + const failingRun = await taskThatErrors.trigger({ message: "Hello, World!" }); + const failedRun = await waitForRunToComplete(failingRun.id); - const run2 = await simpleChildTask.trigger( - { message: "Hello, World!" }, - { - idempotencyKey: "mmvlgwcidiklyeygen4", - } - ); + console.log("failed run", failedRun); - const run3 = await simpleChildTask.trigger( - { message: "Hello, World again!" }, - { - idempotencyKey: "mmvlgwcidiklyeygen4", - } - ); + const replayableRun = await runs.replay(failedRun.id); + const replayedRun = await waitForRunToExecute(replayableRun.id); - console.log("run2", run2); - console.log("run3", run3); + console.log("replayed run", replayedRun); - const allSchedules = await schedules.list(); + const canceledRun = await runs.cancel(replayedRun.id); + const canceledRunResult = await waitForRunToComplete(canceledRun.id); - console.log("all schedules", allSchedules); - - // Create a schedule - const createdSchedule = await schedules.create({ - task: firstScheduledTask.id, - cron: "0 0 * * *", - externalId: "ext_1234444", - deduplicationKey: "dedup_1234444", - }); - - console.log("created schedule", createdSchedule); - - const retrievedSchedule = await schedules.retrieve(createdSchedule.id); - - console.log("retrieved schedule", retrievedSchedule); - - const updatedSchedule = await schedules.update(createdSchedule.id, { - task: firstScheduledTask.id, - cron: "0 0 1 * *", - externalId: "ext_1234444", - }); - - console.log("updated schedule", updatedSchedule); - - const deactivatedSchedule = await schedules.deactivate(createdSchedule.id); - - console.log("deactivated schedule", deactivatedSchedule); - - const activatedSchedule = await schedules.activate(createdSchedule.id); - - console.log("activated schedule", activatedSchedule); - - const deletedSchedule = await schedules.del(createdSchedule.id); - - console.log("deleted schedule", deletedSchedule); - } catch (error) { - span.recordException(error as Error); - - if (error instanceof APIError) { - console.error("APIError", error); - } else { - console.error("Unknown error", error); - } - } finally { - span.end(); - } - }); + console.log("canceled run", canceledRunResult); } -// run(); -uploadEnvVars().catch(console.error); +async function doListRuns() { + let pageCount = 0; + + let page = await runs.list({ + limit: 100, + }); + + console.log(`run page #${++pageCount}`); + + // Convenience methods are provided for manually paginating: + while (page.hasNextPage()) { + page = await page.getNextPage(); + console.log(`run page #${++pageCount}`); + } + + while (page.hasPreviousPage()) { + page = await page.getPreviousPage(); + console.log(`run page #${--pageCount}`); + } + + for await (const run of runs.list({ + status: ["COMPLETED"], + period: "1y", + })) { + console.log(run); + } + + let withResponse = await runs + .list({ + limit: 100, + }) + .withResponse(); + + console.log( + "withResponse", + withResponse.response.status, + withResponse.response.headers, + withResponse.data.data.length + ); + + configure({ + secretKey: process.env.TRIGGER_ACCESS_TOKEN, + }); + + for await (const run of runs.list("yubjwjsfkxnylobaqvqz", { + status: ["COMPLETED"], + period: "1y", + env: ["dev", "staging", "prod"], + })) { + console.log(run.env.name, run.isTest, run.id, run.status, run.createdAt); + } +} + +async function waitForRunToComplete(runId: string) { + let run = await runs.retrieve(runId); + + while (!run.isCompleted) { + console.log("run is not completed, waiting...", run); + await new Promise((resolve) => setTimeout(resolve, 2000)); + run = await runs.retrieve(runId); + } + + console.log("run is completed", run); + + return run; +} + +async function waitForRunToExecute(runId: string) { + let run = await runs.retrieve(runId); + + while (!run.isExecuting) { + console.log("run is not executing, waiting...", run); + await new Promise((resolve) => setTimeout(resolve, 2000)); + run = await runs.retrieve(runId); + } + + console.log("run is executing", run); + + return run; +} + +async function doSchedules() { + const allSchedules = await schedules.list(); + + console.log("all schedules", allSchedules); + + // Create a schedule + const createdSchedule = await schedules.create({ + task: firstScheduledTask.id, + cron: "0 0 * * *", + externalId: "ext_1234444", + deduplicationKey: "dedup_1234444", + }); + + console.log("created schedule", createdSchedule); + + const retrievedSchedule = await schedules.retrieve(createdSchedule.id); + + console.log("retrieved schedule", retrievedSchedule); + + const updatedSchedule = await schedules.update(createdSchedule.id, { + task: firstScheduledTask.id, + cron: "0 0 1 * *", + externalId: "ext_1234444", + }); + + console.log("updated schedule", updatedSchedule); + + const deactivatedSchedule = await schedules.deactivate(createdSchedule.id); + + console.log("deactivated schedule", deactivatedSchedule); + + const activatedSchedule = await schedules.activate(createdSchedule.id); + + console.log("activated schedule", activatedSchedule); + + const deletedSchedule = await schedules.del(createdSchedule.id); + + console.log("deleted schedule", deletedSchedule); +} + +async function doScheduleLists() { + let pageCount = 0; + + let page = await schedules.list({ + perPage: 2, + }); + + console.log(`schedule page #${++pageCount}`); + + // Convenience methods are provided for manually paginating: + while (page.hasNextPage()) { + page = await page.getNextPage(); + console.log(`schedule page #${++pageCount}`); + } + + while (page.hasPreviousPage()) { + page = await page.getPreviousPage(); + console.log(`schedule page #${--pageCount}`); + } + + for await (const schedule of schedules.list({ + perPage: 2, + })) { + console.log(schedule.id, schedule.task, schedule.generator); + } +} + +// doRuns().catch(console.error); +doListRuns().catch(console.error); +// doScheduleLists().catch(console.error); +// doSchedules().catch(console.error); +// doEnvVars().catch(console.error);