From c385db63be9f6fc76e4b21262103151e4c9097fa Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 8 Mar 2024 10:49:42 +0000 Subject: [PATCH] v3: cli deploy command (#931) * WIP proxy/deploy * WIP, registry proxy in express working * A couple of notes, preparing for indexing * Move the changes to prod-worker into the new file * Deploy command working with indexing and runs (docker provider only for now) * Removed ts-expect-error directive * Fixed build command --- .env.example | 10 +- .vscode/launch.json | 8 + apps/coordinator/.env.example | 1 - apps/coordinator/src/index.ts | 4 +- apps/coordinator/tsconfig.json | 2 + apps/docker-provider/.env.example | 5 +- apps/docker-provider/src/index.ts | 20 +- apps/webapp/app/entry.server.tsx | 1 + apps/webapp/app/env.server.ts | 5 + ...ployments.$deploymentId.start-indexing.ts} | 18 +- .../api.v1.deployments.$deploymentId.ts | 68 +++ apps/webapp/app/routes/api.v1.deployments.ts | 45 ++ .../routes/api.v1.projects.$projectRef.dev.ts | 2 + .../api.v1.projects.$projectRef.envvars.ts | 5 +- .../api.v1.projects.$projectRef.prod.ts | 2 + apps/webapp/app/services/worker.server.ts | 26 +- apps/webapp/app/v3/handleSocketIo.server.ts | 9 +- .../v3/marqs/sharedQueueConsumer.server.ts | 86 ++- apps/webapp/app/v3/registryProxy.server.ts | 496 +++++++++++++++ .../app/v3/remoteImageBuilder.server.ts | 23 + .../services/createBackgroundWorker.server.ts | 221 +++---- .../createDeployedBackgroundWorker.server.ts | 58 ++ .../v3/services/createImageDetails.server.ts | 75 --- .../app/v3/services/indexDeployment.server.ts | 53 ++ .../app/v3/services/indexTasks.server.ts | 53 -- .../services/initializeDeployment.server.ts | 50 ++ .../startDeploymentIndexing.server.ts | 29 + .../app/v3/utils/calculateNextBuildVersion.ts | 23 + apps/webapp/package.json | 2 + apps/webapp/server.ts | 62 +- packages/cli-v3/package.json | 10 +- packages/cli-v3/src/Containerfile.prod | 31 +- packages/cli-v3/src/apiClient.ts | 77 ++- packages/cli-v3/src/cli/index.ts | 3 +- packages/cli-v3/src/commands/build.ts | 15 +- packages/cli-v3/src/commands/deploy.ts | 571 ++++++++++++++++++ packages/cli-v3/src/commands/dev.tsx | 20 +- packages/cli-v3/src/utilities/fileSystem.ts | 30 +- .../src/{dev => utilities}/ink-table.tsx | 0 .../cli-v3/src/utilities/installPackages.ts | 17 + packages/cli-v3/src/utilities/session.ts | 4 + .../common}/register-tracing.ts | 0 .../src/{ => workers}/dev/backgroundWorker.ts | 6 +- .../src/{ => workers}/dev/worker-facade.ts | 5 +- .../{ => workers}/prod/backgroundWorker.ts | 5 +- .../prod/entry-point.ts} | 49 +- .../prod/worker-facade.ts} | 29 +- packages/cli-v3/tsup.prod-facade.config.ts | 16 - ...acade.config.ts => tsup.workers.config.ts} | 9 +- packages/core-apps/src/provider.ts | 2 + packages/core/src/v3/schemas/api.ts | 63 +- packages/core/src/v3/schemas/resources.ts | 2 +- packages/core/src/v3/schemas/schemas.ts | 8 +- .../migration.sql | 70 +++ .../migration.sql | 14 + .../migration.sql | 8 + .../migration.sql | 20 + packages/database/prisma/schema.prisma | 147 +++-- pnpm-lock.yaml | 156 ++++- references/v3-catalog/tsconfig.json | 4 +- 60 files changed, 2311 insertions(+), 542 deletions(-) rename apps/webapp/app/routes/{api.v1.projects.$projectRef.image-details.ts => api.v1.deployments.$deploymentId.start-indexing.ts} (69%) create mode 100644 apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts create mode 100644 apps/webapp/app/routes/api.v1.deployments.ts create mode 100644 apps/webapp/app/v3/registryProxy.server.ts create mode 100644 apps/webapp/app/v3/remoteImageBuilder.server.ts create mode 100644 apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts delete mode 100644 apps/webapp/app/v3/services/createImageDetails.server.ts create mode 100644 apps/webapp/app/v3/services/indexDeployment.server.ts delete mode 100644 apps/webapp/app/v3/services/indexTasks.server.ts create mode 100644 apps/webapp/app/v3/services/initializeDeployment.server.ts create mode 100644 apps/webapp/app/v3/services/startDeploymentIndexing.server.ts create mode 100644 apps/webapp/app/v3/utils/calculateNextBuildVersion.ts create mode 100644 packages/cli-v3/src/commands/deploy.ts rename packages/cli-v3/src/{dev => utilities}/ink-table.tsx (100%) rename packages/cli-v3/src/{dev => workers/common}/register-tracing.ts (100%) rename packages/cli-v3/src/{ => workers}/dev/backgroundWorker.ts (98%) rename packages/cli-v3/src/{ => workers}/dev/worker-facade.ts (98%) rename packages/cli-v3/src/{ => workers}/prod/backgroundWorker.ts (98%) rename packages/cli-v3/src/{prod-worker.ts => workers/prod/entry-point.ts} (87%) rename packages/cli-v3/src/{prod-facade.ts => workers/prod/worker-facade.ts} (90%) delete mode 100644 packages/cli-v3/tsup.prod-facade.config.ts rename packages/cli-v3/{tsup.facade.config.ts => tsup.workers.config.ts} (54%) create mode 100644 packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql create mode 100644 packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql create mode 100644 packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql create mode 100644 packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql diff --git a/.env.example b/.env.example index 1ce00e294..3de1889bf 100644 --- a/.env.example +++ b/.env.example @@ -49,4 +49,12 @@ CLOUD_SLACK_CLIENT_SECRET= # v3 variables PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32` -COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32` \ No newline at end of file +COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32` + +# Uncomment the following line to enable the registry proxy +# ENABLE_REGISTRY_PROXY=true +# DEPOT_TOKEN= +# DEPOT_PROJECT_ID= +# CONTAINER_REGISTRY_ORIGIN= +# CONTAINER_REGISTRY_USERNAME= +# CONTAINER_REGISTRY_PASSWORD= \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index cceb8db29..25375c084 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -37,6 +37,14 @@ "cwd": "${workspaceFolder}/references/v3-catalog", "sourceMaps": true }, + { + "type": "node-terminal", + "request": "launch", + "name": "Debug V3 Deploy CLI", + "command": "pnpm exec trigger.dev deploy", + "cwd": "${workspaceFolder}/references/v3-catalog", + "sourceMaps": true + }, { "type": "node", "request": "attach", diff --git a/apps/coordinator/.env.example b/apps/coordinator/.env.example index 2a416edcf..472939071 100644 --- a/apps/coordinator/.env.example +++ b/apps/coordinator/.env.example @@ -1,4 +1,3 @@ HTTP_SERVER_PORT=8020 - PLATFORM_ENABLED=true PLATFORM_WS_PORT=3030 \ No newline at end of file diff --git a/apps/coordinator/src/index.ts b/apps/coordinator/src/index.ts index 266664fc4..9b10cf0b8 100644 --- a/apps/coordinator/src/index.ts +++ b/apps/coordinator/src/index.ts @@ -286,10 +286,10 @@ class TaskCoordinator { try { setSocketDataFromHeader("podName", "x-pod-name"); setSocketDataFromHeader("contentHash", "x-trigger-content-hash"); - setSocketDataFromHeader("cliPackageVersion", "x-trigger-cli-package-version"); setSocketDataFromHeader("projectRef", "x-trigger-project-ref"); setSocketDataFromHeader("attemptId", "x-trigger-attempt-id"); setSocketDataFromHeader("envId", "x-trigger-env-id"); + setSocketDataFromHeader("deploymentId", "x-trigger-deployment-id"); } catch (error) { logger(error); socket.disconnect(true); @@ -390,8 +390,8 @@ class TaskCoordinator { version: "v1", projectRef: socket.data.projectRef, envId: socket.data.envId, + deploymentId: message.deploymentId, metadata: { - cliPackageVersion: socket.data.cliPackageVersion, contentHash: socket.data.contentHash, packageVersion: message.packageVersion, tasks: message.tasks, diff --git a/apps/coordinator/tsconfig.json b/apps/coordinator/tsconfig.json index a491b740b..4c3965549 100644 --- a/apps/coordinator/tsconfig.json +++ b/apps/coordinator/tsconfig.json @@ -1,4 +1,6 @@ { + "include": ["./src/**/*.ts"], + "exclude": ["node_modules"], "compilerOptions": { "target": "es2016", "module": "commonjs", diff --git a/apps/docker-provider/.env.example b/apps/docker-provider/.env.example index 6083fd511..cdf3cf9cc 100644 --- a/apps/docker-provider/.env.example +++ b/apps/docker-provider/.env.example @@ -1,4 +1,7 @@ HTTP_SERVER_PORT=8050 PLATFORM_WS_PORT=3030 -PLATFORM_SECRET=provider-secret \ No newline at end of file +PLATFORM_SECRET=provider-secret +# Use this if you are on macOS +# COORDINATOR_HOST="host.docker.internal" +# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts index 4e1bfecf8..50095a5ba 100644 --- a/apps/docker-provider/src/index.ts +++ b/apps/docker-provider/src/index.ts @@ -4,15 +4,29 @@ import { SimpleLogger, TaskOperations, ProviderShell } from "@trigger.dev/core-a const MACHINE_NAME = process.env.MACHINE_NAME || "local"; const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020; +const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1"; +const OTEL_EXPORTER_OTLP_ENDPOINT = + process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318"; const logger = new SimpleLogger(`[${MACHINE_NAME}]`); class DockerTaskOperations implements TaskOperations { - async index(opts: { contentHash: string; imageTag: string; envId: string }) { + async index(opts: { + contentHash: string; + imageTag: string; + envId: string; + apiKey: string; + apiUrl: string; + }) { const containerName = this.#getIndexContainerName(opts.contentHash); + logger.log(`Indexing task ${opts.imageTag}`, { + host: COORDINATOR_HOST, + port: COORDINATOR_PORT, + }); + const { exitCode } = logger.debug( - await $`docker run --rm -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e INDEX_TASKS=true --network=host --pull=never --name=${containerName} ${opts.imageTag}` + await $`docker run --rm -e TRIGGER_SECRET_KEY=${opts.apiKey} -e TRIGGER_API_URL=${opts.apiUrl} -e COORDINATOR_HOST=${COORDINATOR_HOST} -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e INDEX_TASKS=true --name=${containerName} ${opts.imageTag}` ); if (exitCode !== 0) { @@ -24,7 +38,7 @@ class DockerTaskOperations implements TaskOperations { const containerName = this.#getRunContainerName(opts.attemptId); const { exitCode } = logger.debug( - await $`docker run -d -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e TRIGGER_ATTEMPT_ID=${opts.attemptId} --network=host --pull=never --name=${containerName} ${opts.image}` + await $`docker run -d -e OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} -e COORDINATOR_HOST=${COORDINATOR_HOST} -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e TRIGGER_ATTEMPT_ID=${opts.attemptId} --name=${containerName} ${opts.image}` ); if (exitCode !== 0) { diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 8930fcef6..d2966f9f3 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -197,3 +197,4 @@ const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer export { wss } from "./v3/handleWebsockets.server"; export { socketIo } from "./v3/handleSocketIo.server"; +export { registryProxy } from "./v3/registryProxy.server"; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index e7051e6cf..43936a232 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -76,6 +76,11 @@ const EnvironmentSchema = z.object({ IMAGE_REPO: z.string().default("task"), PROVIDER_SECRET: z.string().default("provider-secret"), COORDINATOR_SECRET: z.string().default("coordinator-secret"), + DEPOT_TOKEN: z.string().optional(), + DEPOT_PROJECT_ID: z.string().optional(), + CONTAINER_REGISTRY_ORIGIN: z.string().optional(), + CONTAINER_REGISTRY_USERNAME: z.string().optional(), + CONTAINER_REGISTRY_PASSWORD: z.string().optional(), }); export type Environment = z.infer; diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.start-indexing.ts similarity index 69% rename from apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts rename to apps/webapp/app/routes/api.v1.deployments.$deploymentId.start-indexing.ts index 6cbcc3ffa..f71d99c8a 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.start-indexing.ts @@ -1,12 +1,12 @@ import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; -import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3"; +import { StartDeploymentIndexingRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; -import { CreateImageDetailsService } from "~/v3/services/createImageDetails.server"; +import { StartDeploymentIndexing } from "~/v3/services/startDeploymentIndexing.server"; const ParamsSchema = z.object({ - projectRef: z.string(), + deploymentId: z.string(), }); export async function action({ request, params }: ActionFunctionArgs) { @@ -31,23 +31,23 @@ export async function action({ request, params }: ActionFunctionArgs) { const authenticatedEnv = authenticationResult.environment; - const { projectRef } = parsedParams.data; + const { deploymentId } = parsedParams.data; const rawBody = await request.json(); - const body = CreateImageDetailsRequestBody.safeParse(rawBody); + const body = StartDeploymentIndexingRequestBody.safeParse(rawBody); if (!body.success) { return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 }); } - const service = new CreateImageDetailsService(); + const service = new StartDeploymentIndexing(); - const imageDetails = await service.call(projectRef, authenticatedEnv, body.data); + const deployment = await service.call(authenticatedEnv, deploymentId, body.data); return json( { - id: imageDetails.friendlyId, - contentHash: imageDetails.contentHash, + id: deployment.friendlyId, + contentHash: deployment.contentHash, }, { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts new file mode 100644 index 000000000..825e20a69 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts @@ -0,0 +1,68 @@ +import { LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; + +const ParamsSchema = z.object({ + deploymentId: z.string(), +}); + +export async function loader({ request, params }: LoaderFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + // Next authenticate the request + const authenticationResult = await authenticateApiRequest(request); + + if (!authenticationResult) { + logger.info("Invalid or missing api key", { url: request.url }); + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const authenticatedEnv = authenticationResult.environment; + + const { deploymentId } = parsedParams.data; + + const deployment = await prisma.workerDeployment.findUnique({ + where: { + friendlyId: deploymentId, + environmentId: authenticatedEnv.id, + }, + include: { + worker: { + include: { + tasks: true, + }, + }, + }, + }); + + if (!deployment) { + return json({ error: "Deployment not found" }, { status: 404 }); + } + + return json({ + id: deployment.friendlyId, + status: deployment.status, + contentHash: deployment.contentHash, + shortCode: deployment.shortCode, + version: deployment.version, + imageReference: deployment.imageReference, + worker: deployment.worker + ? { + id: deployment.worker.friendlyId, + version: deployment.worker.version, + tasks: deployment.worker.tasks.map((task) => ({ + id: task.friendlyId, + slug: task.slug, + filePath: task.filePath, + exportName: task.exportName, + })), + } + : undefined, + }); +} diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts new file mode 100644 index 000000000..cf420119a --- /dev/null +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -0,0 +1,45 @@ +import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { InitializeDeploymentRequestBody } from "@trigger.dev/core/v3"; +import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { InitializeDeploymentService } from "~/v3/services/initializeDeployment.server"; + +export async function action({ request, params }: ActionFunctionArgs) { + // Ensure this is a POST request + if (request.method.toUpperCase() !== "POST") { + return { status: 405, body: "Method Not Allowed" }; + } + + // Next authenticate the request + const authenticationResult = await authenticateApiRequest(request); + + if (!authenticationResult) { + logger.info("Invalid or missing api key", { url: request.url }); + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const rawBody = await request.json(); + const body = InitializeDeploymentRequestBody.safeParse(rawBody); + + if (!body.success) { + return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 }); + } + + const authenticatedEnv = authenticationResult.environment; + + const service = new InitializeDeploymentService(); + + const { deployment, imageTag } = await service.call(authenticatedEnv, body.data); + + return json( + { + id: deployment.friendlyId, + contentHash: deployment.contentHash, + shortCode: deployment.shortCode, + version: deployment.version, + externalBuildData: deployment.externalBuildData, + imageTag, + }, + { status: 200 } + ); +} diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.dev.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.dev.ts index 8b25e2142..b72c41007 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.dev.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.dev.ts @@ -3,6 +3,7 @@ import { json } from "@remix-run/server-runtime"; import { GetProjectDevResponse } from "@trigger.dev/core/v3"; import { z } from "zod"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; @@ -58,6 +59,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const result: GetProjectDevResponse = { apiKey: devEnvironment.apiKey, name: project.name, + apiUrl: env.APP_ORIGIN, }; return json(result); diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts index 6c9fd78dd..536405046 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts @@ -1,11 +1,8 @@ -import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime"; -import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; +import { LoaderFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; -import { logger } from "~/services/logger.server"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; -import { CreateBackgroundWorkerService } from "~/v3/services/createBackgroundWorker.server"; const ParamsSchema = z.object({ projectRef: z.string(), diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts index 4bab1ac39..dfac13f08 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts @@ -3,6 +3,7 @@ import { json } from "@remix-run/server-runtime"; import { GetProjectDevResponse } from "@trigger.dev/core/v3"; import { z } from "zod"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; @@ -56,6 +57,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const result: GetProjectDevResponse = { apiKey: prodEnvironment.apiKey, name: project.name, + apiUrl: env.APP_ORIGIN, }; return json(result); diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts index cf0fe76ef..e3a269160 100644 --- a/apps/webapp/app/services/worker.server.ts +++ b/apps/webapp/app/services/worker.server.ts @@ -4,32 +4,32 @@ import { z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { ZodWorker } from "~/platform/zodWorker.server"; +import { IndexDeploymentService } from "~/v3/services/indexDeployment.server"; +import { ExpireDispatcherService } from "./dispatchers/expireDispatcher.server"; +import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralEventDispatcher.server"; import { sendEmail } from "./email.server"; import { IndexEndpointService } from "./endpoints/indexEndpoint.server"; import { PerformEndpointIndexService } from "./endpoints/performEndpointIndexService"; +import { ProbeEndpointService } from "./endpoints/probeEndpoint.server"; import { RecurringEndpointIndexService } from "./endpoints/recurringEndpointIndex.server"; import { DeliverEventService } from "./events/deliverEvent.server"; import { InvokeDispatcherService } from "./events/invokeDispatcher.server"; import { integrationAuthRepository } from "./externalApis/integrationAuthRepository.server"; import { IntegrationConnectionCreatedService } from "./externalApis/integrationConnectionCreated.server"; +import { executionRateLimiter } from "./runExecutionRateLimiter.server"; +import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server"; +import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server"; import { MissingConnectionCreatedService } from "./runs/missingConnectionCreated.server"; import { PerformRunExecutionV3Service } from "./runs/performRunExecutionV3.server"; +import { ResumeRunService } from "./runs/resumeRun.server"; import { StartRunService } from "./runs/startRun.server"; import { DeliverScheduledEventService } from "./schedules/deliverScheduledEvent.server"; import { ActivateSourceService } from "./sources/activateSource.server"; import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server"; +import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server"; import { PerformTaskOperationService } from "./tasks/performTaskOperation.server"; import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout.server"; -import { ProbeEndpointService } from "./endpoints/probeEndpoint.server"; -import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server"; -import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server"; import { ResumeTaskService } from "./tasks/resumeTask.server"; -import { ExpireDispatcherService } from "./dispatchers/expireDispatcher.server"; -import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralEventDispatcher.server"; -import { ResumeRunService } from "./runs/resumeRun.server"; -import { executionRateLimiter } from "./runExecutionRateLimiter.server"; -import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server"; -import { IndexTasksService } from "~/v3/services/indexTasks.server"; const workerCatalog = { indexEndpoint: z.object({ @@ -104,7 +104,7 @@ const workerCatalog = { id: z.string(), }), // v3 tasks - indexTasks: z.object({ + "v3.indexDeployment": z.object({ id: z.string(), }), }; @@ -434,11 +434,11 @@ function getWorkerQueue() { }, }, // v3 tasks - indexTasks: { + "v3.indexDeployment": { priority: 0, maxAttempts: 5, handler: async (payload, job) => { - const service = new IndexTasksService(); + const service = new IndexDeploymentService(); return await service.call(payload.id); }, @@ -538,4 +538,4 @@ function getTaskOperationWorkerQueue() { }); } -export { executionWorker, workerQueue, taskOperationWorker }; +export { executionWorker, taskOperationWorker, workerQueue }; diff --git a/apps/webapp/app/v3/handleSocketIo.server.ts b/apps/webapp/app/v3/handleSocketIo.server.ts index ccd867889..5a3c1a6b9 100644 --- a/apps/webapp/app/v3/handleSocketIo.server.ts +++ b/apps/webapp/app/v3/handleSocketIo.server.ts @@ -17,6 +17,7 @@ import { CompleteAttemptService } from "./services/completeAttempt.server"; import { CreateBackgroundWorkerService } from "./services/createBackgroundWorker.server"; import { logger } from "~/services/logger.server"; import { findEnvironmentById } from "~/models/runtimeEnvironment.server"; +import { CreateDeployedBackgroundWorkerService } from "./services/createDeployedBackgroundWorker.server"; export const socketIo = singleton("socketIo", initalizeIoServer); @@ -76,13 +77,13 @@ function createCoordinatorNamespace(io: Server) { return { success: false }; } - const createCheckpoint = new CreateBackgroundWorkerService(); - await createCheckpoint.call(message.projectRef, environment, { - localOnly: true, + const service = new CreateDeployedBackgroundWorkerService(); + const worker = await service.call(message.projectRef, environment, message.deploymentId, { + localOnly: false, metadata: message.metadata, }); - return { success: true }; + return { success: !!worker }; } catch (error) { logger.error("Error while creating worker", { error }); return { success: false }; diff --git a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts index 4cc6afd9c..50d26a1f6 100644 --- a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts +++ b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts @@ -291,25 +291,29 @@ export class SharedQueueConsumer { return; } - const backgroundWorker = await prisma.backgroundWorker.findFirst({ + const deployment = await prisma.workerDeployment.findFirst({ where: { - runtimeEnvironmentId: existingTaskRun.runtimeEnvironmentId, + environmentId: existingTaskRun.runtimeEnvironmentId, projectId: existingTaskRun.projectId, - imageDetails: { - some: {}, + status: "DEPLOYED", + imageReference: { + not: null, }, }, orderBy: { updatedAt: "desc", }, include: { - tasks: true, - imageDetails: true, + worker: { + include: { + tasks: true, + }, + }, }, }); - if (!backgroundWorker) { - logger.error("No matching background worker found for task run", { + if (!deployment || !deployment.worker) { + logger.error("No matching deployment found for task run", { queueMessage: message.data, messageId: message.messageId, }); @@ -318,7 +322,18 @@ export class SharedQueueConsumer { return; } - const backgroundTask = backgroundWorker.tasks.find( + if (!deployment.imageReference) { + logger.error("Deployment is missing an image reference", { + queueMessage: message.data, + messageId: message.messageId, + deployment: deployment.id, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const backgroundTask = deployment.worker.tasks.find( (task) => task.slug === existingTaskRun.taskIdentifier ); @@ -326,8 +341,9 @@ export class SharedQueueConsumer { logger.warn("No matching background task found for task run", { taskRun: existingTaskRun.id, taskIdentifier: existingTaskRun.taskIdentifier, - backgroundWorker: backgroundWorker.id, - taskSlugs: backgroundWorker.tasks.map((task) => task.slug), + deployment: deployment.id, + backgroundWorker: deployment.worker.id, + taskSlugs: deployment.worker.tasks.map((task) => task.slug), }); await marqs?.acknowledgeMessage(message.messageId); @@ -357,7 +373,8 @@ export class SharedQueueConsumer { logger.warn("Failed to lock task run", { taskRun: existingTaskRun.id, taskIdentifier: existingTaskRun.taskIdentifier, - backgroundWorker: backgroundWorker.id, + deployment: deployment.id, + backgroundWorker: deployment.worker.id, messageId: message.messageId, }); @@ -402,11 +419,11 @@ export class SharedQueueConsumer { try { await this._sender.send("BACKGROUND_WORKER_MESSAGE", { - backgroundWorkerId: backgroundWorker.friendlyId, + backgroundWorkerId: deployment.worker.friendlyId, data: { type: "SCHEDULE_ATTEMPT", id: taskRunAttempt.id, - image: backgroundWorker.imageDetails[0].tag, + image: deployment.imageReference, envId: environment.id, }, }); @@ -492,30 +509,29 @@ export class SharedQueueConsumer { return; } - const backgroundWorker = await prisma.backgroundWorker.findFirst({ + const deployment = await prisma.workerDeployment.findFirst({ where: { - runtimeEnvironmentId: resumableRun.runtimeEnvironmentId, + environmentId: resumableRun.runtimeEnvironmentId, projectId: resumableRun.projectId, - imageDetails: { - some: {}, + status: "DEPLOYED", + imageReference: { + not: null, }, }, orderBy: { createdAt: "desc", }, include: { - tasks: true, - imageDetails: { - take: 1, - orderBy: { - updatedAt: "desc", + worker: { + include: { + tasks: true, }, }, }, }); - if (!backgroundWorker) { - logger.error("No matching background worker found for task run", { + if (!deployment || !deployment.worker) { + logger.error("No matching deployment found for task run", { queueMessage: message.data, messageId: message.messageId, }); @@ -524,7 +540,18 @@ export class SharedQueueConsumer { return; } - const backgroundTask = backgroundWorker.tasks.find( + if (!deployment.imageReference) { + logger.error("Deployment is missing an image reference", { + queueMessage: message.data, + messageId: message.messageId, + deployment: deployment.id, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const backgroundTask = deployment.worker.tasks.find( (task) => task.slug === resumableRun.taskIdentifier ); @@ -532,8 +559,9 @@ export class SharedQueueConsumer { logger.warn("No matching background task found for task run", { taskRun: resumableRun.id, taskIdentifier: resumableRun.taskIdentifier, - backgroundWorker: backgroundWorker.id, - taskSlugs: backgroundWorker.tasks.map((task) => task.slug), + deployment: deployment.id, + backgroundWorker: deployment.worker.id, + taskSlugs: deployment.worker.tasks.map((task) => task.slug), }); await marqs?.acknowledgeMessage(message.messageId); @@ -623,7 +651,7 @@ export class SharedQueueConsumer { socketIo.coordinatorNamespace.emit("RESUME", { version: "v1", attemptId: resumableAttempt.id, - image: backgroundWorker.imageDetails[0].tag, + image: deployment.imageReference, completions, executions, }); diff --git a/apps/webapp/app/v3/registryProxy.server.ts b/apps/webapp/app/v3/registryProxy.server.ts new file mode 100644 index 000000000..f0c05797d --- /dev/null +++ b/apps/webapp/app/v3/registryProxy.server.ts @@ -0,0 +1,496 @@ +import { IncomingMessage, ServerResponse } from "http"; +import Redis, { RedisOptions } from "ioredis"; +import { RequestOptions, request as httpRequest } from "node:https"; +import { z } from "zod"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { authenticatePersonalAccessToken } from "~/services/personalAccessToken.server"; +import { singleton } from "~/utils/singleton"; +import { jwtDecode } from "jwt-decode"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { mkdtemp } from "fs/promises"; +import { createReadStream, createWriteStream } from "node:fs"; +import { pipeline } from "node:stream/promises"; +import { unlinkSync } from "fs"; + +const TokenResponseBody = z.object({ + token: z.string(), +}); + +const CACHED_BEARER_TOKEN_BUFFER_IN_SECONDS = 10; + +type RegistryProxyOptions = { + origin: string; + auth: { username: string; password: string }; + redis?: RedisOptions; +}; + +export class RegistryProxy { + private redis?: Redis; + + constructor(private readonly options: RegistryProxyOptions) { + if (options.redis) { + this.redis = new Redis(options.redis); + } + } + + get origin() { + return this.options.origin; + } + + get host() { + return new URL(this.options.origin).host; + } + + // If the imageReference includes a hostname, rewrite it to point to the proxy + // e.g. eric-webapp.trigger.dev/trigger/yubjwjsfkxnylobaqvqz:20240306.41.prod@sha256:8b48dd2866bc8878644d2880bbe35a27e66cf6ff78aa1e489d7fdde5e228faf1 + // should be rewritten to ${this.host}/trigger/yubjwjsfkxnylobaqvqz:20240306.41.prod@sha256:8b48dd2866bc8878644d2880bbe35a27e66cf6ff78aa1e489d7fdde5e228faf1 + // This will work with image references that don't include the @sha256:... part + public rewriteImageReference(imageReference: string) { + const parts = parseDockerImageReference(imageReference); + + if (parts.registry) { + return rebuildDockerImageReference({ + ...parts, + registry: this.host, + }); + } + + return imageReference; + } + + public async call(request: IncomingMessage, response: ServerResponse) { + await this.#proxyRequest(request, response); + } + + // Proxies the request to the registry + async #proxyRequest(request: IncomingMessage, response: ServerResponse) { + const credentials = this.#getBasicAuthCredentials(request); + + if (!credentials) { + logger.debug("Returning 401 because credentials are missing"); + + response.writeHead(401, { + "WWW-Authenticate": 'Basic realm="Access to the registry"', + }); + + response.end("Unauthorized"); + + return; + } + + // Authenticate the request + const authentication = await authenticatePersonalAccessToken(credentials.password); + + if (!authentication) { + logger.debug("Returning 401 because authentication failed"); + + response.writeHead(401, { + "WWW-Authenticate": 'Basic realm="Access to the registry"', + }); + + response.end("Unauthorized"); + + return; + } + + // construct a new url based on the url passed in and the registry url + const url = new URL(this.options.origin); + const options = { + hostname: url.hostname, + path: request.url, + method: request.method, + headers: { ...request.headers }, + }; + + delete options.headers["host"]; + // delete options.headers["connection"]; + // delete options.headers["accept-encoding"]; + delete options.headers["authorization"]; + // delete options.headers["content-length"]; + delete options.headers["cf-ray"]; + delete options.headers["cf-visitor"]; + delete options.headers["cf-ipcountry"]; + delete options.headers["cf-connecting-ip"]; + delete options.headers["cf-warp-tag-id"]; + + // Add a custom Authorization header for the proxied request + options.headers["authorization"] = `Basic ${Buffer.from( + `${this.options.auth.username}:${this.options.auth.password}` + ).toString("base64")}`; + + let tempFilePath: string | undefined; + let cleanupTempFile: () => void = () => {}; + + if ( + options.method === "POST" || + (options.method === "PUT" && request.headers["content-length"]) + ) { + tempFilePath = await streamRequestBodyToTempFile(request); + + cleanupTempFile = () => { + if (tempFilePath) { + logger.debug("Cleaning up temp file", { tempFilePath }); + unlinkSync(tempFilePath); + } + }; + + logger.debug("Streamed request body to temp file", { tempFilePath }); + } + + const makeProxiedRequest = (tokenOptions: RequestOptions, attempts: number = 1) => { + logger.debug("Proxying request", { + request: tokenOptions, + attempts, + originalHeaders: request.headers, + }); + + if (attempts > 10) { + logger.error("Too many attempts to proxy request", { + attempts, + }); + + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end("Internal Server Error: Too many attempts to proxy request"); + + return cleanupTempFile(); + } + + const proxyReq = httpRequest(tokenOptions, async (proxyRes) => { + // If challenged for bearer token auth, handle it here + if (proxyRes.statusCode === 401 && proxyRes.headers["www-authenticate"]) { + logger.debug("Received 401 with WWW-Authenticate, attempting to fetch bearer token", { + authenticate: proxyRes.headers["www-authenticate"], + }); + + const bearerToken = await this.#getBearerToken(proxyRes.headers["www-authenticate"]); + + if (bearerToken && tokenOptions.headers) { + tokenOptions.headers["authorization"] = `Bearer ${bearerToken}`; + makeProxiedRequest(tokenOptions, attempts + 1); // Retry request with bearer token + return; + } else { + // Handle failed token fetch or lack of WWW-Authenticate handling + response.writeHead(401, { "Content-Type": "text/plain" }); + response.end("Failed to authenticate with the registry using bearer token"); + return cleanupTempFile(); + } + } + + if (proxyRes.statusCode === 401) { + logger.debug("Received 401, but there is no www-authenticate value", { + headers: proxyRes.headers, + }); + + response.writeHead(401, { "Content-Type": "text/plain" }); + response.end("Unauthorized"); + return cleanupTempFile(); + } + + if (proxyRes.statusCode === 301) { + logger.debug("Received 301, attempting to follow redirect", { + location: proxyRes.headers["location"], + }); + + const redirectOptions = { + ...tokenOptions, + path: proxyRes.headers["location"], + }; + + makeProxiedRequest(redirectOptions, attempts + 1); + return; + } + + if (!proxyRes.statusCode) { + logger.error("No status code in the response", { + headers: proxyRes.headers, + statusMessage: proxyRes.statusMessage, + }); + + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end("Internal Server Error: No status code in the response"); + + return cleanupTempFile(); + } + + const headers = { ...proxyRes.headers }; + + // Rewrite location headers to point to the proxy + if (headers["location"]) { + const proxiedLocation = new URL(headers.location); + + // Only rewrite the location header if the host is the same as the registry + if (proxiedLocation.host === this.host) { + if (!request.headers.host) { + // Return a 500 if the host header is missing + logger.error("Host header is missing in the request", { + headers: request.headers, + }); + + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end("Internal Server Error: Host header is missing in the request"); + return cleanupTempFile(); + } + + proxiedLocation.host = request.headers.host; + + headers["location"] = proxiedLocation.href; + + logger.debug("Rewriting location response header", { + originalLocation: proxyRes.headers["location"], + proxiedLocation: headers["location"], + proxiedLocationUrl: proxiedLocation.href, + }); + } + } + + logger.debug("Proxying successful response", { + method: tokenOptions.method, + path: tokenOptions.path, + statusCode: proxyRes.statusCode, + responseHeaders: headers, + }); + + // Proceed as normal if not a 401 or after getting a bearer token + response.writeHead(proxyRes.statusCode, headers); + proxyRes.pipe(response, { end: true }); + }); + + if (tempFilePath) { + const readStream = createReadStream(tempFilePath); + + readStream.pipe(proxyReq, { end: true }); + } else { + proxyReq.end(); + } + + proxyReq.on("error", (error) => { + logger.error("Error proxying request", { error: error.message }); + response.writeHead(500, { "Content-Type": "text/plain" }); + response.end(`Internal Server Error: ${error.message}`); + }); + }; + + makeProxiedRequest(options); + } + + #getBasicAuthCredentials(request: IncomingMessage) { + const headers = request.headers; + + logger.debug("Getting basic auth credentials with headers", { + headers, + }); + + const authHeader = headers["authorization"]; + + if (!authHeader) { + return; + } + + const [type, credentials] = authHeader.split(" "); + + if (type.toLowerCase() !== "basic") { + return; + } + + const decoded = Buffer.from(credentials, "base64").toString("utf-8"); + const [username, password] = decoded.split(":"); + + return { username, password }; + } + + async #getBearerToken(authenticateHeader: string): Promise { + try { + // Create a md5 hash of the authenticate header to use as a cache key + const cacheKey = `token:${createHash("md5").update(authenticateHeader).digest("hex")}`; + + const cachedToken = await this.#getCachedToken(cacheKey); + + if (cachedToken) { + return cachedToken; + } + + // Parse the WWW-Authenticate header to extract realm and service + const realmMatch = authenticateHeader.match(/realm="([^"]+)"/); + const serviceMatch = authenticateHeader.match(/service="([^"]+)"/); + // Optionally, we could also extract and use the scope parameter if required + const scopeMatch = authenticateHeader.match(/scope="([^"]+)"/); + + if (!realmMatch || !serviceMatch) { + logger.error("Failed to parse WWW-Authenticate header", { authenticateHeader }); + return; + } + + const realm = realmMatch[1]; + const service = serviceMatch[1]; + // Construct the URL for fetching the token + let authUrl = `${realm}?service=${encodeURIComponent(service)}`; + // Include scope in the request if needed + if (scopeMatch) { + const scope = scopeMatch[1]; + authUrl += `&scope=${encodeURIComponent(scope)}`; + } + + authUrl += `&account=${encodeURIComponent(this.options.auth.username)}`; + + logger.debug("Fetching bearer token", { authUrl }); + + // Make the request to the authentication service + const response = await fetch(authUrl, { + headers: { + authorization: + "Basic " + + Buffer.from(`${this.options.auth.username}:${this.options.auth.password}`).toString( + "base64" + ), + }, + }); + + if (!response.ok) { + logger.debug("Failed to fetch bearer token", { + status: response.status, + statusText: response.statusText, + }); + return; + } + + const rawBody = await response.json(); + const body = TokenResponseBody.safeParse(rawBody); + + if (!body.success) { + logger.error("Failed to parse token response", { body: rawBody }); + return; + } + + logger.debug("Fetched bearer token", { token: body.data.token }); + + await this.#setCachedToken(body.data.token, cacheKey); + + return body.data.token; + } catch (error) { + logger.error("Failed to fetch bearer token", { + error: error instanceof Error ? error.message : error, + }); + } + } + + async #getCachedToken(key: string) { + if (!this.redis) { + return; + } + + const cachedToken = await this.redis.get(key); + + if (cachedToken) { + const decoded = jwtDecode(cachedToken); + const expiry = decoded.exp; + + if (expiry && expiry > Date.now() / 1000 + CACHED_BEARER_TOKEN_BUFFER_IN_SECONDS) { + return cachedToken; + } + } + } + + async #setCachedToken(token: string, key: string) { + if (!this.redis) { + return; + } + + const decoded = jwtDecode(token); + + if (decoded.exp) { + await this.redis.set(key, token, "EXAT", decoded.exp); + } + } +} + +export const registryProxy = singleton("registryProxy", initializeProxy); + +function initializeProxy() { + if ( + !env.CONTAINER_REGISTRY_ORIGIN || + !env.CONTAINER_REGISTRY_USERNAME || + !env.CONTAINER_REGISTRY_PASSWORD + ) { + return; + } + + return new RegistryProxy({ + origin: env.CONTAINER_REGISTRY_ORIGIN, + auth: { + username: env.CONTAINER_REGISTRY_USERNAME, + password: env.CONTAINER_REGISTRY_PASSWORD, + }, + }); +} + +async function streamRequestBodyToTempFile(request: IncomingMessage): Promise { + const tempDir = await mkdtemp(`${tmpdir()}/`); + const tempFilePath = `${tempDir}/requestBody.tmp`; + const writeStream = createWriteStream(tempFilePath); + + await pipeline(request, writeStream); + + return tempFilePath; +} + +type DockerImageParts = { + registry?: string; + repo: string; + tag?: string; + digest?: string; +}; + +function parseDockerImageReference(imageReference: string): DockerImageParts { + const parts: DockerImageParts = { repo: "" }; // Initialize with an empty repo which we'll fill later + + // Splitting by '@' to separate the digest (if exists) + const atSplit = imageReference.split("@"); + if (atSplit.length > 1) { + parts.digest = atSplit[1]; + imageReference = atSplit[0]; + } + + // Splitting by ':' to separate the tag (if exists) + const colonSplit = imageReference.split(":"); + if (colonSplit.length > 1 && !colonSplit[1].includes("/")) { + // Ensuring we don't split a registry port + parts.tag = colonSplit.pop(); // The last part is the tag, remove it from the array + imageReference = colonSplit.join(":"); // Join back in case there was more than one colon + } + + // Now, the remaining part is "registry/repo" or just "repo" + const slashIndex = imageReference.indexOf("/"); + if ( + slashIndex !== -1 && + (imageReference.startsWith("localhost") || slashIndex > imageReference.indexOf(".")) + ) { + parts.registry = imageReference.substring(0, slashIndex); + parts.repo = imageReference.substring(slashIndex + 1); + } else { + parts.repo = imageReference; // If there's no registry, the whole thing is the repo + } + + return parts; +} + +function rebuildDockerImageReference(parts: DockerImageParts): string { + let imageReference = ""; + + if (parts.registry) { + imageReference += `${parts.registry}/`; + } + + imageReference += parts.repo; // Repo is now guaranteed to be defined + + if (parts.tag) { + imageReference += `:${parts.tag}`; + } + + if (parts.digest) { + imageReference += `@${parts.digest}`; + } + + return imageReference; +} diff --git a/apps/webapp/app/v3/remoteImageBuilder.server.ts b/apps/webapp/app/v3/remoteImageBuilder.server.ts new file mode 100644 index 000000000..8f7992c29 --- /dev/null +++ b/apps/webapp/app/v3/remoteImageBuilder.server.ts @@ -0,0 +1,23 @@ +import { depot } from "@depot/sdk-node"; +import { env } from "~/env.server"; + +export async function createRemoteImageBuild() { + if (!env.DEPOT_TOKEN || !env.DEPOT_PROJECT_ID) { + return; + } + + const result = await depot.build.v1.BuildService.createBuild( + { projectId: env.DEPOT_PROJECT_ID }, + { + headers: { + Authorization: `Bearer ${env.DEPOT_TOKEN}`, + }, + } + ); + + return { + projectId: env.DEPOT_PROJECT_ID, + buildToken: result.buildToken, + buildId: result.buildId, + }; +} diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 3b3d911b7..289c744a7 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -1,11 +1,12 @@ -import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; +import { CreateBackgroundWorkerRequestBody, TaskResource } from "@trigger.dev/core/v3"; import type { BackgroundWorker } from "@trigger.dev/database"; +import { PrismaClientOrTransaction } from "~/db.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { marqs } from "../marqs.server"; +import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion"; import { BaseService } from "./baseService.server"; -import { $transaction } from "~/db.server"; export class CreateBackgroundWorkerService extends BaseService { public async call( @@ -16,150 +17,108 @@ export class CreateBackgroundWorkerService extends BaseService { return this.traceWithEnv("call", environment, async (span) => { span.setAttribute("projectRef", projectRef); - const backgroundWorker = await $transaction(this._prisma, async (tx) => { - const project = await this._prisma.project.findUniqueOrThrow({ - where: { - externalRef: projectRef, - environments: { - some: { - id: environment.id, - }, + const project = await this._prisma.project.findUniqueOrThrow({ + where: { + externalRef: projectRef, + environments: { + some: { + id: environment.id, }, }, - include: { - backgroundWorkers: { - where: { - runtimeEnvironmentId: environment.id, - }, - orderBy: { - createdAt: "desc", - }, - take: 1, - }, - }, - }); - - const latestBackgroundWorker = project.backgroundWorkers[0]; - - if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) { - return latestBackgroundWorker; - } - - const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version); - - logger.debug(`Creating background worker`, { - nextVersion, - lastVersion: project.backgroundWorkers[0]?.version, - }); - - const backgroundWorker = await this._prisma.backgroundWorker.create({ - data: { - friendlyId: generateFriendlyId("worker"), - version: nextVersion, - runtimeEnvironmentId: environment.id, - projectId: project.id, - metadata: body.metadata, - contentHash: body.metadata.contentHash, - cliVersion: body.metadata.cliPackageVersion, - sdkVersion: body.metadata.packageVersion, - }, - }); - - if (environment.type !== "DEVELOPMENT") { - await this._prisma.imageDetails.update({ + }, + include: { + backgroundWorkers: { where: { - projectId_runtimeEnvironmentId_contentHash: { - projectId: environment.projectId, - runtimeEnvironmentId: environment.id, - contentHash: backgroundWorker.contentHash, - }, - }, - data: { - backgroundWorkerId: backgroundWorker.id, - }, - }); - } - - for (const task of body.metadata.tasks) { - await this._prisma.backgroundWorkerTask.create({ - data: { - friendlyId: generateFriendlyId("task"), - projectId: project.id, runtimeEnvironmentId: environment.id, - workerId: backgroundWorker.id, - slug: task.id, - filePath: task.filePath, - exportName: task.exportName, - retryConfig: task.retry, - queueConfig: task.queue, }, - }); - - const queueName = task.queue?.name ?? `task/${task.id}`; - - const taskQueue = await this._prisma.taskQueue.upsert({ - where: { - runtimeEnvironmentId_name: { - runtimeEnvironmentId: environment.id, - name: queueName, - }, + orderBy: { + createdAt: "desc", }, - update: { - concurrencyLimit: task.queue?.concurrencyLimit, - rateLimit: task.queue?.rateLimit, - }, - create: { - friendlyId: generateFriendlyId("queue"), - name: queueName, - concurrencyLimit: task.queue?.concurrencyLimit, - runtimeEnvironmentId: environment.id, - projectId: project.id, - rateLimit: task.queue?.rateLimit, - type: task.queue?.name ? "NAMED" : "VIRTUAL", - }, - }); - - if (taskQueue.concurrencyLimit) { - await marqs?.updateQueueConcurrency( - environment, - taskQueue.name, - taskQueue.concurrencyLimit - ); - } - } - - return backgroundWorker; + take: 1, + }, + }, }); - if (!backgroundWorker) { - throw new Error("Failed to create background worker"); + const latestBackgroundWorker = project.backgroundWorkers[0]; + + if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) { + return latestBackgroundWorker; } + const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version); + + logger.debug(`Creating background worker`, { + nextVersion, + lastVersion: project.backgroundWorkers[0]?.version, + }); + + const backgroundWorker = await this._prisma.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + version: nextVersion, + runtimeEnvironmentId: environment.id, + projectId: project.id, + metadata: body.metadata, + contentHash: body.metadata.contentHash, + cliVersion: body.metadata.cliPackageVersion, + sdkVersion: body.metadata.packageVersion, + }, + }); + + await createBackgroundTasks(body.metadata.tasks, backgroundWorker, environment, this._prisma); + return backgroundWorker; }); } } -// Calculate next build version based on the previous version -// Version formats are YYYYMMDD.1, YYYYMMDD.2, etc. -// If there is no previous version, start at Todays date and .1 -function calculateNextBuildVersion(latestVersion?: string | null): string { - const today = new Date(); - const year = today.getFullYear(); - const month = today.getMonth() + 1; - const day = today.getDate(); - const todayFormatted = `${year}${month < 10 ? "0" : ""}${month}${day < 10 ? "0" : ""}${day}`; +export async function createBackgroundTasks( + tasks: TaskResource[], + worker: BackgroundWorker, + env: AuthenticatedEnvironment, + prisma: PrismaClientOrTransaction +) { + for (const task of tasks) { + await prisma.backgroundWorkerTask.create({ + data: { + friendlyId: generateFriendlyId("task"), + projectId: worker.projectId, + runtimeEnvironmentId: worker.runtimeEnvironmentId, + workerId: worker.id, + slug: task.id, + filePath: task.filePath, + exportName: task.exportName, + retryConfig: task.retry, + queueConfig: task.queue, + }, + }); - if (!latestVersion) { - return `${todayFormatted}.1`; + const queueName = task.queue?.name ?? `task/${task.id}`; + + const taskQueue = await prisma.taskQueue.upsert({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: worker.runtimeEnvironmentId, + name: queueName, + }, + }, + update: { + concurrencyLimit: task.queue?.concurrencyLimit, + rateLimit: task.queue?.rateLimit, + }, + create: { + friendlyId: generateFriendlyId("queue"), + name: queueName, + concurrencyLimit: task.queue?.concurrencyLimit, + runtimeEnvironmentId: worker.runtimeEnvironmentId, + projectId: worker.projectId, + rateLimit: task.queue?.rateLimit, + type: task.queue?.name ? "NAMED" : "VIRTUAL", + }, + }); + + if (taskQueue.concurrencyLimit) { + await marqs?.updateQueueConcurrency(env, taskQueue.name, taskQueue.concurrencyLimit); + } } - - const [date, buildNumber] = latestVersion.split("."); - - if (date === todayFormatted) { - const nextBuildNumber = parseInt(buildNumber, 10) + 1; - return `${date}.${nextBuildNumber}`; - } - - return `${todayFormatted}.1`; } diff --git a/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts new file mode 100644 index 000000000..43bd38929 --- /dev/null +++ b/apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts @@ -0,0 +1,58 @@ +import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; +import type { BackgroundWorker } from "@trigger.dev/database"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { BaseService } from "./baseService.server"; +import { createBackgroundTasks } from "./createBackgroundWorker.server"; + +export class CreateDeployedBackgroundWorkerService extends BaseService { + public async call( + projectRef: string, + environment: AuthenticatedEnvironment, + deploymentId: string, + body: CreateBackgroundWorkerRequestBody + ): Promise { + return this.traceWithEnv("call", environment, async (span) => { + span.setAttribute("projectRef", projectRef); + + const deployment = await this._prisma.workerDeployment.findUnique({ + where: { + friendlyId: deploymentId, + }, + }); + + if (!deployment) { + return; + } + + const backgroundWorker = await this._prisma.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + version: deployment.version, + runtimeEnvironmentId: environment.id, + projectId: environment.projectId, + metadata: body.metadata, + contentHash: body.metadata.contentHash, + cliVersion: body.metadata.cliPackageVersion, + sdkVersion: body.metadata.packageVersion, + }, + }); + + await createBackgroundTasks(body.metadata.tasks, backgroundWorker, environment, this._prisma); + + // Link the deployment with the background worker + await this._prisma.workerDeployment.update({ + where: { + id: deployment.id, + }, + data: { + workerId: backgroundWorker.id, + status: "DEPLOYED", + deployedAt: new Date(), + }, + }); + + return backgroundWorker; + }); + } +} diff --git a/apps/webapp/app/v3/services/createImageDetails.server.ts b/apps/webapp/app/v3/services/createImageDetails.server.ts deleted file mode 100644 index ba55b8f37..000000000 --- a/apps/webapp/app/v3/services/createImageDetails.server.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3"; -import type { ImageDetails } from "@trigger.dev/database"; -import { PrismaClient, prisma } from "~/db.server"; -import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; -import { logger } from "~/services/logger.server"; -import { generateFriendlyId } from "../friendlyIdentifiers"; -import { env } from "~/env.server"; -import { workerQueue } from "~/services/worker.server"; - -function escapeStringForRegex(rawString: string) { - return rawString.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&").replace(/-/g, "\\x2d"); -} - -export class CreateImageDetailsService { - #prismaClient: PrismaClient; - - constructor(prismaClient: PrismaClient = prisma) { - this.#prismaClient = prismaClient; - } - - public async call( - projectRef: string, - environment: AuthenticatedEnvironment, - body: CreateImageDetailsRequestBody - ): Promise { - const allowedTagPrefix = escapeStringForRegex(`${env.IMAGE_REGISTRY}/${env.IMAGE_REPO}:`); - - if (!body.metadata.imageTag.match(`^${allowedTagPrefix}`)) { - if (env.NODE_ENV !== "development") { - throw new Error("Forbidden image tag"); - } - } - - const project = await this.#prismaClient.project.findUniqueOrThrow({ - where: { - externalRef: projectRef, - environments: { - some: { - id: environment.id, - }, - }, - }, - }); - - logger.debug(`Creating image details`, { - imageTag: body.metadata.imageTag, - }); - - const imageDetails = await this.#prismaClient.imageDetails.upsert({ - where: { - projectId_runtimeEnvironmentId_contentHash: { - contentHash: body.metadata.contentHash, - runtimeEnvironmentId: environment.id, - projectId: project.id, - }, - }, - create: { - contentHash: body.metadata.contentHash, - friendlyId: generateFriendlyId("image"), - tag: body.metadata.imageTag, - runtimeEnvironmentId: environment.id, - projectId: project.id, - metadata: body.metadata, - }, - update: { - tag: body.metadata.imageTag, - metadata: body.metadata, - }, - }); - - await workerQueue.enqueue("indexTasks", { id: imageDetails.id }); - - return imageDetails; - } -} diff --git a/apps/webapp/app/v3/services/indexDeployment.server.ts b/apps/webapp/app/v3/services/indexDeployment.server.ts new file mode 100644 index 000000000..43ad3cf09 --- /dev/null +++ b/apps/webapp/app/v3/services/indexDeployment.server.ts @@ -0,0 +1,53 @@ +import { logger } from "~/services/logger.server"; +import { socketIo } from "../handleSocketIo.server"; +import { BaseService } from "./baseService.server"; +import { env } from "~/env.server"; + +export class IndexDeploymentService extends BaseService { + public async call(id: string) { + const deployment = await this._prisma.workerDeployment.findUnique({ + where: { + id, + }, + include: { + environment: true, + }, + }); + + if (!deployment) { + logger.error(`No worker deployment with this ID: ${id}`); + return; + } + + if (!deployment.imageReference) { + logger.error(`No image reference for worker deployment: ${id}`); + return; + } + + if (deployment.workerId) { + logger.debug( + `Deployment have already been indexed for ${deployment.friendlyId}. Refreshing worker timestamp.` + ); + + await this._prisma.backgroundWorker.update({ + where: { + id: deployment.workerId, + }, + data: { + updatedAt: new Date(), + }, + }); + return; + } + + // just broadcast for now - there should only ever be one provider connected + socketIo.providerNamespace.emit("INDEX", { + version: "v1", + contentHash: deployment.contentHash, + imageTag: deployment.imageReference, + envId: deployment.environmentId, + apiKey: deployment.environment.apiKey, + apiUrl: env.APP_ORIGIN, + }); + } +} diff --git a/apps/webapp/app/v3/services/indexTasks.server.ts b/apps/webapp/app/v3/services/indexTasks.server.ts deleted file mode 100644 index a0446cb5b..000000000 --- a/apps/webapp/app/v3/services/indexTasks.server.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { PrismaClient, prisma } from "~/db.server"; -import { socketIo } from "../handleSocketIo.server"; -import { logger } from "~/services/logger.server"; - -export type IndexTasksServiceOptions = { - idempotencyKey?: string; - triggerVersion?: string; - traceContext?: Record; -}; - -export class IndexTasksService { - #prismaClient: PrismaClient; - - constructor(prismaClient: PrismaClient = prisma) { - this.#prismaClient = prismaClient; - } - - public async call(imageDetailsId: string) { - const imageDetails = await this.#prismaClient.imageDetails.findUnique({ - where: { - id: imageDetailsId, - }, - }); - - if (!imageDetails) { - logger.error(`No image details with this ID: ${imageDetailsId}`); - return; - } - - if (imageDetails.backgroundWorkerId) { - logger.debug( - `Image details have already been indexed for ${imageDetails.friendlyId}. Refreshing worker timestamp.` - ); - await this.#prismaClient.backgroundWorker.update({ - where: { - id: imageDetails.backgroundWorkerId, - }, - data: { - updatedAt: new Date(), - }, - }); - return; - } - - // just broadcast for now - there should only ever be one provider connected - socketIo.providerNamespace.emit("INDEX", { - version: "v1", - contentHash: imageDetails.contentHash, - imageTag: imageDetails.tag, - envId: imageDetails.runtimeEnvironmentId, - }); - } -} diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts new file mode 100644 index 000000000..dafbb89cc --- /dev/null +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -0,0 +1,50 @@ +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { BaseService } from "./baseService.server"; +import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { customAlphabet } from "nanoid"; +import { createRemoteImageBuild } from "../remoteImageBuilder.server"; +import { InitializeDeploymentRequestBody } from "@trigger.dev/core/v3"; + +const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 8); + +export class InitializeDeploymentService extends BaseService { + public async call( + environment: AuthenticatedEnvironment, + payload: InitializeDeploymentRequestBody + ) { + return this.traceWithEnv("call", environment, async (span) => { + const latestDeployment = await this._prisma.workerDeployment.findFirst({ + where: { + environmentId: environment.id, + }, + orderBy: { + createdAt: "desc", + }, + take: 1, + }); + + const nextVersion = calculateNextBuildVersion(latestDeployment?.version); + + // Try and create a depot build and get back the external build data + const externalBuildData = await createRemoteImageBuild(); + + const deployment = await this._prisma.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: payload.contentHash, + shortCode: nanoid(8), + version: nextVersion, + status: "BUILDING", + environmentId: environment.id, + projectId: environment.projectId, + externalBuildData, + }, + }); + + const imageTag = `trigger/${environment.project.externalRef}:${deployment.version}.${environment.slug}`; + + return { deployment, imageTag }; + }); + } +} diff --git a/apps/webapp/app/v3/services/startDeploymentIndexing.server.ts b/apps/webapp/app/v3/services/startDeploymentIndexing.server.ts new file mode 100644 index 000000000..6370297f9 --- /dev/null +++ b/apps/webapp/app/v3/services/startDeploymentIndexing.server.ts @@ -0,0 +1,29 @@ +import { StartDeploymentIndexingRequestBody } from "@trigger.dev/core/v3"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { workerQueue } from "~/services/worker.server"; +import { BaseService } from "./baseService.server"; +import { registryProxy } from "../registryProxy.server"; + +export class StartDeploymentIndexing extends BaseService { + public async call( + environment: AuthenticatedEnvironment, + deploymentId: string, + body: StartDeploymentIndexingRequestBody + ) { + const deployment = await this._prisma.workerDeployment.update({ + where: { + friendlyId: deploymentId, + }, + data: { + imageReference: registryProxy + ? registryProxy.rewriteImageReference(body.imageReference) + : body.imageReference, + status: "DEPLOYING", + }, + }); + + await workerQueue.enqueue("v3.indexDeployment", { id: deployment.id }); + + return deployment; + } +} diff --git a/apps/webapp/app/v3/utils/calculateNextBuildVersion.ts b/apps/webapp/app/v3/utils/calculateNextBuildVersion.ts new file mode 100644 index 000000000..a529259d8 --- /dev/null +++ b/apps/webapp/app/v3/utils/calculateNextBuildVersion.ts @@ -0,0 +1,23 @@ +// Calculate next build version based on the previous version +// Version formats are YYYYMMDD.1, YYYYMMDD.2, etc. +// If there is no previous version, start at Todays date and .1 +export function calculateNextBuildVersion(latestVersion?: string | null): string { + const today = new Date(); + const year = today.getFullYear(); + const month = today.getMonth() + 1; + const day = today.getDate(); + const todayFormatted = `${year}${month < 10 ? "0" : ""}${month}${day < 10 ? "0" : ""}${day}`; + + if (!latestVersion) { + return `${todayFormatted}.1`; + } + + const [date, buildNumber] = latestVersion.split("."); + + if (date === todayFormatted) { + const nextBuildNumber = parseInt(buildNumber, 10) + 1; + return `${date}.${nextBuildNumber}`; + } + + return `${todayFormatted}.1`; +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 935950400..696a984d2 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -41,6 +41,7 @@ "@codemirror/view": "^6.5.0", "@conform-to/react": "^0.6.1", "@conform-to/zod": "^0.6.1", + "@depot/sdk-node": "^0.5.0", "@headlessui/react": "^1.7.8", "@heroicons/react": "^2.0.12", "@highlight-run/node": "^3.1.0", @@ -112,6 +113,7 @@ "ioredis": "^5.3.2", "isbot": "^3.6.5", "jsonpointer": "^5.0.1", + "jwt-decode": "^4.0.0", "lodash.omit": "^4.5.0", "lucide-react": "^0.229.0", "marked": "^4.0.18", diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index 98e354966..3d9d335ea 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -7,23 +7,10 @@ import { WebSocketServer } from "ws"; import { broadcastDevReady, logDevReady } from "@remix-run/server-runtime"; import type { Server as IoServer } from "socket.io"; import type { Server as EngineServer } from "engine.io"; +import { RegistryProxy } from "~/v3/registryProxy.server"; const app = express(); -app.use((req, res, next) => { - // helpful headers: - res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`); - - // /clean-urls/ -> /clean-urls - if (req.path.endsWith("/") && req.path.length > 1) { - const query = req.url.slice(req.path.length); - const safepath = req.path.slice(0, -1).replace(/\/+/g, "/"); - res.redirect(301, safepath + query); - return; - } - next(); -}); - if (process.env.DISABLE_COMPRESSION !== "1") { app.use(compression()); } @@ -40,23 +27,54 @@ app.use(express.static("public", { maxAge: "1h" })); app.use(morgan("tiny")); +process.title = "node webapp-server"; + const MODE = process.env.NODE_ENV; const BUILD_DIR = path.join(process.cwd(), "build"); const build = require(BUILD_DIR); -app.all( - "*", - createRequestHandler({ - build, - mode: MODE, - }) -); - const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000; if (process.env.HTTP_SERVER_DISABLED !== "true") { const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo; const wss: WebSocketServer | undefined = build.entry.module.wss; + const registryProxy: RegistryProxy | undefined = build.entry.module.registryProxy; + + if (registryProxy && process.env.ENABLE_REGISTRY_PROXY === "true") { + console.log(`🐳 Enabling container registry proxy to ${registryProxy.origin}`); + + // Adjusted to match /v2 and any subpath under /v2 + app.all("/v2/*", async (req, res) => { + await registryProxy.call(req, res); + }); + + // This might also be necessary if you need to explicitly match /v2 as well + app.all("/v2", async (req, res) => { + await registryProxy.call(req, res); + }); + } + + app.use((req, res, next) => { + // helpful headers: + res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`); + + // /clean-urls/ -> /clean-urls + if (req.path.endsWith("/") && req.path.length > 1) { + const query = req.url.slice(req.path.length); + const safepath = req.path.slice(0, -1).replace(/\/+/g, "/"); + res.redirect(301, safepath + query); + return; + } + next(); + }); + + app.all( + "*", + createRequestHandler({ + build, + mode: MODE, + }) + ); const server = app.listen(port, () => { console.log(`✅ app ready: http://localhost:${port} [NODE_ENV: ${MODE}]`); diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index b3f96b654..38e25ee85 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -64,15 +64,11 @@ "typecheck": "tsc -p tsconfig.check.json", "build": "npm run clean && run-p build:**", "build:main": "tsup", - "build:facade": "tsup --config tsup.facade.config.ts", - "build:prod-facade": "tsup --config tsup.prod-facade.config.ts", - "build:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts", + "build:workers": "tsup --config tsup.workers.config.ts", "build:prod-containerfile": "cpy --flat src/Containerfile.prod dist/", "dev": "npm run clean && run-p dev:**", "dev:main": "tsup --watch", - "dev:facade": "tsup --config tsup.facade.config.ts --watch", - "dev:prod-facade": "tsup --config tsup.prod-facade.config.ts --watch", - "dev:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts --watch", + "dev:workers": "tsup --config tsup.workers.config.ts --watch", "dev:prod-containerfile": "npm-watch", "clean": "rimraf dist", "start": "node dist/index.js", @@ -80,6 +76,7 @@ }, "dependencies": { "@clack/prompts": "^0.7.0", + "@depot/cli": "0.0.1-cli.2.55.0", "@opentelemetry/api": "^1.7.0", "@opentelemetry/api-logs": "^0.48.0", "@opentelemetry/auto-instrumentations-node": "^0.40.3", @@ -131,6 +128,7 @@ "source-map-support": "^0.5.21", "supports-color": "^9.4.0", "terminal-link": "^3.0.0", + "tiny-invariant": "^1.2.0", "update-check": "^1.5.4", "url": "^0.11.1", "ws": "^8.12.0", diff --git a/packages/cli-v3/src/Containerfile.prod b/packages/cli-v3/src/Containerfile.prod index 9a710f728..bd3672a67 100644 --- a/packages/cli-v3/src/Containerfile.prod +++ b/packages/cli-v3/src/Containerfile.prod @@ -2,17 +2,32 @@ FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dd RUN apk add --no-cache dumb-init -FROM base - -ENV TRIGGER_CONTENT_HASH=__CONTENT_HASH__ -ENV TRIGGER_PROJECT_DIR=__PROJECT_DIR__ -ENV TRIGGER_PROJECT_REF=__PROJECT_REF__ -ENV TRIGGER_CLI_PACKAGE_VERSION=__CLI_PACKAGE_VERSION__ - WORKDIR /app +# Install dependencies +COPY --chown=node:node package.json package-lock.json ./ +RUN npm ci --no-fund --no-audit && npm cache clean --force + +# Development or production stage builds upon the base stage +FROM base AS final + +# Copy the rest of the application COPY --chown=node:node . . +# Use ARG for build-time variables +ARG TRIGGER_PROJECT_ID +ARG TRIGGER_DEPLOYMENT_ID +ARG TRIGGER_DEPLOYMENT_VERSION +ARG TRIGGER_CONTENT_HASH +ARG TRIGGER_PROJECT_REF + +ENV TRIGGER_PROJECT_ID=${TRIGGER_PROJECT_ID} \ + TRIGGER_DEPLOYMENT_ID=${TRIGGER_DEPLOYMENT_ID} \ + TRIGGER_DEPLOYMENT_VERSION=${TRIGGER_DEPLOYMENT_VERSION} \ + TRIGGER_CONTENT_HASH=${TRIGGER_CONTENT_HASH} \ + TRIGGER_PROJECT_REF=${TRIGGER_PROJECT_REF} \ + NODE_ENV=production + USER node -CMD [ "dumb-init", "node", "index.mjs" ] \ No newline at end of file +CMD [ "dumb-init", "node", "index.js" ] \ No newline at end of file diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 989e84125..db40a2351 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -5,10 +5,13 @@ import { WhoAmIResponseSchema, CreateBackgroundWorkerRequestBody, CreateBackgroundWorkerResponse, - CreateImageDetailsRequestBody, - CreateImageDetailsResponse, + StartDeploymentIndexingResponseBody, GetProjectDevResponse, GetEnvironmentVariablesResponseBody, + InitializeDeploymentResponseBody, + InitializeDeploymentRequestBody, + StartDeploymentIndexingRequestBody, + GetDeploymentResponseBody, } from "@trigger.dev/core/v3"; export class CliApiClient { @@ -72,25 +75,6 @@ export class CliApiClient { ); } - async createImageDetails(projectRef: string, body: CreateImageDetailsRequestBody) { - if (!this.accessToken) { - throw new Error("createImageDetails: No access token"); - } - - return zodfetch( - CreateImageDetailsResponse, - `${this.apiURL}/api/v1/projects/${projectRef}/image-details`, - { - method: "POST", - headers: { - Authorization: `Bearer ${this.accessToken}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(body), - } - ); - } - async getProjectDevEnv({ projectRef }: { projectRef: string }) { if (!this.accessToken) { throw new Error("getProjectDevEnv: No access token"); @@ -133,6 +117,57 @@ export class CliApiClient { } ); } + + async initializeDeployment(body: InitializeDeploymentRequestBody) { + if (!this.accessToken) { + throw new Error("initializeDeployment: No access token"); + } + + return zodfetch(InitializeDeploymentResponseBody, `${this.apiURL}/api/v1/deployments`, { + method: "POST", + headers: { + Authorization: `Bearer ${this.accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + } + + async startDeploymentIndexing(deploymentId: string, body: StartDeploymentIndexingRequestBody) { + if (!this.accessToken) { + throw new Error("startDeploymentIndexing: No access token"); + } + + return zodfetch( + StartDeploymentIndexingResponseBody, + `${this.apiURL}/api/v1/deployments/${deploymentId}/start-indexing`, + { + method: "POST", + headers: { + Authorization: `Bearer ${this.accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + + async getDeployment(deploymentId: string) { + if (!this.accessToken) { + throw new Error("getDeployment: No access token"); + } + + return zodfetch( + GetDeploymentResponseBody, + `${this.apiURL}/api/v1/deployments/${deploymentId}`, + { + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: "application/json", + }, + } + ); + } } type ApiResult = diff --git a/packages/cli-v3/src/cli/index.ts b/packages/cli-v3/src/cli/index.ts index 54525b35d..3958fabaf 100644 --- a/packages/cli-v3/src/cli/index.ts +++ b/packages/cli-v3/src/cli/index.ts @@ -9,6 +9,7 @@ import { COMMAND_NAME } from "../consts.js"; import { getVersion } from "../utilities/getVersion.js"; import { printInitialBanner } from "../utilities/initialBanner.js"; import { configureBuildCommand } from "../commands/build.js"; +import { configureDeployCommand } from "../commands/deploy.js"; export const program = new Command(); @@ -56,9 +57,9 @@ program } }); - configureBuildCommand(program); configureDevCommand(program); +configureDeployCommand(program); program .command("update") diff --git a/packages/cli-v3/src/commands/build.ts b/packages/cli-v3/src/commands/build.ts index 462cf6491..b2b70a398 100644 --- a/packages/cli-v3/src/commands/build.ts +++ b/packages/cli-v3/src/commands/build.ts @@ -98,13 +98,7 @@ async function startBuild( apiKey: prodEnv.data.apiKey, }); - const envClient = new CliApiClient(authorization.apiUrl, prodEnv.data.apiKey); - await envClient.createImageDetails(config.project, { - metadata: { - contentHash: buildResult.contentHash, - imageTag: buildResult.imageTag, - }, - }); + logger.log(`⎔ Finished building ${buildResult.imageTag}`); } catch (e) { throw e; } @@ -142,7 +136,7 @@ async function runBuild( logger.log(chalk.green(`Typecheck succeeded.\n`)); } - logger.log(chalk.dim("⎔ Bundling tasks...")); + logger.log(chalk.dim("⎔ Building tasks...")); const result = await build({ stdin: { @@ -150,7 +144,7 @@ async function runBuild( resolveDir: process.cwd(), sourcefile: "__entryPoint.ts", }, - bundle: true, + bundle: false, metafile: true, write: false, minify: false, @@ -160,9 +154,6 @@ async function runBuild( format: "esm", target: ["node18", "es2020"], outdir: "out", - banner: { - js: "import { createRequire } from 'module';const require = createRequire(import.meta.url);", - }, }); if (result.errors.length > 0) { diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts new file mode 100644 index 000000000..56fe6dc6a --- /dev/null +++ b/packages/cli-v3/src/commands/deploy.ts @@ -0,0 +1,571 @@ +import { intro, spinner } from "@clack/prompts"; +import { depot } from "@depot/cli"; +import { ResolvedConfig } from "@trigger.dev/core/v3"; +import { Command } from "commander"; +import { Metafile, build } from "esbuild"; +import { execa } from "execa"; +import { resolve as importResolve } from "import-meta-resolve"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { copyFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { exit } from "node:process"; +import { setTimeout } from "node:timers/promises"; +import invariant from "tiny-invariant"; +import { z } from "zod"; +import * as packageJson from "../../package.json"; +import { CliApiClient } from "../apiClient"; +import { CommonCommandOptions } from "../cli/common.js"; +import { getConfigPath, readConfig } from "../utilities/configFiles.js"; +import { createTempDir, readJSONFile, writeJSONFile } from "../utilities/fileSystem"; +import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; +import { detectPackageNameFromImportPath } from "../utilities/installPackages"; +import { logger } from "../utilities/logger.js"; +import { isLoggedIn } from "../utilities/session.js"; +import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles"; + +const DeployCommandOptions = CommonCommandOptions.extend({ + skipTypecheck: z.boolean().optional(), +}); + +type DeployCommandOptions = z.infer; + +export function configureDeployCommand(program: Command) { + program + .command("deploy") + .description("Deploy your Trigger.dev v3 project to the cloud.") + .argument("[path]", "The path to the project", ".") + .option("-T, --skip-typecheck", "Whether to skip the pre-build typecheck") + .option( + "-l, --log-level ", + "The log level to use (debug, info, log, warn, error, none)", + "log" + ) + .action(async (path, options) => { + try { + await deployCommand(path, options); + } catch (e) { + //todo error reporting + throw e; + } + }); +} + +export async function deployCommand(dir: string, anyOptions: unknown) { + const options = DeployCommandOptions.safeParse(anyOptions); + + if (!options.success) { + throw new Error(`Invalid options: ${options.error}`); + } + + const authorization = await isLoggedIn(); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + logger.error( + `Failed to connect to ${authorization.config?.apiUrl}. Are you sure it's the correct URL?` + ); + } else { + logger.error("You must login first. Use `trigger.dev login` to login."); + } + process.exitCode = 1; + return; + } + + if (options.data.logLevel) { + logger.loggerLevel = options.data.logLevel; + } + + await printStandloneInitialBanner(true); + + const configPath = await getConfigPath(dir); + const config = await readConfig(configPath); + + const apiClient = new CliApiClient(authorization.config.apiUrl, authorization.config.accessToken); + + const prodEnv = await apiClient.getProjectProdEnv({ projectRef: config.project }); + + if (!prodEnv.success) { + throw new Error(prodEnv.error); + } + + const environmentClient = new CliApiClient(authorization.config.apiUrl, prodEnv.data.apiKey); + + intro(`Preparing to deploy "${prodEnv.data.name}" (${config.project})`); + + // Step 1: Build the project into a temporary directory + const compilation = await compileProject(config, options.data); + + const deploymentSpinner = spinner(); + + deploymentSpinner.start("Initializing deployment"); + + // Step 2: Initialize a deployment on the server (response will have everything we need to build an image) + const deploymentResponse = await environmentClient.initializeDeployment({ + contentHash: compilation.contentHash, + }); + + if (!deploymentResponse.success) { + deploymentSpinner.stop(`Failed to initialize deployment: ${deploymentResponse.error}`); + exit(1); + } + + deploymentSpinner.message(`Deploying version ${deploymentResponse.data.version}`); + + // If the deployment doesn't have any externalBuildData, then we can't use the remote image builder + // TODO: handle this and allow the user to the build and push the image themselves + if (!deploymentResponse.data.externalBuildData) { + deploymentSpinner.stop( + `Failed to initialize deployment. The deployment does not have any external build data. Support for local building coming soon.` + ); + exit(1); + } + + const registryHost = new URL(prodEnv.data.apiUrl).host; + + const image = await buildAndPushImage({ + registryHost, + auth: authorization.config.accessToken, + imageTag: deploymentResponse.data.imageTag, + buildId: deploymentResponse.data.externalBuildData.buildId, + buildToken: deploymentResponse.data.externalBuildData.buildToken, + buildProjectId: deploymentResponse.data.externalBuildData.projectId, + cwd: compilation.path, + projectId: config.project, + deploymentId: deploymentResponse.data.id, + deploymentVersion: deploymentResponse.data.version, + contentHash: deploymentResponse.data.contentHash, + projectRef: config.project, + }); + + if (!image.ok) { + deploymentSpinner.stop(`Failed to build and push image: ${image.error}`); + exit(1); + } + + const imageReference = `${registryHost}/${image.image}${image.digest ? `@${image.digest}` : ""}`; + + deploymentSpinner.message( + `${deploymentResponse.data.version} image uploaded, starting indexing process` + ); + + logger.debug(`Image built and pushed: ${imageReference}`); + + // Need to update the deployment with the image and start the deployment (indexing) + // registry.digitalocean.com/trigger/yubjwjsfkxnylobaqvqz:20240306.41.prod@sha256:8b48dd2866bc8878644d2880bbe35a27e66cf6ff78aa1e489d7fdde5e228faf1 + // Step 5: Update the deployment with the image and start the deployment (indexing) + const startIndexingResponse = await environmentClient.startDeploymentIndexing( + deploymentResponse.data.id, + { + imageReference, + } + ); + + if (!startIndexingResponse.success) { + deploymentSpinner.stop(`Failed to start indexing: ${startIndexingResponse.error}`); + exit(1); + } + + // Step 6: Wait for the deployment to finish and print the result + const completedDeployment = await waitForDeploymentToComplete( + deploymentResponse.data.id, + environmentClient + ); + + if (!completedDeployment) { + deploymentSpinner.stop(`Deployment failed to complete`); + exit(1); + } + + deploymentSpinner.stop(`Deployment completed successfully, you can now use this version`); +} + +// Poll every 1 second for the deployment to complete +async function waitForDeploymentToComplete( + deploymentId: string, + client: CliApiClient, + timeoutInSeconds: number = 60 +) { + const start = Date.now(); + + while (true) { + if (Date.now() - start > timeoutInSeconds * 1000) { + return; + } + + const deployment = await client.getDeployment(deploymentId); + + if (!deployment.success) { + throw new Error(deployment.error); + } + + logger.debug(`Deployment status: ${deployment.data.status}`); + + if (deployment.data.status === "DEPLOYED") { + return deployment.data; + } + + await setTimeout(1000); + } +} + +type BuildAndPushImageOptions = { + registryHost: string; + auth: string; + imageTag: string; + buildId: string; + buildToken: string; + buildProjectId: string; + cwd: string; + projectId: string; + deploymentId: string; + deploymentVersion: string; + contentHash: string; + projectRef: string; +}; + +type BuildAndPushImageResults = + | { + ok: true; + image: string; + digest?: string; + } + | { + ok: false; + error: string; + }; + +async function buildAndPushImage( + options: BuildAndPushImageOptions +): Promise { + // Step 3: Ensure we are "logged in" to our registry by writing to $HOME/.docker/config.json + // TODO: make sure this works on windows + const dockerConfigDir = await ensureLoggedIntoDockerRegistry(options.registryHost, { + username: "trigger", + password: options.auth, + }); + + const args = [ + "build", + "-f", + "Containerfile", + "--platform", + "linux/amd64", + "--provenance", + "false", + "--build-arg", + `TRIGGER_PROJECT_ID=${options.projectId}`, + "--build-arg", + `TRIGGER_DEPLOYMENT_ID=${options.deploymentId}`, + "--build-arg", + `TRIGGER_DEPLOYMENT_VERSION=${options.deploymentVersion}`, + "--build-arg", + `TRIGGER_CONTENT_HASH=${options.contentHash}`, + "--build-arg", + `TRIGGER_PROJECT_REF=${options.projectRef}`, + "-t", + `${options.registryHost}/${options.imageTag}`, + "--push", + ".", + ]; + + logger.debug(`depot ${args.join(" ")}`); + + // Step 4: Build and push the image + const childProcess = depot(args, { + cwd: options.cwd, + env: { + DEPOT_BUILD_ID: options.buildId, + DEPOT_TOKEN: options.buildToken, + DEPOT_PROJECT_ID: options.buildProjectId, + DEPOT_NO_SUMMARY_LINK: "1", + DOCKER_CONFIG: dockerConfigDir, + }, + }); + + const errors: string[] = []; + + try { + await new Promise((res, rej) => { + // For some reason everything is output on stderr, not stdout + childProcess.stderr?.on("data", (data: Buffer) => { + const text = data.toString(); + + errors.push(text); + }); + + childProcess.on("error", (e) => rej(e)); + childProcess.on("close", () => res()); + }); + + const digest = extractImageDigest(errors); + + return { + ok: true, + image: options.imageTag, + digest, + }; + } catch (e) { + return { + ok: false, + error: e instanceof Error ? e.message : JSON.stringify(e), + }; + } +} + +function extractImageDigest(outputs: string[]) { + const imageDigestRegex = /sha256:[a-f0-9]{64}/; + + for (const line of outputs) { + if (line.includes("pushing manifest")) { + const imageDigestMatch = line.match(imageDigestRegex); + if (imageDigestMatch) { + return imageDigestMatch[0]; + } + } + } +} + +async function compileProject(config: ResolvedConfig, options: DeployCommandOptions) { + if (!options.skipTypecheck) { + await typecheckProject(config, options); + } + + const createAuthCodeSpinner = spinner(); + createAuthCodeSpinner.start(`Compiling project "${config.project}" in "${config.projectDir}"`); + + const taskFiles = await gatherTaskFiles(config); + const workerFacade = readFileSync( + new URL(importResolve("./workers/prod/worker-facade.js", import.meta.url)).href.replace( + "file://", + "" + ), + "utf-8" + ); + + const registerTracingPath = new URL( + importResolve("./workers/common/register-tracing.js", import.meta.url) + ).href.replace("file://", ""); + + const workerContents = workerFacade + .replace("__TASKS__", createTaskFileImports(taskFiles)) + .replace("__REGISTER_TRACING__", `import { tracingSDK } from "${registerTracingPath}";`); + + const result = await build({ + stdin: { + contents: workerContents, + resolveDir: process.cwd(), + sourcefile: "__entryPoint.ts", + }, + bundle: true, + metafile: true, + write: false, + minify: false, + sourcemap: "external", // does not set the //# sourceMappingURL= comment in the file, we handle it ourselves + packages: "external", // https://esbuild.github.io/api/#packages + logLevel: "error", + platform: "node", + format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching + target: ["node18", "es2020"], + outdir: "out", + define: { + TRIGGER_API_URL: `"${config.triggerUrl}"`, + }, + }); + + if (result.errors.length > 0) { + createAuthCodeSpinner.stop("Build failed"); + exit(1); + } + + const entryPointContents = readFileSync( + new URL(importResolve("./workers/prod/entry-point.js", import.meta.url)).href.replace( + "file://", + "" + ), + "utf-8" + ); + + const entryPointResult = await build({ + stdin: { + contents: entryPointContents, + resolveDir: process.cwd(), + sourcefile: "index.ts", + }, + bundle: true, + metafile: true, + write: false, + minify: false, + sourcemap: false, + packages: "external", // https://esbuild.github.io/api/#packages + logLevel: "error", + platform: "node", + format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching + target: ["node18", "es2020"], + outdir: "out", + }); + + if (entryPointResult.errors.length > 0) { + createAuthCodeSpinner.stop("Build failed"); + exit(1); + } + + // Create a tmp directory to store the build + const tempDir = await createTempDir(); + + logger.debug(`Writing compiled files to ${tempDir}`); + + // Get the metaOutput for the result build + const metaOutput = result.metafile!.outputs[join("out", "stdin.js")]; + + invariant(metaOutput, "Meta output for the result build is missing"); + + // Get the metaOutput for the entryPoint build + const entryPointMetaOutput = entryPointResult.metafile!.outputs[join("out", "stdin.js")]; + + invariant(entryPointMetaOutput, "Meta output for the entryPoint build is missing"); + + // Get the outputFile and the sourceMapFile for the result build + const workerOutputFile = result.outputFiles.find( + (file) => file.path === join(config.projectDir, "out", "stdin.js") + ); + + invariant(workerOutputFile, "Output file for the result build is missing"); + + const workerSourcemapFile = result.outputFiles.find( + (file) => file.path === join(config.projectDir, "out", "stdin.js.map") + ); + + invariant(workerSourcemapFile, "Sourcemap file for the result build is missing"); + + // Get the outputFile for the entryPoint build + + const entryPointOutputFile = entryPointResult.outputFiles.find( + (file) => file.path === join(config.projectDir, "out", "stdin.js") + ); + + invariant(entryPointOutputFile, "Output file for the entryPoint build is missing"); + + // Save the result outputFile to /tmp/dir/worker.js (and make sure to map the sourceMap to the correct location in the file) + await writeFile( + join(tempDir, "worker.js"), + `${workerOutputFile.text}\n//# sourceMappingURL=worker.js.map` + ); + // Save the sourceMapFile to /tmp/dir/worker.js.map + await writeFile(join(tempDir, "worker.js.map"), workerSourcemapFile.text); + // Save the entryPoint outputFile to /tmp/dir/index.js + await writeFile(join(tempDir, "index.js"), entryPointOutputFile.text); + + // Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json + const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports]; + const projectPackageJson = await readJSONFile(join(config.projectDir, "package.json")); + const dependencies = gatherRequiredDependencies(allImports, projectPackageJson); + + await writeJSONFile(join(tempDir, "package.json"), { + name: "trigger-worker", + version: "0.0.0", + description: "", + dependencies, + }); + + createAuthCodeSpinner.stop(`Project "${config.project}" compiled successfully`); + + // Run npm install --package-lock-only in /tmp/dir to produce a package-lock.json + const resolvingDepsSpinner = spinner(); + + resolvingDepsSpinner.start("Resolving dependencies"); + + await execa("npm", ["install", "--package-lock-only"], { + cwd: tempDir, + }); + + resolvingDepsSpinner.stop("Dependencies resolved"); + // Write the Containerfile to /tmp/dir/Containerfile + const containerFilePath = new URL( + importResolve("./Containerfile.prod", import.meta.url) + ).href.replace("file://", ""); + // Copy the Containerfile to /tmp/dir/Containerfile + await copyFile(containerFilePath, join(tempDir, "Containerfile")); + + const contentHasher = createHash("sha256"); + contentHasher.update(Buffer.from(entryPointOutputFile.text)); + contentHasher.update(Buffer.from(workerOutputFile.text)); + // Sort the dependencies by key to ensure consistent hashing + const sortedDependencies = Object.fromEntries( + Object.entries(dependencies).sort(([a], [b]) => a.localeCompare(b)) + ); + + contentHasher.update(Buffer.from(JSON.stringify(sortedDependencies))); + + const contentHash = contentHasher.digest("hex"); + + return { path: tempDir, contentHash }; +} + +async function typecheckProject(config: ResolvedConfig, options: DeployCommandOptions) { + const createAuthCodeSpinner = spinner(); + createAuthCodeSpinner.start("Typechecking project"); + + await setTimeout(2000); + + createAuthCodeSpinner.stop(`Project typechecked with 0 errors`); +} + +// Returns the dependencies that are required by the output that are found in output and the CLI package dependencies +// Returns the dependency names and the version to use (taken from the CLI deps package.json) +function gatherRequiredDependencies( + imports: Metafile["outputs"][string]["imports"], + externalPackageJson?: { dependencies: Record } +) { + const dependencies: Record = {}; + + for (const file of imports) { + if (file.kind !== "require-call" || !file.external) { + continue; + } + + const packageName = detectPackageNameFromImportPath(file.path); + + if (dependencies[packageName]) { + continue; + } + + const externalDependencyVersion = (externalPackageJson?.dependencies ?? {})[packageName]; + + if (externalDependencyVersion) { + dependencies[packageName] = externalDependencyVersion; + continue; + } + + const internalDependencyVersion = (packageJson.dependencies as Record)[ + packageName + ]; + + if (internalDependencyVersion) { + dependencies[packageName] = internalDependencyVersion; + } + } + + return dependencies; +} + +async function ensureLoggedIntoDockerRegistry( + registryHost: string, + auth: { username: string; password: string } +) { + const tmpDir = await createTempDir(); + // Read the current docker config + const dockerConfigPath = join(tmpDir, "config.json"); + + await writeJSONFile(dockerConfigPath, { + auths: { + [registryHost]: { + auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"), + }, + }, + }); + + logger.debug(`Writing docker config to ${dockerConfigPath}`); + + return tmpDir; +} diff --git a/packages/cli-v3/src/commands/dev.tsx b/packages/cli-v3/src/commands/dev.tsx index e82de398a..f3ca76412 100644 --- a/packages/cli-v3/src/commands/dev.tsx +++ b/packages/cli-v3/src/commands/dev.tsx @@ -25,12 +25,13 @@ import { z } from "zod"; import * as packageJson from "../../package.json"; import { CliApiClient } from "../apiClient"; import { CommonCommandOptions } from "../cli/common.js"; -import { BackgroundWorker, BackgroundWorkerCoordinator } from "../dev/backgroundWorker.js"; +import { BackgroundWorker, BackgroundWorkerCoordinator } from "../workers/dev/backgroundWorker.js"; import { getConfigPath, readConfig } from "../utilities/configFiles"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger.js"; import { isLoggedIn } from "../utilities/session.js"; import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles"; +import { detectPackageNameFromImportPath } from "../utilities/installPackages"; let apiClient: CliApiClient | undefined; @@ -643,20 +644,3 @@ function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) { return dependencies; } - -// Expects path to be in the format: -// - source-map-support/register.js -// - @opentelemetry/api -// - zod -// -// With the result being: -// - source-map-support -// - @opentelemetry/api -// - zod -function detectPackageNameFromImportPath(path: string): string { - if (path.startsWith("@")) { - return path.split("/").slice(0, 2).join("/"); - } else { - return path.split("/")[0] as string; - } -} diff --git a/packages/cli-v3/src/utilities/fileSystem.ts b/packages/cli-v3/src/utilities/fileSystem.ts index 1862a37b2..f7dfde2b1 100644 --- a/packages/cli-v3/src/utilities/fileSystem.ts +++ b/packages/cli-v3/src/utilities/fileSystem.ts @@ -1,7 +1,8 @@ import fsSync from "fs"; import fsModule, { writeFile } from "fs/promises"; import fs from "node:fs"; -import pathModule from "path"; +import { tmpdir } from "node:os"; +import pathModule from "node:path"; // Creates a file at the given path, if the directory doesn't exist it will be created export async function createFile(path: string, contents: string): Promise { @@ -52,8 +53,22 @@ export async function readJSONFile(path: string) { return JSON.parse(fileContents); } +export async function safeFeadJSONFile(path: string) { + try { + const fileExists = await pathExists(path); + + if (!fileExists) return; + + const fileContents = await readFile(path); + + return JSON.parse(fileContents); + } catch { + return; + } +} + export async function writeJSONFile(path: string, json: any) { - await writeFile(path, JSON.stringify(json, null, 2)); + await writeFile(path, JSON.stringify(json), "utf8"); } export function readJSONFileSync(path: string) { @@ -69,3 +84,14 @@ export function safeDeleteFileSync(path: string) { // ignore error } } + +// Create a temporary directory within the OS's temp directory +export async function createTempDir(): Promise { + // Generate a unique temp directory path + const tempDirPath: string = pathModule.join(tmpdir(), "trigger-"); + + // Create the temp directory synchronously and return the path + const directory = await fsModule.mkdtemp(tempDirPath); + + return directory; +} diff --git a/packages/cli-v3/src/dev/ink-table.tsx b/packages/cli-v3/src/utilities/ink-table.tsx similarity index 100% rename from packages/cli-v3/src/dev/ink-table.tsx rename to packages/cli-v3/src/utilities/ink-table.tsx diff --git a/packages/cli-v3/src/utilities/installPackages.ts b/packages/cli-v3/src/utilities/installPackages.ts index 186ec7e6f..cbf9b3fdb 100644 --- a/packages/cli-v3/src/utilities/installPackages.ts +++ b/packages/cli-v3/src/utilities/installPackages.ts @@ -90,3 +90,20 @@ async function getPackageVersion(path: string) { return undefined; } } + +// Expects path to be in the format: +// - source-map-support/register.js +// - @opentelemetry/api +// - zod +// +// With the result being: +// - source-map-support +// - @opentelemetry/api +// - zod +export function detectPackageNameFromImportPath(path: string): string { + if (path.startsWith("@")) { + return path.split("/").slice(0, 2).join("/"); + } else { + return path.split("/")[0] as string; + } +} diff --git a/packages/cli-v3/src/utilities/session.ts b/packages/cli-v3/src/utilities/session.ts index 5c5308004..88beefca4 100644 --- a/packages/cli-v3/src/utilities/session.ts +++ b/packages/cli-v3/src/utilities/session.ts @@ -15,6 +15,10 @@ export async function isLoggedIn() { return { ok: false as const, error: userData.error, + config: { + apiUrl: config.apiUrl, + accessToken: config.accessToken, + }, }; } diff --git a/packages/cli-v3/src/dev/register-tracing.ts b/packages/cli-v3/src/workers/common/register-tracing.ts similarity index 100% rename from packages/cli-v3/src/dev/register-tracing.ts rename to packages/cli-v3/src/workers/common/register-tracing.ts diff --git a/packages/cli-v3/src/dev/backgroundWorker.ts b/packages/cli-v3/src/workers/dev/backgroundWorker.ts similarity index 98% rename from packages/cli-v3/src/dev/backgroundWorker.ts rename to packages/cli-v3/src/workers/dev/backgroundWorker.ts index ce2522f3d..2d7392aa3 100644 --- a/packages/cli-v3/src/dev/backgroundWorker.ts +++ b/packages/cli-v3/src/workers/dev/backgroundWorker.ts @@ -22,9 +22,9 @@ import { Evt } from "evt"; import { ChildProcess, fork } from "node:child_process"; import { dirname, resolve } from "node:path"; import terminalLink from "terminal-link"; -import { safeDeleteFileSync } from "../utilities/fileSystem.js"; -import { installPackages } from "../utilities/installPackages.js"; -import { logger } from "../utilities/logger.js"; +import { safeDeleteFileSync } from "../../utilities/fileSystem.js"; +import { installPackages } from "../../utilities/installPackages.js"; +import { logger } from "../../utilities/logger.js"; export type CurrentWorkers = BackgroundWorkerCoordinator["currentWorkers"]; export class BackgroundWorkerCoordinator { diff --git a/packages/cli-v3/src/dev/worker-facade.ts b/packages/cli-v3/src/workers/dev/worker-facade.ts similarity index 98% rename from packages/cli-v3/src/dev/worker-facade.ts rename to packages/cli-v3/src/workers/dev/worker-facade.ts index 4bee6a2dd..29f961876 100644 --- a/packages/cli-v3/src/dev/worker-facade.ts +++ b/packages/cli-v3/src/workers/dev/worker-facade.ts @@ -31,13 +31,12 @@ import { taskContextManager, workerToChildMessages, type BackgroundWorkerProperties, - type TracingDiagnosticLogLevel, } from "@trigger.dev/core/v3"; -import * as packageJson from "../../package.json"; +import * as packageJson from "../../../package.json"; import { Resource } from "@opentelemetry/resources"; import { flattenAttributes } from "@trigger.dev/core/v3"; -import { TaskMetadataWithFunctions } from "../types.js"; +import { TaskMetadataWithFunctions } from "../../types.js"; const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger }); const consoleInterceptor = new ConsoleInterceptor(otelLogger); diff --git a/packages/cli-v3/src/prod/backgroundWorker.ts b/packages/cli-v3/src/workers/prod/backgroundWorker.ts similarity index 98% rename from packages/cli-v3/src/prod/backgroundWorker.ts rename to packages/cli-v3/src/workers/prod/backgroundWorker.ts index 7ce66e853..4f556bdf4 100644 --- a/packages/cli-v3/src/prod/backgroundWorker.ts +++ b/packages/cli-v3/src/workers/prod/backgroundWorker.ts @@ -17,7 +17,7 @@ import { } from "@trigger.dev/core/v3"; import { Evt } from "evt"; import { ChildProcess, fork } from "node:child_process"; -import { safeDeleteFileSync } from "../utilities/fileSystem"; +import { safeDeleteFileSync } from "../../utilities/fileSystem"; class UnexpectedExitError extends Error { constructor(public code: number) { @@ -88,7 +88,7 @@ export class ProdBackgroundWorker { safeDeleteFileSync(`${this.path}.map`); } - async initialize() { + async initialize(options?: { env?: Record }) { if (this._initialized) { throw new Error("Worker already initialized"); } @@ -100,6 +100,7 @@ export class ProdBackgroundWorker { stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], env: { ...this.params.env, + ...options?.env, }, }); diff --git a/packages/cli-v3/src/prod-worker.ts b/packages/cli-v3/src/workers/prod/entry-point.ts similarity index 87% rename from packages/cli-v3/src/prod-worker.ts rename to packages/cli-v3/src/workers/prod/entry-point.ts index 553ace1ac..2bbc63042 100644 --- a/packages/cli-v3/src/prod-worker.ts +++ b/packages/cli-v3/src/workers/prod/entry-point.ts @@ -6,7 +6,7 @@ import { } from "@trigger.dev/core/v3"; import { HttpReply, getTextBody, SimpleLogger, getRandomPortNumber } from "@trigger.dev/core-apps"; import { createServer } from "node:http"; -import { ProdBackgroundWorker } from "./prod/backgroundWorker"; +import { ProdBackgroundWorker } from "./backgroundWorker"; const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber()); const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1"; @@ -24,8 +24,8 @@ class ProdWorker { private projectDir = process.env.TRIGGER_PROJECT_DIR!; private projectRef = process.env.TRIGGER_PROJECT_REF!; private envId = process.env.TRIGGER_ENV_ID!; - private cliPackageVersion = process.env.TRIGGER_CLI_PACKAGE_VERSION!; private attemptId = process.env.TRIGGER_ATTEMPT_ID || "index-only"; + private deploymentId = process.env.TRIGGER_DEPLOYMENT_ID!; private executing = false; private completed = false; @@ -44,7 +44,7 @@ class ProdWorker { ) { this.#coordinatorSocket = this.#createCoordinatorSocket(); - this.#backgroundWorker = new ProdBackgroundWorker(this.#getWorkerEntryPath(this.contentHash), { + this.#backgroundWorker = new ProdBackgroundWorker("worker.js", { projectDir: this.projectDir, env: { TRIGGER_API_URL: this.apiUrl, @@ -73,6 +73,18 @@ class ProdWorker { } #createCoordinatorSocket() { + logger.log("connecting to coordinator", { + host: COORDINATOR_HOST, + port: COORDINATOR_PORT, + deploymentId: this.deploymentId, + podName: POD_NAME, + machineName: MACHINE_NAME, + contentHash: this.contentHash, + projectRef: this.projectRef, + attemptId: this.attemptId, + envId: this.envId, + }); + const coordinatorConnection = new ZodSocketConnection({ namespace: "prod-worker", host: COORDINATOR_HOST, @@ -83,10 +95,10 @@ class ProdWorker { "x-machine-name": MACHINE_NAME, "x-pod-name": POD_NAME, "x-trigger-content-hash": this.contentHash, - "x-trigger-cli-package-version": this.cliPackageVersion, "x-trigger-project-ref": this.projectRef, "x-trigger-attempt-id": this.attemptId, "x-trigger-env-id": this.envId, + "x-trigger-deployment-id": this.deploymentId, }, handlers: { RESUME: async (message) => { @@ -131,6 +143,7 @@ class ProdWorker { const { success } = await socket.emitWithAck("INDEX_TASKS", { version: "v1", + deploymentId: this.deploymentId, ...taskResources, }); @@ -259,12 +272,13 @@ class ProdWorker { return httpServer; } - #getWorkerEntryPath(contentHash: string) { - return `${contentHash}.mjs`; - } - async #initializeWorker() { - await this.#backgroundWorker.initialize(); + // Make an API call for the env vars + // Don't use ApiClient again + // Pass those into this.#backgroundWorker.initialize() + const envVars = await this.#fetchEnvironmentVariables(); + + await this.#backgroundWorker.initialize({ env: envVars }); let packageVersion: string | undefined; @@ -294,6 +308,23 @@ class ProdWorker { }; } + async #fetchEnvironmentVariables(): Promise> { + const response = await fetch(`${this.apiUrl}/api/v1/projects/${this.projectRef}/envvars`, { + method: "GET", + headers: { + Authorization: `Bearer ${this.apiKey}`, + }, + }); + + if (!response.ok) { + return {}; + } + + const data = await response.json(); + + return data?.variables ?? {}; + } + start() { this.#httpServer.listen(this.#httpPort, this.host); } diff --git a/packages/cli-v3/src/prod-facade.ts b/packages/cli-v3/src/workers/prod/worker-facade.ts similarity index 90% rename from packages/cli-v3/src/prod-facade.ts rename to packages/cli-v3/src/workers/prod/worker-facade.ts index 178fef4aa..fe75f2cd3 100644 --- a/packages/cli-v3/src/prod-facade.ts +++ b/packages/cli-v3/src/workers/prod/worker-facade.ts @@ -1,20 +1,9 @@ -// import "source-map-support/register"; -import { TracingSDK } from "@trigger.dev/core/v3"; -// import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai"; +import { type TracingSDK } from "@trigger.dev/core/v3"; +import "source-map-support/register.js"; -// IMPORTANT: this needs to be the first import to work properly -// WARNING: [WARNING] Constructing "ImportInTheMiddle" will crash at run-time because it's an import namespace object, not a constructor [call-import-namespace] -// TODO: https://github.com/open-telemetry/opentelemetry-js/issues/3954 -const tracingSDK = new TracingSDK({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", - resource: new Resource({ - [SemanticInternalAttributes.CLI_VERSION]: packageJson.version, - }), - instrumentations: [ - // new OpenAIInstrumentation(), - ], - diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", -}); +__REGISTER_TRACING__; +declare const __REGISTER_TRACING__: unknown; +declare const tracingSDK: TracingSDK; const otelTracer = tracingSDK.getTracer("trigger-prod-worker", packageJson.version); const otelLogger = tracingSDK.getLogger("trigger-prod-worker", packageJson.version); @@ -22,8 +11,8 @@ const otelLogger = tracingSDK.getLogger("trigger-prod-worker", packageJson.versi import { SpanKind } from "@opentelemetry/api"; import { ConsoleInterceptor, - ProdRuntimeManager, OtelTaskLogger, + ProdRuntimeManager, SemanticInternalAttributes, TaskMetadataWithFilePath, TaskRunContext, @@ -43,12 +32,10 @@ import { workerToChildMessages, type BackgroundWorkerProperties, } from "@trigger.dev/core/v3"; -import * as packageJson from "../package.json"; +import * as packageJson from "../../../package.json"; -import { Resource } from "@opentelemetry/resources"; import { flattenAttributes } from "@trigger.dev/core/v3"; -import { TaskMetadataWithFunctions } from "./types"; -import { TracingDiagnosticLogLevel } from "@trigger.dev/core/v3"; +import { TaskMetadataWithFunctions } from "../../types"; const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger }); const consoleInterceptor = new ConsoleInterceptor(otelLogger); diff --git a/packages/cli-v3/tsup.prod-facade.config.ts b/packages/cli-v3/tsup.prod-facade.config.ts deleted file mode 100644 index 2c03c4520..000000000 --- a/packages/cli-v3/tsup.prod-facade.config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - clean: false, - dts: true, - tsconfig: "tsconfig.json", - splitting: false, - entry: ["src/prod-facade.ts"], - format: ["esm"], - minify: false, - metafile: false, - sourcemap: true, - target: "esnext", - outDir: "dist", - noExternal: ["zod", /traceloop/, /opentelemetry/], -}); diff --git a/packages/cli-v3/tsup.facade.config.ts b/packages/cli-v3/tsup.workers.config.ts similarity index 54% rename from packages/cli-v3/tsup.facade.config.ts rename to packages/cli-v3/tsup.workers.config.ts index 1553c1844..5c53c2e9c 100644 --- a/packages/cli-v3/tsup.facade.config.ts +++ b/packages/cli-v3/tsup.workers.config.ts @@ -5,11 +5,16 @@ export default defineConfig({ dts: false, tsconfig: "tsconfig.json", splitting: false, - entry: ["src/dev/worker-facade.ts", "src/dev/register-tracing.ts"], + entry: [ + "src/workers/dev/worker-facade.ts", + "src/workers/common/register-tracing.ts", + "src/workers/prod/worker-facade.ts", + "src/workers/prod/entry-point.ts", + ], format: ["esm"], minify: false, metafile: false, sourcemap: false, target: "esnext", - outDir: "dist", + outDir: "dist/workers", }); diff --git a/packages/core-apps/src/provider.ts b/packages/core-apps/src/provider.ts index 9b5b4751e..32206100d 100644 --- a/packages/core-apps/src/provider.ts +++ b/packages/core-apps/src/provider.ts @@ -135,6 +135,8 @@ export class ProviderShell implements Provider { contentHash: message.contentHash, imageTag: message.imageTag, envId: message.envId, + apiKey: message.apiKey, + apiUrl: message.apiUrl, }); } catch (error) { logger.error("task index failed", error); diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 3eeb266c1..8975ad949 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -12,6 +12,7 @@ export type WhoAmIResponse = z.infer; export const GetProjectDevResponse = z.object({ apiKey: z.string(), name: z.string(), + apiUrl: z.string(), }); export type GetProjectDevResponse = z.infer; @@ -89,15 +90,67 @@ export type GetEnvironmentVariablesResponseBody = z.infer< typeof GetEnvironmentVariablesResponseBody >; -export const CreateImageDetailsRequestBody = z.object({ - metadata: ImageDetailsMetadata, +export const StartDeploymentIndexingRequestBody = z.object({ + imageReference: z.string(), }); -export type CreateImageDetailsRequestBody = z.infer; +export type StartDeploymentIndexingRequestBody = z.infer; -export const CreateImageDetailsResponse = z.object({ +export const StartDeploymentIndexingResponseBody = z.object({ id: z.string(), contentHash: z.string(), }); -export type CreateImageDetailsResponse = z.infer; +export type StartDeploymentIndexingResponseBody = z.infer< + typeof StartDeploymentIndexingResponseBody +>; + +export const ExternalBuildData = z.object({ + buildId: z.string(), + buildToken: z.string(), + projectId: z.string(), +}); + +export type ExternalBuildData = z.infer; + +export const InitializeDeploymentResponseBody = z.object({ + id: z.string(), + contentHash: z.string(), + shortCode: z.string(), + version: z.string(), + imageTag: z.string(), + externalBuildData: ExternalBuildData.optional(), +}); + +export type InitializeDeploymentResponseBody = z.infer; + +export const InitializeDeploymentRequestBody = z.object({ + contentHash: z.string(), +}); + +export type InitializeDeploymentRequestBody = z.infer; + +export const GetDeploymentResponseBody = z.object({ + id: z.string(), + status: z.enum(["PENDING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED"]), + contentHash: z.string(), + shortCode: z.string(), + version: z.string(), + imageReference: z.string().optional(), + worker: z + .object({ + id: z.string(), + version: z.string(), + tasks: z.array( + z.object({ + id: z.string(), + slug: z.string(), + filePath: z.string(), + exportName: z.string(), + }) + ), + }) + .optional(), +}); + +export type GetDeploymentResponseBody = z.infer; diff --git a/packages/core/src/v3/schemas/resources.ts b/packages/core/src/v3/schemas/resources.ts index 612d8a2db..ae7e5c7f2 100644 --- a/packages/core/src/v3/schemas/resources.ts +++ b/packages/core/src/v3/schemas/resources.ts @@ -14,7 +14,7 @@ export type TaskResource = z.infer; export const BackgroundWorkerMetadata = z.object({ packageVersion: z.string(), contentHash: z.string(), - cliPackageVersion: z.string(), + cliPackageVersion: z.string().optional(), tasks: z.array(TaskResource), }); diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index cc2111b3b..9d95a9c0a 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -62,6 +62,8 @@ export const PlatformToProviderMessages = { imageTag: z.string(), contentHash: z.string(), envId: z.string(), + apiKey: z.string(), + apiUrl: z.string(), }), }, INVOKE: { @@ -111,8 +113,9 @@ export const CoordinatorToPlatformMessages = { version: z.literal("v1").default("v1"), projectRef: z.string(), envId: z.string(), + deploymentId: z.string(), metadata: z.object({ - cliPackageVersion: z.string(), + cliPackageVersion: z.string().optional(), contentHash: z.string(), packageVersion: z.string(), tasks: TaskResource.array(), @@ -227,6 +230,7 @@ export const ProdWorkerToCoordinatorMessages = { INDEX_TASKS: { message: z.object({ version: z.literal("v1").default("v1"), + deploymentId: z.string(), tasks: TaskResource.array(), packageVersion: z.string(), }), @@ -308,10 +312,10 @@ export const CoordinatorToProdWorkerMessages = { }; export const ProdWorkerSocketData = z.object({ - cliPackageVersion: z.string(), contentHash: z.string(), projectRef: z.string(), envId: z.string(), attemptId: z.string(), podName: z.string(), + deploymentId: z.string(), }); diff --git a/packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql b/packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql new file mode 100644 index 000000000..bc1f3c231 --- /dev/null +++ b/packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql @@ -0,0 +1,70 @@ +/* + Warnings: + + - A unique constraint covering the columns `[backgroundWorkerId]` on the table `ImageDetails` will be added. If there are existing duplicate values, this will fail. + +*/ +-- CreateEnum +CREATE TYPE "WorkerDeploymentStatus" AS ENUM ('PENDING', 'DEPLOYING', 'DEPLOYED', 'FAILED', 'CANCELED'); + +-- CreateTable +CREATE TABLE "WorkerDeployment" ( + "id" TEXT NOT NULL, + "friendlyId" TEXT NOT NULL, + "shortCode" TEXT NOT NULL, + "version" TEXT NOT NULL, + "buildId" TEXT NOT NULL, + "buildToken" TEXT NOT NULL, + "status" "WorkerDeploymentStatus" NOT NULL DEFAULT 'PENDING', + "projectId" TEXT NOT NULL, + "environmentId" TEXT NOT NULL, + "workerId" TEXT, + "deployedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "WorkerDeployment_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "WorkerDeploymentPromotion" ( + "id" TEXT NOT NULL, + "label" TEXT NOT NULL, + "deploymentId" TEXT NOT NULL, + "environmentId" TEXT NOT NULL, + + CONSTRAINT "WorkerDeploymentPromotion_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerDeployment_friendlyId_key" ON "WorkerDeployment"("friendlyId"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerDeployment_workerId_key" ON "WorkerDeployment"("workerId"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerDeployment_projectId_shortCode_key" ON "WorkerDeployment"("projectId", "shortCode"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerDeployment_environmentId_version_key" ON "WorkerDeployment"("environmentId", "version"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkerDeploymentPromotion_environmentId_label_key" ON "WorkerDeploymentPromotion"("environmentId", "label"); + +-- CreateIndex +CREATE UNIQUE INDEX "ImageDetails_backgroundWorkerId_key" ON "ImageDetails"("backgroundWorkerId"); + +-- AddForeignKey +ALTER TABLE "WorkerDeployment" ADD CONSTRAINT "WorkerDeployment_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerDeployment" ADD CONSTRAINT "WorkerDeployment_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerDeployment" ADD CONSTRAINT "WorkerDeployment_workerId_fkey" FOREIGN KEY ("workerId") REFERENCES "BackgroundWorker"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerDeploymentPromotion" ADD CONSTRAINT "WorkerDeploymentPromotion_deploymentId_fkey" FOREIGN KEY ("deploymentId") REFERENCES "WorkerDeployment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "WorkerDeploymentPromotion" ADD CONSTRAINT "WorkerDeploymentPromotion_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql b/packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql new file mode 100644 index 000000000..1c02f70ab --- /dev/null +++ b/packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - You are about to drop the column `buildId` on the `WorkerDeployment` table. All the data in the column will be lost. + - You are about to drop the column `buildToken` on the `WorkerDeployment` table. All the data in the column will be lost. + +*/ +-- AlterEnum +ALTER TYPE "WorkerDeploymentStatus" ADD VALUE 'BUILDING'; + +-- AlterTable +ALTER TABLE "WorkerDeployment" DROP COLUMN "buildId", +DROP COLUMN "buildToken", +ADD COLUMN "externalBuildData" JSONB; diff --git a/packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql b/packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql new file mode 100644 index 000000000..96b59c3dd --- /dev/null +++ b/packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Added the required column `contentHash` to the `WorkerDeployment` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "WorkerDeployment" ADD COLUMN "contentHash" TEXT NOT NULL; diff --git a/packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql b/packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql new file mode 100644 index 000000000..683db3f26 --- /dev/null +++ b/packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql @@ -0,0 +1,20 @@ +/* + Warnings: + + - You are about to drop the `ImageDetails` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "ImageDetails" DROP CONSTRAINT "ImageDetails_backgroundWorkerId_fkey"; + +-- DropForeignKey +ALTER TABLE "ImageDetails" DROP CONSTRAINT "ImageDetails_projectId_fkey"; + +-- DropForeignKey +ALTER TABLE "ImageDetails" DROP CONSTRAINT "ImageDetails_runtimeEnvironmentId_fkey"; + +-- AlterTable +ALTER TABLE "WorkerDeployment" ADD COLUMN "imageReference" TEXT; + +-- DropTable +DROP TABLE "ImageDetails"; diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index ad45c12c4..db23db3f5 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -367,30 +367,31 @@ model RuntimeEnvironment { tunnelId String? - endpoints Endpoint[] - jobVersions JobVersion[] - events EventRecord[] - jobRuns JobRun[] - requestDeliveries HttpSourceRequestDelivery[] - jobAliases JobAlias[] - JobQueue JobQueue[] - sources TriggerSource[] - eventDispatchers EventDispatcher[] - scheduleSources ScheduleSource[] - ExternalAccount ExternalAccount[] - httpEndpointEnvironments TriggerHttpEndpointEnvironment[] - concurrencyLimitGroups ConcurrencyLimitGroup[] - keyValueItems KeyValueItem[] - webhookEnvironments WebhookEnvironment[] - webhookRequestDeliveries WebhookRequestDelivery[] - backgroundWorkers BackgroundWorker[] - backgroundWorkerTasks BackgroundWorkerTask[] - taskRuns TaskRun[] - taskQueues TaskQueue[] - batchTaskRuns BatchTaskRun[] - environmentVariableValues EnvironmentVariableValue[] - imageDetails ImageDetails[] - checkpoints Checkpoint[] + endpoints Endpoint[] + jobVersions JobVersion[] + events EventRecord[] + jobRuns JobRun[] + requestDeliveries HttpSourceRequestDelivery[] + jobAliases JobAlias[] + JobQueue JobQueue[] + sources TriggerSource[] + eventDispatchers EventDispatcher[] + scheduleSources ScheduleSource[] + ExternalAccount ExternalAccount[] + httpEndpointEnvironments TriggerHttpEndpointEnvironment[] + concurrencyLimitGroups ConcurrencyLimitGroup[] + keyValueItems KeyValueItem[] + webhookEnvironments WebhookEnvironment[] + webhookRequestDeliveries WebhookRequestDelivery[] + backgroundWorkers BackgroundWorker[] + backgroundWorkerTasks BackgroundWorkerTask[] + taskRuns TaskRun[] + taskQueues TaskQueue[] + batchTaskRuns BatchTaskRun[] + environmentVariableValues EnvironmentVariableValue[] + checkpoints Checkpoint[] + workerDeployments WorkerDeployment[] + workerDeploymentPromotions WorkerDeploymentPromotion[] @@unique([projectId, slug, orgMemberId]) @@unique([projectId, shortcode]) @@ -434,8 +435,8 @@ model Project { taskTags TaskTag[] taskQueues TaskQueue[] environmentVariables EnvironmentVariable[] - imageDetails ImageDetails[] checkpoints Checkpoint[] + WorkerDeployment WorkerDeployment[] } enum ProjectVersion { @@ -1517,10 +1518,11 @@ model BackgroundWorker { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - tasks BackgroundWorkerTask[] - attempts TaskRunAttempt[] - lockedRuns TaskRun[] - imageDetails ImageDetails[] + tasks BackgroundWorkerTask[] + attempts TaskRunAttempt[] + lockedRuns TaskRun[] + + deployment WorkerDeployment? @@unique([projectId, runtimeEnvironmentId, version]) } @@ -1892,31 +1894,6 @@ model EnvironmentVariableValue { @@unique([variableId, environmentId]) } -model ImageDetails { - id String @id @default(cuid()) - - friendlyId String @unique - - contentHash String - tag String - - backgroundWorker BackgroundWorker? @relation(fields: [backgroundWorkerId], references: [id], onDelete: Cascade, onUpdate: Cascade) - backgroundWorkerId String? - - project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) - projectId String - - runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) - runtimeEnvironmentId String - - metadata Json - - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - @@unique([projectId, runtimeEnvironmentId, contentHash]) -} - model Checkpoint { id String @id @default(cuid()) @@ -1943,3 +1920,65 @@ enum CheckpointType { DOCKER KUBERNETES } + +model WorkerDeployment { + id String @id @default(cuid()) + + contentHash String + friendlyId String @unique + shortCode String + version String + + imageReference String? + + externalBuildData Json? + + status WorkerDeploymentStatus @default(PENDING) + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + environment RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + environmentId String + + worker BackgroundWorker? @relation(fields: [workerId], references: [id], onDelete: Cascade, onUpdate: Cascade) + workerId String? @unique + + deployedAt DateTime? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + promotions WorkerDeploymentPromotion[] + + @@unique([projectId, shortCode]) + @@unique([environmentId, version]) +} + +enum WorkerDeploymentStatus { + PENDING + /// This is the status when the image is being built + BUILDING + /// This is the status when the image is built and we are waiting for the indexing to finish + DEPLOYING + /// This is the status when the image is built and indexed, meaning we have everything we need to deploy + DEPLOYED + FAILED + CANCELED +} + +model WorkerDeploymentPromotion { + id String @id @default(cuid()) + + /// This is the promotion label, e.g. "current" + label String + + deployment WorkerDeployment @relation(fields: [deploymentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + deploymentId String + + environment RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + environmentId String + + // Only one promotion per environment can be active at a time + @@unique([environmentId, label]) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 826d983dc..f8749dd72 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -153,6 +153,7 @@ importers: '@codemirror/view': ^6.5.0 '@conform-to/react': ^0.6.1 '@conform-to/zod': ^0.6.1 + '@depot/sdk-node': ^0.5.0 '@headlessui/react': ^1.7.8 '@heroicons/react': ^2.0.12 '@highlight-run/node': ^3.1.0 @@ -275,6 +276,7 @@ importers: ioredis: ^5.3.2 isbot: ^3.6.5 jsonpointer: ^5.0.1 + jwt-decode: ^4.0.0 lodash.omit: ^4.5.0 lucide-react: ^0.229.0 marked: ^4.0.18 @@ -348,6 +350,7 @@ importers: '@codemirror/view': 6.7.2 '@conform-to/react': 0.6.1_react@18.2.0 '@conform-to/zod': 0.6.1_zod@3.22.3 + '@depot/sdk-node': 0.5.0 '@headlessui/react': 1.7.8_biqbaboplfbrettd7655fr4n2y '@heroicons/react': 2.0.13_react@18.2.0 '@highlight-run/node': 3.1.0 @@ -419,6 +422,7 @@ importers: ioredis: 5.3.2 isbot: 3.6.5 jsonpointer: 5.0.1 + jwt-decode: 4.0.0 lodash.omit: 4.5.0 lucide-react: 0.229.0_react@18.2.0 marked: 4.2.5 @@ -553,7 +557,7 @@ importers: devDependencies: eslint: 8.31.0 eslint-config-prettier: 8.6.0_eslint@8.31.0 - eslint-config-turbo: 1.12.4_eslint@8.31.0 + eslint-config-turbo: 1.12.5_eslint@8.31.0 eslint-plugin-react: 7.31.8_eslint@8.31.0 typescript: 4.9.4 @@ -1028,6 +1032,7 @@ importers: packages/cli-v3: specifiers: '@clack/prompts': ^0.7.0 + '@depot/cli': 0.0.1-cli.2.55.0 '@opentelemetry/api': ^1.7.0 '@opentelemetry/api-logs': ^0.48.0 '@opentelemetry/auto-instrumentations-node': ^0.40.3 @@ -1094,6 +1099,7 @@ importers: source-map-support: ^0.5.21 supports-color: ^9.4.0 terminal-link: ^3.0.0 + tiny-invariant: ^1.2.0 tsup: ^8.0.1 type-fest: ^3.6.0 typescript: ^5.3.3 @@ -1105,6 +1111,7 @@ importers: zod: 3.22.3 dependencies: '@clack/prompts': 0.7.0 + '@depot/cli': 0.0.1-cli.2.55.0 '@opentelemetry/api': 1.7.0 '@opentelemetry/api-logs': 0.48.0 '@opentelemetry/auto-instrumentations-node': 0.40.3_rr4fxqkzz7nhz3auplkqn4p6em @@ -1156,6 +1163,7 @@ importers: source-map-support: 0.5.21 supports-color: 9.4.0 terminal-link: 3.0.0 + tiny-invariant: 1.3.1 update-check: 1.5.4 url: 0.11.1 ws: 8.12.0 @@ -5582,6 +5590,10 @@ packages: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} dev: true + /@bufbuild/protobuf/1.7.2: + resolution: {integrity: sha512-i5GE2Dk5ekdlK1TR7SugY4LWRrKSfb5T1Qn4unpIMbfxoeGKERKQ59HG3iYewacGD10SR7UzevfPnh6my4tNmQ==} + dev: false + /@bundled-es-modules/cookie/2.0.0: resolution: {integrity: sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==} dependencies: @@ -6003,12 +6015,137 @@ packages: zod: 3.22.3 dev: false + /@connectrpc/connect-node/0.13.2_@bufbuild+protobuf@1.7.2: + resolution: {integrity: sha512-dAoBuQ+fYFw22KYgZXPw1t19i6TRwNmk6c6XsOYLhBl1Y5vzXoMfMX2jm7lmkmLQFIcM9Xc2EYoUKtjNXH5bjw==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@bufbuild/protobuf': ^1.2.1 + dependencies: + '@bufbuild/protobuf': 1.7.2 + '@connectrpc/connect': 0.13.2_@bufbuild+protobuf@1.7.2 + undici: 5.25.4 + dev: false + + /@connectrpc/connect/0.13.2_@bufbuild+protobuf@1.7.2: + resolution: {integrity: sha512-KZg6EH8gYnQZm/d2IXXMVB2mom/A1dCD8+7JScm2tKN9OQyQSeCJOLqtv/M4M5XVS0Y9JM2/VCbiUwfhl9Rqmg==} + peerDependencies: + '@bufbuild/protobuf': ^1.2.1 + dependencies: + '@bufbuild/protobuf': 1.7.2 + dev: false + /@cspotcode/source-map-support/0.8.1: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} dependencies: '@jridgewell/trace-mapping': 0.3.9 + /@depot/cli-darwin-arm64/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-1uRGmCqd+kuHK2fub0oVt+SctP3mwNX15y17oMilIOpvvJtwMvjwBI73qt2aN0jbx0e3xrDiv5GD4GMzsqNGEw==} + engines: {node: '>=14'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-darwin-x64/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-3Z3fczSJ1WsxhgS0Ui//9CIk6yJS2+JKlwmZ44EbQIPQ4ZyR+ZZDoJa8pmZdLDehRxVLjeXkUdsRPD2K6yrCqA==} + engines: {node: '>=14'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-linux-arm/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-T89KJJjk9By50L6pCrDDAmJFgM3V8DzppMmM7cA1g3biLTjeJZLYgwTrBXu3CfpWMqqLWWOvZ/je8myx7PwBOA==} + engines: {node: '>=14'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-linux-arm64/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-GsnItZk9jawX4f6xNI7d0FveSLNIWNVo7udzmtYmPkGo0hsiNfOxzQixR8cusJKnTOo9wYiHx4pfYNB3Hjs7Ng==} + engines: {node: '>=14'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-linux-ia32/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-Eunct4bkpN1TVhMt8xu9XtURzipz9yEfXidQoX/vijMLZemsvwcMVnuCmUkfyq/m5DJ1ZN8APHvj+9NMrERfaQ==} + engines: {node: '>=14'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-linux-x64/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-G93+XCpzGHYfH5LeT+3A0J3SF7GGQGnjMVoH8w7m5b6GtTu7SFGfat3a2/M07SOtGXa5tz3O56wrgR1jDPc1Fg==} + engines: {node: '>=14'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-win32-arm64/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-y7K48ngTqvSCyHItxW0kapVfYyqtWNYZVeEJ344tSuvq6H3bt8mLWfjH+WF9fT14OAmR1jOt0+ykpta45ehKrA==} + engines: {node: '>=14'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-win32-ia32/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-hsO3NKWZMeiFgtFAkEsj1vrHG49N0fLjkhEEJ7DrUV0OBMUifkScE66E+9+pWJoxWq3B3Bmc/9OPc0XVMfYekQ==} + engines: {node: '>=14'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@depot/cli-win32-x64/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-QKhpSpa1QwWTf79iMNxUR17EFbe7hT/Bs9mhTcR6NiyOB0GeWBpX3ejULpQcfHpVjBBHIXTW/TJ/Unx4N80cIA==} + engines: {node: '>=14'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@depot/cli/0.0.1-cli.2.55.0: + resolution: {integrity: sha512-Y5hlyMxJ8wKeXQKQaYOMCVVzLO42qFHaybcgmCjiZC4CSXkX0+nOL8daoH3Sy0M4U0RcbUijNMJ51oGFq2As9g==} + engines: {node: '>=14'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@depot/cli-darwin-arm64': 0.0.1-cli.2.55.0 + '@depot/cli-darwin-x64': 0.0.1-cli.2.55.0 + '@depot/cli-linux-arm': 0.0.1-cli.2.55.0 + '@depot/cli-linux-arm64': 0.0.1-cli.2.55.0 + '@depot/cli-linux-ia32': 0.0.1-cli.2.55.0 + '@depot/cli-linux-x64': 0.0.1-cli.2.55.0 + '@depot/cli-win32-arm64': 0.0.1-cli.2.55.0 + '@depot/cli-win32-ia32': 0.0.1-cli.2.55.0 + '@depot/cli-win32-x64': 0.0.1-cli.2.55.0 + dev: false + + /@depot/sdk-node/0.5.0: + resolution: {integrity: sha512-Pl9Yji00B6uQpMm5xZU4/eZbzaBo72N6c534ZqLI6nbOEIPH0VX27TVF9lFF3Fl3GHFZ+6X9laJtDn2DrLS2DQ==} + dependencies: + '@bufbuild/protobuf': 1.7.2 + '@connectrpc/connect': 0.13.2_@bufbuild+protobuf@1.7.2 + '@connectrpc/connect-node': 0.13.2_@bufbuild+protobuf@1.7.2 + dev: false + /@discoveryjs/json-ext/0.5.7: resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} engines: {node: '>=10.0.0'} @@ -23839,13 +23976,13 @@ packages: eslint: 8.45.0 dev: true - /eslint-config-turbo/1.12.4_eslint@8.31.0: - resolution: {integrity: sha512-5hqEaV6PNmAYLL4RTmq74OcCt8pgzOLnfDVPG/7PUXpQ0Mpz0gr926oCSFukywKKXjdum3VHD84S7Z9A/DqTAw==} + /eslint-config-turbo/1.12.5_eslint@8.31.0: + resolution: {integrity: sha512-wXytbX+vTzQ6rwgM6sIr447tjYJBlRj5V/eBFNGNXw5Xs1R715ppPYhbmxaFbkrWNQSGJsWRrYGAlyq0sT/OsQ==} peerDependencies: eslint: '>6.6.0' dependencies: eslint: 8.31.0 - eslint-plugin-turbo: 1.12.4_eslint@8.31.0 + eslint-plugin-turbo: 1.12.5_eslint@8.31.0 dev: true /eslint-doc-generator/1.4.3_eslint@8.45.0: @@ -24548,8 +24685,8 @@ packages: - typescript dev: true - /eslint-plugin-turbo/1.12.4_eslint@8.31.0: - resolution: {integrity: sha512-3AGmXvH7E4i/XTWqBrcgu+G7YKZJV/8FrEn79kTd50ilNsv+U3nS2IlcCrQB6Xm2m9avGD9cadLzKDR1/UF2+g==} + /eslint-plugin-turbo/1.12.5_eslint@8.31.0: + resolution: {integrity: sha512-cXy7mCzAdngBTJIWH4DASXHy0vQpujWDBqRTu0YYqCN/QEGsi3HWM+STZEbPYELdjtm5EsN2HshOSSqWnjdRHg==} peerDependencies: eslint: '>6.6.0' dependencies: @@ -28760,6 +28897,11 @@ packages: safe-buffer: 5.2.1 dev: false + /jwt-decode/4.0.0: + resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==} + engines: {node: '>=18'} + dev: false + /keyv/3.1.0: resolution: {integrity: sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==} dependencies: @@ -36854,7 +36996,7 @@ packages: dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 29.6.2_@types+node@18.17.1 + jest: 29.6.2_@types+node@18.15.13 jest-util: 29.6.2 json5: 2.2.3 lodash.memoize: 4.1.2 diff --git a/references/v3-catalog/tsconfig.json b/references/v3-catalog/tsconfig.json index 509f66c9c..d09e514c3 100644 --- a/references/v3-catalog/tsconfig.json +++ b/references/v3-catalog/tsconfig.json @@ -9,7 +9,9 @@ "@trigger.dev/sdk": ["../../packages/trigger-sdk/src/index"], "@trigger.dev/sdk/*": ["../../packages/trigger-sdk/src/*"], "@trigger.dev/core": ["../../packages/core/src/index"], - "@trigger.dev/core/*": ["../../packages/core/src/*"] + "@trigger.dev/core/*": ["../../packages/core/src/*"], + "@trigger.dev/core-apps": ["../../packages/core-apps/src/index"], + "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"] } } }