v3: cli deploy command (#931)
* WIP proxy/deploy * WIP, registry proxy in express working * A couple of notes, preparing for indexing * Move the changes to prod-worker into the new file * Deploy command working with indexing and runs (docker provider only for now) * Removed ts-expect-error directive * Fixed build command
This commit is contained in:
+9
-1
@@ -49,4 +49,12 @@ CLOUD_SLACK_CLIENT_SECRET=
|
||||
|
||||
# v3 variables
|
||||
PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32`
|
||||
COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`
|
||||
COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`
|
||||
|
||||
# Uncomment the following line to enable the registry proxy
|
||||
# ENABLE_REGISTRY_PROXY=true
|
||||
# DEPOT_TOKEN=<Depot org token>
|
||||
# DEPOT_PROJECT_ID=<Depot project id>
|
||||
# CONTAINER_REGISTRY_ORIGIN=<Container registry origin e.g. https://registry.digitalocean.com>
|
||||
# CONTAINER_REGISTRY_USERNAME=<Container registry username e.g. Digital ocean email address>
|
||||
# CONTAINER_REGISTRY_PASSWORD=<Container registry password e.g. Digital ocean PAT>
|
||||
Vendored
+8
@@ -37,6 +37,14 @@
|
||||
"cwd": "${workspaceFolder}/references/v3-catalog",
|
||||
"sourceMaps": true
|
||||
},
|
||||
{
|
||||
"type": "node-terminal",
|
||||
"request": "launch",
|
||||
"name": "Debug V3 Deploy CLI",
|
||||
"command": "pnpm exec trigger.dev deploy",
|
||||
"cwd": "${workspaceFolder}/references/v3-catalog",
|
||||
"sourceMaps": true
|
||||
},
|
||||
{
|
||||
"type": "node",
|
||||
"request": "attach",
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
HTTP_SERVER_PORT=8020
|
||||
|
||||
PLATFORM_ENABLED=true
|
||||
PLATFORM_WS_PORT=3030
|
||||
@@ -286,10 +286,10 @@ class TaskCoordinator {
|
||||
try {
|
||||
setSocketDataFromHeader("podName", "x-pod-name");
|
||||
setSocketDataFromHeader("contentHash", "x-trigger-content-hash");
|
||||
setSocketDataFromHeader("cliPackageVersion", "x-trigger-cli-package-version");
|
||||
setSocketDataFromHeader("projectRef", "x-trigger-project-ref");
|
||||
setSocketDataFromHeader("attemptId", "x-trigger-attempt-id");
|
||||
setSocketDataFromHeader("envId", "x-trigger-env-id");
|
||||
setSocketDataFromHeader("deploymentId", "x-trigger-deployment-id");
|
||||
} catch (error) {
|
||||
logger(error);
|
||||
socket.disconnect(true);
|
||||
@@ -390,8 +390,8 @@ class TaskCoordinator {
|
||||
version: "v1",
|
||||
projectRef: socket.data.projectRef,
|
||||
envId: socket.data.envId,
|
||||
deploymentId: message.deploymentId,
|
||||
metadata: {
|
||||
cliPackageVersion: socket.data.cliPackageVersion,
|
||||
contentHash: socket.data.contentHash,
|
||||
packageVersion: message.packageVersion,
|
||||
tasks: message.tasks,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
{
|
||||
"include": ["./src/**/*.ts"],
|
||||
"exclude": ["node_modules"],
|
||||
"compilerOptions": {
|
||||
"target": "es2016",
|
||||
"module": "commonjs",
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
HTTP_SERVER_PORT=8050
|
||||
|
||||
PLATFORM_WS_PORT=3030
|
||||
PLATFORM_SECRET=provider-secret
|
||||
PLATFORM_SECRET=provider-secret
|
||||
# Use this if you are on macOS
|
||||
# COORDINATOR_HOST="host.docker.internal"
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
|
||||
@@ -4,15 +4,29 @@ import { SimpleLogger, TaskOperations, ProviderShell } from "@trigger.dev/core-a
|
||||
|
||||
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
||||
const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
|
||||
const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
||||
const OTEL_EXPORTER_OTLP_ENDPOINT =
|
||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
|
||||
|
||||
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
|
||||
|
||||
class DockerTaskOperations implements TaskOperations {
|
||||
async index(opts: { contentHash: string; imageTag: string; envId: string }) {
|
||||
async index(opts: {
|
||||
contentHash: string;
|
||||
imageTag: string;
|
||||
envId: string;
|
||||
apiKey: string;
|
||||
apiUrl: string;
|
||||
}) {
|
||||
const containerName = this.#getIndexContainerName(opts.contentHash);
|
||||
|
||||
logger.log(`Indexing task ${opts.imageTag}`, {
|
||||
host: COORDINATOR_HOST,
|
||||
port: COORDINATOR_PORT,
|
||||
});
|
||||
|
||||
const { exitCode } = logger.debug(
|
||||
await $`docker run --rm -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e INDEX_TASKS=true --network=host --pull=never --name=${containerName} ${opts.imageTag}`
|
||||
await $`docker run --rm -e TRIGGER_SECRET_KEY=${opts.apiKey} -e TRIGGER_API_URL=${opts.apiUrl} -e COORDINATOR_HOST=${COORDINATOR_HOST} -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e INDEX_TASKS=true --name=${containerName} ${opts.imageTag}`
|
||||
);
|
||||
|
||||
if (exitCode !== 0) {
|
||||
@@ -24,7 +38,7 @@ class DockerTaskOperations implements TaskOperations {
|
||||
const containerName = this.#getRunContainerName(opts.attemptId);
|
||||
|
||||
const { exitCode } = logger.debug(
|
||||
await $`docker run -d -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e TRIGGER_ATTEMPT_ID=${opts.attemptId} --network=host --pull=never --name=${containerName} ${opts.image}`
|
||||
await $`docker run -d -e OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT} -e COORDINATOR_HOST=${COORDINATOR_HOST} -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e TRIGGER_ATTEMPT_ID=${opts.attemptId} --name=${containerName} ${opts.image}`
|
||||
);
|
||||
|
||||
if (exitCode !== 0) {
|
||||
|
||||
@@ -197,3 +197,4 @@ const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer
|
||||
|
||||
export { wss } from "./v3/handleWebsockets.server";
|
||||
export { socketIo } from "./v3/handleSocketIo.server";
|
||||
export { registryProxy } from "./v3/registryProxy.server";
|
||||
|
||||
@@ -76,6 +76,11 @@ const EnvironmentSchema = z.object({
|
||||
IMAGE_REPO: z.string().default("task"),
|
||||
PROVIDER_SECRET: z.string().default("provider-secret"),
|
||||
COORDINATOR_SECRET: z.string().default("coordinator-secret"),
|
||||
DEPOT_TOKEN: z.string().optional(),
|
||||
DEPOT_PROJECT_ID: z.string().optional(),
|
||||
CONTAINER_REGISTRY_ORIGIN: z.string().optional(),
|
||||
CONTAINER_REGISTRY_USERNAME: z.string().optional(),
|
||||
CONTAINER_REGISTRY_PASSWORD: z.string().optional(),
|
||||
});
|
||||
|
||||
export type Environment = z.infer<typeof EnvironmentSchema>;
|
||||
|
||||
+9
-9
@@ -1,12 +1,12 @@
|
||||
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
|
||||
import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3";
|
||||
import { StartDeploymentIndexingRequestBody } from "@trigger.dev/core/v3";
|
||||
import { z } from "zod";
|
||||
import { authenticateApiRequest } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { CreateImageDetailsService } from "~/v3/services/createImageDetails.server";
|
||||
import { StartDeploymentIndexing } from "~/v3/services/startDeploymentIndexing.server";
|
||||
|
||||
const ParamsSchema = z.object({
|
||||
projectRef: z.string(),
|
||||
deploymentId: z.string(),
|
||||
});
|
||||
|
||||
export async function action({ request, params }: ActionFunctionArgs) {
|
||||
@@ -31,23 +31,23 @@ export async function action({ request, params }: ActionFunctionArgs) {
|
||||
|
||||
const authenticatedEnv = authenticationResult.environment;
|
||||
|
||||
const { projectRef } = parsedParams.data;
|
||||
const { deploymentId } = parsedParams.data;
|
||||
|
||||
const rawBody = await request.json();
|
||||
const body = CreateImageDetailsRequestBody.safeParse(rawBody);
|
||||
const body = StartDeploymentIndexingRequestBody.safeParse(rawBody);
|
||||
|
||||
if (!body.success) {
|
||||
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
|
||||
}
|
||||
|
||||
const service = new CreateImageDetailsService();
|
||||
const service = new StartDeploymentIndexing();
|
||||
|
||||
const imageDetails = await service.call(projectRef, authenticatedEnv, body.data);
|
||||
const deployment = await service.call(authenticatedEnv, deploymentId, body.data);
|
||||
|
||||
return json(
|
||||
{
|
||||
id: imageDetails.friendlyId,
|
||||
contentHash: imageDetails.contentHash,
|
||||
id: deployment.friendlyId,
|
||||
contentHash: deployment.contentHash,
|
||||
},
|
||||
{ status: 200 }
|
||||
);
|
||||
@@ -0,0 +1,68 @@
|
||||
import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
|
||||
import { z } from "zod";
|
||||
import { prisma } from "~/db.server";
|
||||
import { authenticateApiRequest } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
|
||||
const ParamsSchema = z.object({
|
||||
deploymentId: z.string(),
|
||||
});
|
||||
|
||||
export async function loader({ request, params }: LoaderFunctionArgs) {
|
||||
const parsedParams = ParamsSchema.safeParse(params);
|
||||
|
||||
if (!parsedParams.success) {
|
||||
return json({ error: "Invalid params" }, { status: 400 });
|
||||
}
|
||||
|
||||
// Next authenticate the request
|
||||
const authenticationResult = await authenticateApiRequest(request);
|
||||
|
||||
if (!authenticationResult) {
|
||||
logger.info("Invalid or missing api key", { url: request.url });
|
||||
return json({ error: "Invalid or Missing API key" }, { status: 401 });
|
||||
}
|
||||
|
||||
const authenticatedEnv = authenticationResult.environment;
|
||||
|
||||
const { deploymentId } = parsedParams.data;
|
||||
|
||||
const deployment = await prisma.workerDeployment.findUnique({
|
||||
where: {
|
||||
friendlyId: deploymentId,
|
||||
environmentId: authenticatedEnv.id,
|
||||
},
|
||||
include: {
|
||||
worker: {
|
||||
include: {
|
||||
tasks: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
return json({ error: "Deployment not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
return json({
|
||||
id: deployment.friendlyId,
|
||||
status: deployment.status,
|
||||
contentHash: deployment.contentHash,
|
||||
shortCode: deployment.shortCode,
|
||||
version: deployment.version,
|
||||
imageReference: deployment.imageReference,
|
||||
worker: deployment.worker
|
||||
? {
|
||||
id: deployment.worker.friendlyId,
|
||||
version: deployment.worker.version,
|
||||
tasks: deployment.worker.tasks.map((task) => ({
|
||||
id: task.friendlyId,
|
||||
slug: task.slug,
|
||||
filePath: task.filePath,
|
||||
exportName: task.exportName,
|
||||
})),
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
|
||||
import { InitializeDeploymentRequestBody } from "@trigger.dev/core/v3";
|
||||
import { authenticateApiRequest } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { InitializeDeploymentService } from "~/v3/services/initializeDeployment.server";
|
||||
|
||||
export async function action({ request, params }: ActionFunctionArgs) {
|
||||
// Ensure this is a POST request
|
||||
if (request.method.toUpperCase() !== "POST") {
|
||||
return { status: 405, body: "Method Not Allowed" };
|
||||
}
|
||||
|
||||
// Next authenticate the request
|
||||
const authenticationResult = await authenticateApiRequest(request);
|
||||
|
||||
if (!authenticationResult) {
|
||||
logger.info("Invalid or missing api key", { url: request.url });
|
||||
return json({ error: "Invalid or Missing API key" }, { status: 401 });
|
||||
}
|
||||
|
||||
const rawBody = await request.json();
|
||||
const body = InitializeDeploymentRequestBody.safeParse(rawBody);
|
||||
|
||||
if (!body.success) {
|
||||
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
|
||||
}
|
||||
|
||||
const authenticatedEnv = authenticationResult.environment;
|
||||
|
||||
const service = new InitializeDeploymentService();
|
||||
|
||||
const { deployment, imageTag } = await service.call(authenticatedEnv, body.data);
|
||||
|
||||
return json(
|
||||
{
|
||||
id: deployment.friendlyId,
|
||||
contentHash: deployment.contentHash,
|
||||
shortCode: deployment.shortCode,
|
||||
version: deployment.version,
|
||||
externalBuildData: deployment.externalBuildData,
|
||||
imageTag,
|
||||
},
|
||||
{ status: 200 }
|
||||
);
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { json } from "@remix-run/server-runtime";
|
||||
import { GetProjectDevResponse } from "@trigger.dev/core/v3";
|
||||
import { z } from "zod";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
|
||||
|
||||
@@ -58,6 +59,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
|
||||
const result: GetProjectDevResponse = {
|
||||
apiKey: devEnvironment.apiKey,
|
||||
name: project.name,
|
||||
apiUrl: env.APP_ORIGIN,
|
||||
};
|
||||
|
||||
return json(result);
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
|
||||
import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
|
||||
import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
|
||||
import { z } from "zod";
|
||||
import { prisma } from "~/db.server";
|
||||
import { authenticateApiRequest } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
|
||||
import { CreateBackgroundWorkerService } from "~/v3/services/createBackgroundWorker.server";
|
||||
|
||||
const ParamsSchema = z.object({
|
||||
projectRef: z.string(),
|
||||
|
||||
@@ -3,6 +3,7 @@ import { json } from "@remix-run/server-runtime";
|
||||
import { GetProjectDevResponse } from "@trigger.dev/core/v3";
|
||||
import { z } from "zod";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
|
||||
|
||||
@@ -56,6 +57,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
|
||||
const result: GetProjectDevResponse = {
|
||||
apiKey: prodEnvironment.apiKey,
|
||||
name: project.name,
|
||||
apiUrl: env.APP_ORIGIN,
|
||||
};
|
||||
|
||||
return json(result);
|
||||
|
||||
@@ -4,32 +4,32 @@ import { z } from "zod";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { ZodWorker } from "~/platform/zodWorker.server";
|
||||
import { IndexDeploymentService } from "~/v3/services/indexDeployment.server";
|
||||
import { ExpireDispatcherService } from "./dispatchers/expireDispatcher.server";
|
||||
import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralEventDispatcher.server";
|
||||
import { sendEmail } from "./email.server";
|
||||
import { IndexEndpointService } from "./endpoints/indexEndpoint.server";
|
||||
import { PerformEndpointIndexService } from "./endpoints/performEndpointIndexService";
|
||||
import { ProbeEndpointService } from "./endpoints/probeEndpoint.server";
|
||||
import { RecurringEndpointIndexService } from "./endpoints/recurringEndpointIndex.server";
|
||||
import { DeliverEventService } from "./events/deliverEvent.server";
|
||||
import { InvokeDispatcherService } from "./events/invokeDispatcher.server";
|
||||
import { integrationAuthRepository } from "./externalApis/integrationAuthRepository.server";
|
||||
import { IntegrationConnectionCreatedService } from "./externalApis/integrationConnectionCreated.server";
|
||||
import { executionRateLimiter } from "./runExecutionRateLimiter.server";
|
||||
import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server";
|
||||
import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server";
|
||||
import { MissingConnectionCreatedService } from "./runs/missingConnectionCreated.server";
|
||||
import { PerformRunExecutionV3Service } from "./runs/performRunExecutionV3.server";
|
||||
import { ResumeRunService } from "./runs/resumeRun.server";
|
||||
import { StartRunService } from "./runs/startRun.server";
|
||||
import { DeliverScheduledEventService } from "./schedules/deliverScheduledEvent.server";
|
||||
import { ActivateSourceService } from "./sources/activateSource.server";
|
||||
import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server";
|
||||
import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server";
|
||||
import { PerformTaskOperationService } from "./tasks/performTaskOperation.server";
|
||||
import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout.server";
|
||||
import { ProbeEndpointService } from "./endpoints/probeEndpoint.server";
|
||||
import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server";
|
||||
import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server";
|
||||
import { ResumeTaskService } from "./tasks/resumeTask.server";
|
||||
import { ExpireDispatcherService } from "./dispatchers/expireDispatcher.server";
|
||||
import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralEventDispatcher.server";
|
||||
import { ResumeRunService } from "./runs/resumeRun.server";
|
||||
import { executionRateLimiter } from "./runExecutionRateLimiter.server";
|
||||
import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server";
|
||||
import { IndexTasksService } from "~/v3/services/indexTasks.server";
|
||||
|
||||
const workerCatalog = {
|
||||
indexEndpoint: z.object({
|
||||
@@ -104,7 +104,7 @@ const workerCatalog = {
|
||||
id: z.string(),
|
||||
}),
|
||||
// v3 tasks
|
||||
indexTasks: z.object({
|
||||
"v3.indexDeployment": z.object({
|
||||
id: z.string(),
|
||||
}),
|
||||
};
|
||||
@@ -434,11 +434,11 @@ function getWorkerQueue() {
|
||||
},
|
||||
},
|
||||
// v3 tasks
|
||||
indexTasks: {
|
||||
"v3.indexDeployment": {
|
||||
priority: 0,
|
||||
maxAttempts: 5,
|
||||
handler: async (payload, job) => {
|
||||
const service = new IndexTasksService();
|
||||
const service = new IndexDeploymentService();
|
||||
|
||||
return await service.call(payload.id);
|
||||
},
|
||||
@@ -538,4 +538,4 @@ function getTaskOperationWorkerQueue() {
|
||||
});
|
||||
}
|
||||
|
||||
export { executionWorker, workerQueue, taskOperationWorker };
|
||||
export { executionWorker, taskOperationWorker, workerQueue };
|
||||
|
||||
@@ -17,6 +17,7 @@ import { CompleteAttemptService } from "./services/completeAttempt.server";
|
||||
import { CreateBackgroundWorkerService } from "./services/createBackgroundWorker.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import { CreateDeployedBackgroundWorkerService } from "./services/createDeployedBackgroundWorker.server";
|
||||
|
||||
export const socketIo = singleton("socketIo", initalizeIoServer);
|
||||
|
||||
@@ -76,13 +77,13 @@ function createCoordinatorNamespace(io: Server) {
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
const createCheckpoint = new CreateBackgroundWorkerService();
|
||||
await createCheckpoint.call(message.projectRef, environment, {
|
||||
localOnly: true,
|
||||
const service = new CreateDeployedBackgroundWorkerService();
|
||||
const worker = await service.call(message.projectRef, environment, message.deploymentId, {
|
||||
localOnly: false,
|
||||
metadata: message.metadata,
|
||||
});
|
||||
|
||||
return { success: true };
|
||||
return { success: !!worker };
|
||||
} catch (error) {
|
||||
logger.error("Error while creating worker", { error });
|
||||
return { success: false };
|
||||
|
||||
@@ -291,25 +291,29 @@ export class SharedQueueConsumer {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundWorker = await prisma.backgroundWorker.findFirst({
|
||||
const deployment = await prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
runtimeEnvironmentId: existingTaskRun.runtimeEnvironmentId,
|
||||
environmentId: existingTaskRun.runtimeEnvironmentId,
|
||||
projectId: existingTaskRun.projectId,
|
||||
imageDetails: {
|
||||
some: {},
|
||||
status: "DEPLOYED",
|
||||
imageReference: {
|
||||
not: null,
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
updatedAt: "desc",
|
||||
},
|
||||
include: {
|
||||
tasks: true,
|
||||
imageDetails: true,
|
||||
worker: {
|
||||
include: {
|
||||
tasks: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!backgroundWorker) {
|
||||
logger.error("No matching background worker found for task run", {
|
||||
if (!deployment || !deployment.worker) {
|
||||
logger.error("No matching deployment found for task run", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
});
|
||||
@@ -318,7 +322,18 @@ export class SharedQueueConsumer {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundTask = backgroundWorker.tasks.find(
|
||||
if (!deployment.imageReference) {
|
||||
logger.error("Deployment is missing an image reference", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
deployment: deployment.id,
|
||||
});
|
||||
await marqs?.acknowledgeMessage(message.messageId);
|
||||
setTimeout(() => this.#doWork(), 100);
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundTask = deployment.worker.tasks.find(
|
||||
(task) => task.slug === existingTaskRun.taskIdentifier
|
||||
);
|
||||
|
||||
@@ -326,8 +341,9 @@ export class SharedQueueConsumer {
|
||||
logger.warn("No matching background task found for task run", {
|
||||
taskRun: existingTaskRun.id,
|
||||
taskIdentifier: existingTaskRun.taskIdentifier,
|
||||
backgroundWorker: backgroundWorker.id,
|
||||
taskSlugs: backgroundWorker.tasks.map((task) => task.slug),
|
||||
deployment: deployment.id,
|
||||
backgroundWorker: deployment.worker.id,
|
||||
taskSlugs: deployment.worker.tasks.map((task) => task.slug),
|
||||
});
|
||||
|
||||
await marqs?.acknowledgeMessage(message.messageId);
|
||||
@@ -357,7 +373,8 @@ export class SharedQueueConsumer {
|
||||
logger.warn("Failed to lock task run", {
|
||||
taskRun: existingTaskRun.id,
|
||||
taskIdentifier: existingTaskRun.taskIdentifier,
|
||||
backgroundWorker: backgroundWorker.id,
|
||||
deployment: deployment.id,
|
||||
backgroundWorker: deployment.worker.id,
|
||||
messageId: message.messageId,
|
||||
});
|
||||
|
||||
@@ -402,11 +419,11 @@ export class SharedQueueConsumer {
|
||||
|
||||
try {
|
||||
await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId: backgroundWorker.friendlyId,
|
||||
backgroundWorkerId: deployment.worker.friendlyId,
|
||||
data: {
|
||||
type: "SCHEDULE_ATTEMPT",
|
||||
id: taskRunAttempt.id,
|
||||
image: backgroundWorker.imageDetails[0].tag,
|
||||
image: deployment.imageReference,
|
||||
envId: environment.id,
|
||||
},
|
||||
});
|
||||
@@ -492,30 +509,29 @@ export class SharedQueueConsumer {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundWorker = await prisma.backgroundWorker.findFirst({
|
||||
const deployment = await prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
runtimeEnvironmentId: resumableRun.runtimeEnvironmentId,
|
||||
environmentId: resumableRun.runtimeEnvironmentId,
|
||||
projectId: resumableRun.projectId,
|
||||
imageDetails: {
|
||||
some: {},
|
||||
status: "DEPLOYED",
|
||||
imageReference: {
|
||||
not: null,
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
include: {
|
||||
tasks: true,
|
||||
imageDetails: {
|
||||
take: 1,
|
||||
orderBy: {
|
||||
updatedAt: "desc",
|
||||
worker: {
|
||||
include: {
|
||||
tasks: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!backgroundWorker) {
|
||||
logger.error("No matching background worker found for task run", {
|
||||
if (!deployment || !deployment.worker) {
|
||||
logger.error("No matching deployment found for task run", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
});
|
||||
@@ -524,7 +540,18 @@ export class SharedQueueConsumer {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundTask = backgroundWorker.tasks.find(
|
||||
if (!deployment.imageReference) {
|
||||
logger.error("Deployment is missing an image reference", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
deployment: deployment.id,
|
||||
});
|
||||
await marqs?.acknowledgeMessage(message.messageId);
|
||||
setTimeout(() => this.#doWork(), 100);
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundTask = deployment.worker.tasks.find(
|
||||
(task) => task.slug === resumableRun.taskIdentifier
|
||||
);
|
||||
|
||||
@@ -532,8 +559,9 @@ export class SharedQueueConsumer {
|
||||
logger.warn("No matching background task found for task run", {
|
||||
taskRun: resumableRun.id,
|
||||
taskIdentifier: resumableRun.taskIdentifier,
|
||||
backgroundWorker: backgroundWorker.id,
|
||||
taskSlugs: backgroundWorker.tasks.map((task) => task.slug),
|
||||
deployment: deployment.id,
|
||||
backgroundWorker: deployment.worker.id,
|
||||
taskSlugs: deployment.worker.tasks.map((task) => task.slug),
|
||||
});
|
||||
|
||||
await marqs?.acknowledgeMessage(message.messageId);
|
||||
@@ -623,7 +651,7 @@ export class SharedQueueConsumer {
|
||||
socketIo.coordinatorNamespace.emit("RESUME", {
|
||||
version: "v1",
|
||||
attemptId: resumableAttempt.id,
|
||||
image: backgroundWorker.imageDetails[0].tag,
|
||||
image: deployment.imageReference,
|
||||
completions,
|
||||
executions,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
import { IncomingMessage, ServerResponse } from "http";
|
||||
import Redis, { RedisOptions } from "ioredis";
|
||||
import { RequestOptions, request as httpRequest } from "node:https";
|
||||
import { z } from "zod";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { authenticatePersonalAccessToken } from "~/services/personalAccessToken.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { jwtDecode } from "jwt-decode";
|
||||
import { createHash } from "node:crypto";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mkdtemp } from "fs/promises";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { unlinkSync } from "fs";
|
||||
|
||||
const TokenResponseBody = z.object({
|
||||
token: z.string(),
|
||||
});
|
||||
|
||||
const CACHED_BEARER_TOKEN_BUFFER_IN_SECONDS = 10;
|
||||
|
||||
type RegistryProxyOptions = {
|
||||
origin: string;
|
||||
auth: { username: string; password: string };
|
||||
redis?: RedisOptions;
|
||||
};
|
||||
|
||||
export class RegistryProxy {
|
||||
private redis?: Redis;
|
||||
|
||||
constructor(private readonly options: RegistryProxyOptions) {
|
||||
if (options.redis) {
|
||||
this.redis = new Redis(options.redis);
|
||||
}
|
||||
}
|
||||
|
||||
get origin() {
|
||||
return this.options.origin;
|
||||
}
|
||||
|
||||
get host() {
|
||||
return new URL(this.options.origin).host;
|
||||
}
|
||||
|
||||
// If the imageReference includes a hostname, rewrite it to point to the proxy
|
||||
// e.g. eric-webapp.trigger.dev/trigger/yubjwjsfkxnylobaqvqz:20240306.41.prod@sha256:8b48dd2866bc8878644d2880bbe35a27e66cf6ff78aa1e489d7fdde5e228faf1
|
||||
// should be rewritten to ${this.host}/trigger/yubjwjsfkxnylobaqvqz:20240306.41.prod@sha256:8b48dd2866bc8878644d2880bbe35a27e66cf6ff78aa1e489d7fdde5e228faf1
|
||||
// This will work with image references that don't include the @sha256:... part
|
||||
public rewriteImageReference(imageReference: string) {
|
||||
const parts = parseDockerImageReference(imageReference);
|
||||
|
||||
if (parts.registry) {
|
||||
return rebuildDockerImageReference({
|
||||
...parts,
|
||||
registry: this.host,
|
||||
});
|
||||
}
|
||||
|
||||
return imageReference;
|
||||
}
|
||||
|
||||
public async call(request: IncomingMessage, response: ServerResponse) {
|
||||
await this.#proxyRequest(request, response);
|
||||
}
|
||||
|
||||
// Proxies the request to the registry
|
||||
async #proxyRequest(request: IncomingMessage, response: ServerResponse) {
|
||||
const credentials = this.#getBasicAuthCredentials(request);
|
||||
|
||||
if (!credentials) {
|
||||
logger.debug("Returning 401 because credentials are missing");
|
||||
|
||||
response.writeHead(401, {
|
||||
"WWW-Authenticate": 'Basic realm="Access to the registry"',
|
||||
});
|
||||
|
||||
response.end("Unauthorized");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Authenticate the request
|
||||
const authentication = await authenticatePersonalAccessToken(credentials.password);
|
||||
|
||||
if (!authentication) {
|
||||
logger.debug("Returning 401 because authentication failed");
|
||||
|
||||
response.writeHead(401, {
|
||||
"WWW-Authenticate": 'Basic realm="Access to the registry"',
|
||||
});
|
||||
|
||||
response.end("Unauthorized");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// construct a new url based on the url passed in and the registry url
|
||||
const url = new URL(this.options.origin);
|
||||
const options = {
|
||||
hostname: url.hostname,
|
||||
path: request.url,
|
||||
method: request.method,
|
||||
headers: { ...request.headers },
|
||||
};
|
||||
|
||||
delete options.headers["host"];
|
||||
// delete options.headers["connection"];
|
||||
// delete options.headers["accept-encoding"];
|
||||
delete options.headers["authorization"];
|
||||
// delete options.headers["content-length"];
|
||||
delete options.headers["cf-ray"];
|
||||
delete options.headers["cf-visitor"];
|
||||
delete options.headers["cf-ipcountry"];
|
||||
delete options.headers["cf-connecting-ip"];
|
||||
delete options.headers["cf-warp-tag-id"];
|
||||
|
||||
// Add a custom Authorization header for the proxied request
|
||||
options.headers["authorization"] = `Basic ${Buffer.from(
|
||||
`${this.options.auth.username}:${this.options.auth.password}`
|
||||
).toString("base64")}`;
|
||||
|
||||
let tempFilePath: string | undefined;
|
||||
let cleanupTempFile: () => void = () => {};
|
||||
|
||||
if (
|
||||
options.method === "POST" ||
|
||||
(options.method === "PUT" && request.headers["content-length"])
|
||||
) {
|
||||
tempFilePath = await streamRequestBodyToTempFile(request);
|
||||
|
||||
cleanupTempFile = () => {
|
||||
if (tempFilePath) {
|
||||
logger.debug("Cleaning up temp file", { tempFilePath });
|
||||
unlinkSync(tempFilePath);
|
||||
}
|
||||
};
|
||||
|
||||
logger.debug("Streamed request body to temp file", { tempFilePath });
|
||||
}
|
||||
|
||||
const makeProxiedRequest = (tokenOptions: RequestOptions, attempts: number = 1) => {
|
||||
logger.debug("Proxying request", {
|
||||
request: tokenOptions,
|
||||
attempts,
|
||||
originalHeaders: request.headers,
|
||||
});
|
||||
|
||||
if (attempts > 10) {
|
||||
logger.error("Too many attempts to proxy request", {
|
||||
attempts,
|
||||
});
|
||||
|
||||
response.writeHead(500, { "Content-Type": "text/plain" });
|
||||
response.end("Internal Server Error: Too many attempts to proxy request");
|
||||
|
||||
return cleanupTempFile();
|
||||
}
|
||||
|
||||
const proxyReq = httpRequest(tokenOptions, async (proxyRes) => {
|
||||
// If challenged for bearer token auth, handle it here
|
||||
if (proxyRes.statusCode === 401 && proxyRes.headers["www-authenticate"]) {
|
||||
logger.debug("Received 401 with WWW-Authenticate, attempting to fetch bearer token", {
|
||||
authenticate: proxyRes.headers["www-authenticate"],
|
||||
});
|
||||
|
||||
const bearerToken = await this.#getBearerToken(proxyRes.headers["www-authenticate"]);
|
||||
|
||||
if (bearerToken && tokenOptions.headers) {
|
||||
tokenOptions.headers["authorization"] = `Bearer ${bearerToken}`;
|
||||
makeProxiedRequest(tokenOptions, attempts + 1); // Retry request with bearer token
|
||||
return;
|
||||
} else {
|
||||
// Handle failed token fetch or lack of WWW-Authenticate handling
|
||||
response.writeHead(401, { "Content-Type": "text/plain" });
|
||||
response.end("Failed to authenticate with the registry using bearer token");
|
||||
return cleanupTempFile();
|
||||
}
|
||||
}
|
||||
|
||||
if (proxyRes.statusCode === 401) {
|
||||
logger.debug("Received 401, but there is no www-authenticate value", {
|
||||
headers: proxyRes.headers,
|
||||
});
|
||||
|
||||
response.writeHead(401, { "Content-Type": "text/plain" });
|
||||
response.end("Unauthorized");
|
||||
return cleanupTempFile();
|
||||
}
|
||||
|
||||
if (proxyRes.statusCode === 301) {
|
||||
logger.debug("Received 301, attempting to follow redirect", {
|
||||
location: proxyRes.headers["location"],
|
||||
});
|
||||
|
||||
const redirectOptions = {
|
||||
...tokenOptions,
|
||||
path: proxyRes.headers["location"],
|
||||
};
|
||||
|
||||
makeProxiedRequest(redirectOptions, attempts + 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!proxyRes.statusCode) {
|
||||
logger.error("No status code in the response", {
|
||||
headers: proxyRes.headers,
|
||||
statusMessage: proxyRes.statusMessage,
|
||||
});
|
||||
|
||||
response.writeHead(500, { "Content-Type": "text/plain" });
|
||||
response.end("Internal Server Error: No status code in the response");
|
||||
|
||||
return cleanupTempFile();
|
||||
}
|
||||
|
||||
const headers = { ...proxyRes.headers };
|
||||
|
||||
// Rewrite location headers to point to the proxy
|
||||
if (headers["location"]) {
|
||||
const proxiedLocation = new URL(headers.location);
|
||||
|
||||
// Only rewrite the location header if the host is the same as the registry
|
||||
if (proxiedLocation.host === this.host) {
|
||||
if (!request.headers.host) {
|
||||
// Return a 500 if the host header is missing
|
||||
logger.error("Host header is missing in the request", {
|
||||
headers: request.headers,
|
||||
});
|
||||
|
||||
response.writeHead(500, { "Content-Type": "text/plain" });
|
||||
response.end("Internal Server Error: Host header is missing in the request");
|
||||
return cleanupTempFile();
|
||||
}
|
||||
|
||||
proxiedLocation.host = request.headers.host;
|
||||
|
||||
headers["location"] = proxiedLocation.href;
|
||||
|
||||
logger.debug("Rewriting location response header", {
|
||||
originalLocation: proxyRes.headers["location"],
|
||||
proxiedLocation: headers["location"],
|
||||
proxiedLocationUrl: proxiedLocation.href,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("Proxying successful response", {
|
||||
method: tokenOptions.method,
|
||||
path: tokenOptions.path,
|
||||
statusCode: proxyRes.statusCode,
|
||||
responseHeaders: headers,
|
||||
});
|
||||
|
||||
// Proceed as normal if not a 401 or after getting a bearer token
|
||||
response.writeHead(proxyRes.statusCode, headers);
|
||||
proxyRes.pipe(response, { end: true });
|
||||
});
|
||||
|
||||
if (tempFilePath) {
|
||||
const readStream = createReadStream(tempFilePath);
|
||||
|
||||
readStream.pipe(proxyReq, { end: true });
|
||||
} else {
|
||||
proxyReq.end();
|
||||
}
|
||||
|
||||
proxyReq.on("error", (error) => {
|
||||
logger.error("Error proxying request", { error: error.message });
|
||||
response.writeHead(500, { "Content-Type": "text/plain" });
|
||||
response.end(`Internal Server Error: ${error.message}`);
|
||||
});
|
||||
};
|
||||
|
||||
makeProxiedRequest(options);
|
||||
}
|
||||
|
||||
#getBasicAuthCredentials(request: IncomingMessage) {
|
||||
const headers = request.headers;
|
||||
|
||||
logger.debug("Getting basic auth credentials with headers", {
|
||||
headers,
|
||||
});
|
||||
|
||||
const authHeader = headers["authorization"];
|
||||
|
||||
if (!authHeader) {
|
||||
return;
|
||||
}
|
||||
|
||||
const [type, credentials] = authHeader.split(" ");
|
||||
|
||||
if (type.toLowerCase() !== "basic") {
|
||||
return;
|
||||
}
|
||||
|
||||
const decoded = Buffer.from(credentials, "base64").toString("utf-8");
|
||||
const [username, password] = decoded.split(":");
|
||||
|
||||
return { username, password };
|
||||
}
|
||||
|
||||
async #getBearerToken(authenticateHeader: string): Promise<string | undefined> {
|
||||
try {
|
||||
// Create a md5 hash of the authenticate header to use as a cache key
|
||||
const cacheKey = `token:${createHash("md5").update(authenticateHeader).digest("hex")}`;
|
||||
|
||||
const cachedToken = await this.#getCachedToken(cacheKey);
|
||||
|
||||
if (cachedToken) {
|
||||
return cachedToken;
|
||||
}
|
||||
|
||||
// Parse the WWW-Authenticate header to extract realm and service
|
||||
const realmMatch = authenticateHeader.match(/realm="([^"]+)"/);
|
||||
const serviceMatch = authenticateHeader.match(/service="([^"]+)"/);
|
||||
// Optionally, we could also extract and use the scope parameter if required
|
||||
const scopeMatch = authenticateHeader.match(/scope="([^"]+)"/);
|
||||
|
||||
if (!realmMatch || !serviceMatch) {
|
||||
logger.error("Failed to parse WWW-Authenticate header", { authenticateHeader });
|
||||
return;
|
||||
}
|
||||
|
||||
const realm = realmMatch[1];
|
||||
const service = serviceMatch[1];
|
||||
// Construct the URL for fetching the token
|
||||
let authUrl = `${realm}?service=${encodeURIComponent(service)}`;
|
||||
// Include scope in the request if needed
|
||||
if (scopeMatch) {
|
||||
const scope = scopeMatch[1];
|
||||
authUrl += `&scope=${encodeURIComponent(scope)}`;
|
||||
}
|
||||
|
||||
authUrl += `&account=${encodeURIComponent(this.options.auth.username)}`;
|
||||
|
||||
logger.debug("Fetching bearer token", { authUrl });
|
||||
|
||||
// Make the request to the authentication service
|
||||
const response = await fetch(authUrl, {
|
||||
headers: {
|
||||
authorization:
|
||||
"Basic " +
|
||||
Buffer.from(`${this.options.auth.username}:${this.options.auth.password}`).toString(
|
||||
"base64"
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.debug("Failed to fetch bearer token", {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rawBody = await response.json();
|
||||
const body = TokenResponseBody.safeParse(rawBody);
|
||||
|
||||
if (!body.success) {
|
||||
logger.error("Failed to parse token response", { body: rawBody });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug("Fetched bearer token", { token: body.data.token });
|
||||
|
||||
await this.#setCachedToken(body.data.token, cacheKey);
|
||||
|
||||
return body.data.token;
|
||||
} catch (error) {
|
||||
logger.error("Failed to fetch bearer token", {
|
||||
error: error instanceof Error ? error.message : error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async #getCachedToken(key: string) {
|
||||
if (!this.redis) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cachedToken = await this.redis.get(key);
|
||||
|
||||
if (cachedToken) {
|
||||
const decoded = jwtDecode(cachedToken);
|
||||
const expiry = decoded.exp;
|
||||
|
||||
if (expiry && expiry > Date.now() / 1000 + CACHED_BEARER_TOKEN_BUFFER_IN_SECONDS) {
|
||||
return cachedToken;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #setCachedToken(token: string, key: string) {
|
||||
if (!this.redis) {
|
||||
return;
|
||||
}
|
||||
|
||||
const decoded = jwtDecode(token);
|
||||
|
||||
if (decoded.exp) {
|
||||
await this.redis.set(key, token, "EXAT", decoded.exp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const registryProxy = singleton("registryProxy", initializeProxy);
|
||||
|
||||
function initializeProxy() {
|
||||
if (
|
||||
!env.CONTAINER_REGISTRY_ORIGIN ||
|
||||
!env.CONTAINER_REGISTRY_USERNAME ||
|
||||
!env.CONTAINER_REGISTRY_PASSWORD
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
return new RegistryProxy({
|
||||
origin: env.CONTAINER_REGISTRY_ORIGIN,
|
||||
auth: {
|
||||
username: env.CONTAINER_REGISTRY_USERNAME,
|
||||
password: env.CONTAINER_REGISTRY_PASSWORD,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function streamRequestBodyToTempFile(request: IncomingMessage): Promise<string> {
|
||||
const tempDir = await mkdtemp(`${tmpdir()}/`);
|
||||
const tempFilePath = `${tempDir}/requestBody.tmp`;
|
||||
const writeStream = createWriteStream(tempFilePath);
|
||||
|
||||
await pipeline(request, writeStream);
|
||||
|
||||
return tempFilePath;
|
||||
}
|
||||
|
||||
type DockerImageParts = {
|
||||
registry?: string;
|
||||
repo: string;
|
||||
tag?: string;
|
||||
digest?: string;
|
||||
};
|
||||
|
||||
function parseDockerImageReference(imageReference: string): DockerImageParts {
|
||||
const parts: DockerImageParts = { repo: "" }; // Initialize with an empty repo which we'll fill later
|
||||
|
||||
// Splitting by '@' to separate the digest (if exists)
|
||||
const atSplit = imageReference.split("@");
|
||||
if (atSplit.length > 1) {
|
||||
parts.digest = atSplit[1];
|
||||
imageReference = atSplit[0];
|
||||
}
|
||||
|
||||
// Splitting by ':' to separate the tag (if exists)
|
||||
const colonSplit = imageReference.split(":");
|
||||
if (colonSplit.length > 1 && !colonSplit[1].includes("/")) {
|
||||
// Ensuring we don't split a registry port
|
||||
parts.tag = colonSplit.pop(); // The last part is the tag, remove it from the array
|
||||
imageReference = colonSplit.join(":"); // Join back in case there was more than one colon
|
||||
}
|
||||
|
||||
// Now, the remaining part is "registry/repo" or just "repo"
|
||||
const slashIndex = imageReference.indexOf("/");
|
||||
if (
|
||||
slashIndex !== -1 &&
|
||||
(imageReference.startsWith("localhost") || slashIndex > imageReference.indexOf("."))
|
||||
) {
|
||||
parts.registry = imageReference.substring(0, slashIndex);
|
||||
parts.repo = imageReference.substring(slashIndex + 1);
|
||||
} else {
|
||||
parts.repo = imageReference; // If there's no registry, the whole thing is the repo
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
function rebuildDockerImageReference(parts: DockerImageParts): string {
|
||||
let imageReference = "";
|
||||
|
||||
if (parts.registry) {
|
||||
imageReference += `${parts.registry}/`;
|
||||
}
|
||||
|
||||
imageReference += parts.repo; // Repo is now guaranteed to be defined
|
||||
|
||||
if (parts.tag) {
|
||||
imageReference += `:${parts.tag}`;
|
||||
}
|
||||
|
||||
if (parts.digest) {
|
||||
imageReference += `@${parts.digest}`;
|
||||
}
|
||||
|
||||
return imageReference;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { depot } from "@depot/sdk-node";
|
||||
import { env } from "~/env.server";
|
||||
|
||||
export async function createRemoteImageBuild() {
|
||||
if (!env.DEPOT_TOKEN || !env.DEPOT_PROJECT_ID) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await depot.build.v1.BuildService.createBuild(
|
||||
{ projectId: env.DEPOT_PROJECT_ID },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${env.DEPOT_TOKEN}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
projectId: env.DEPOT_PROJECT_ID,
|
||||
buildToken: result.buildToken,
|
||||
buildId: result.buildId,
|
||||
};
|
||||
}
|
||||
@@ -1,11 +1,12 @@
|
||||
import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
|
||||
import { CreateBackgroundWorkerRequestBody, TaskResource } from "@trigger.dev/core/v3";
|
||||
import type { BackgroundWorker } from "@trigger.dev/database";
|
||||
import { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { marqs } from "../marqs.server";
|
||||
import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { $transaction } from "~/db.server";
|
||||
|
||||
export class CreateBackgroundWorkerService extends BaseService {
|
||||
public async call(
|
||||
@@ -16,150 +17,108 @@ export class CreateBackgroundWorkerService extends BaseService {
|
||||
return this.traceWithEnv("call", environment, async (span) => {
|
||||
span.setAttribute("projectRef", projectRef);
|
||||
|
||||
const backgroundWorker = await $transaction(this._prisma, async (tx) => {
|
||||
const project = await this._prisma.project.findUniqueOrThrow({
|
||||
where: {
|
||||
externalRef: projectRef,
|
||||
environments: {
|
||||
some: {
|
||||
id: environment.id,
|
||||
},
|
||||
const project = await this._prisma.project.findUniqueOrThrow({
|
||||
where: {
|
||||
externalRef: projectRef,
|
||||
environments: {
|
||||
some: {
|
||||
id: environment.id,
|
||||
},
|
||||
},
|
||||
include: {
|
||||
backgroundWorkers: {
|
||||
where: {
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
take: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const latestBackgroundWorker = project.backgroundWorkers[0];
|
||||
|
||||
if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) {
|
||||
return latestBackgroundWorker;
|
||||
}
|
||||
|
||||
const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version);
|
||||
|
||||
logger.debug(`Creating background worker`, {
|
||||
nextVersion,
|
||||
lastVersion: project.backgroundWorkers[0]?.version,
|
||||
});
|
||||
|
||||
const backgroundWorker = await this._prisma.backgroundWorker.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("worker"),
|
||||
version: nextVersion,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
metadata: body.metadata,
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
},
|
||||
});
|
||||
|
||||
if (environment.type !== "DEVELOPMENT") {
|
||||
await this._prisma.imageDetails.update({
|
||||
},
|
||||
include: {
|
||||
backgroundWorkers: {
|
||||
where: {
|
||||
projectId_runtimeEnvironmentId_contentHash: {
|
||||
projectId: environment.projectId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
contentHash: backgroundWorker.contentHash,
|
||||
},
|
||||
},
|
||||
data: {
|
||||
backgroundWorkerId: backgroundWorker.id,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
for (const task of body.metadata.tasks) {
|
||||
await this._prisma.backgroundWorkerTask.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("task"),
|
||||
projectId: project.id,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
workerId: backgroundWorker.id,
|
||||
slug: task.id,
|
||||
filePath: task.filePath,
|
||||
exportName: task.exportName,
|
||||
retryConfig: task.retry,
|
||||
queueConfig: task.queue,
|
||||
},
|
||||
});
|
||||
|
||||
const queueName = task.queue?.name ?? `task/${task.id}`;
|
||||
|
||||
const taskQueue = await this._prisma.taskQueue.upsert({
|
||||
where: {
|
||||
runtimeEnvironmentId_name: {
|
||||
runtimeEnvironmentId: environment.id,
|
||||
name: queueName,
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
update: {
|
||||
concurrencyLimit: task.queue?.concurrencyLimit,
|
||||
rateLimit: task.queue?.rateLimit,
|
||||
},
|
||||
create: {
|
||||
friendlyId: generateFriendlyId("queue"),
|
||||
name: queueName,
|
||||
concurrencyLimit: task.queue?.concurrencyLimit,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
rateLimit: task.queue?.rateLimit,
|
||||
type: task.queue?.name ? "NAMED" : "VIRTUAL",
|
||||
},
|
||||
});
|
||||
|
||||
if (taskQueue.concurrencyLimit) {
|
||||
await marqs?.updateQueueConcurrency(
|
||||
environment,
|
||||
taskQueue.name,
|
||||
taskQueue.concurrencyLimit
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return backgroundWorker;
|
||||
take: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!backgroundWorker) {
|
||||
throw new Error("Failed to create background worker");
|
||||
const latestBackgroundWorker = project.backgroundWorkers[0];
|
||||
|
||||
if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) {
|
||||
return latestBackgroundWorker;
|
||||
}
|
||||
|
||||
const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version);
|
||||
|
||||
logger.debug(`Creating background worker`, {
|
||||
nextVersion,
|
||||
lastVersion: project.backgroundWorkers[0]?.version,
|
||||
});
|
||||
|
||||
const backgroundWorker = await this._prisma.backgroundWorker.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("worker"),
|
||||
version: nextVersion,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
metadata: body.metadata,
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
},
|
||||
});
|
||||
|
||||
await createBackgroundTasks(body.metadata.tasks, backgroundWorker, environment, this._prisma);
|
||||
|
||||
return backgroundWorker;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate next build version based on the previous version
|
||||
// Version formats are YYYYMMDD.1, YYYYMMDD.2, etc.
|
||||
// If there is no previous version, start at Todays date and .1
|
||||
function calculateNextBuildVersion(latestVersion?: string | null): string {
|
||||
const today = new Date();
|
||||
const year = today.getFullYear();
|
||||
const month = today.getMonth() + 1;
|
||||
const day = today.getDate();
|
||||
const todayFormatted = `${year}${month < 10 ? "0" : ""}${month}${day < 10 ? "0" : ""}${day}`;
|
||||
export async function createBackgroundTasks(
|
||||
tasks: TaskResource[],
|
||||
worker: BackgroundWorker,
|
||||
env: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction
|
||||
) {
|
||||
for (const task of tasks) {
|
||||
await prisma.backgroundWorkerTask.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("task"),
|
||||
projectId: worker.projectId,
|
||||
runtimeEnvironmentId: worker.runtimeEnvironmentId,
|
||||
workerId: worker.id,
|
||||
slug: task.id,
|
||||
filePath: task.filePath,
|
||||
exportName: task.exportName,
|
||||
retryConfig: task.retry,
|
||||
queueConfig: task.queue,
|
||||
},
|
||||
});
|
||||
|
||||
if (!latestVersion) {
|
||||
return `${todayFormatted}.1`;
|
||||
const queueName = task.queue?.name ?? `task/${task.id}`;
|
||||
|
||||
const taskQueue = await prisma.taskQueue.upsert({
|
||||
where: {
|
||||
runtimeEnvironmentId_name: {
|
||||
runtimeEnvironmentId: worker.runtimeEnvironmentId,
|
||||
name: queueName,
|
||||
},
|
||||
},
|
||||
update: {
|
||||
concurrencyLimit: task.queue?.concurrencyLimit,
|
||||
rateLimit: task.queue?.rateLimit,
|
||||
},
|
||||
create: {
|
||||
friendlyId: generateFriendlyId("queue"),
|
||||
name: queueName,
|
||||
concurrencyLimit: task.queue?.concurrencyLimit,
|
||||
runtimeEnvironmentId: worker.runtimeEnvironmentId,
|
||||
projectId: worker.projectId,
|
||||
rateLimit: task.queue?.rateLimit,
|
||||
type: task.queue?.name ? "NAMED" : "VIRTUAL",
|
||||
},
|
||||
});
|
||||
|
||||
if (taskQueue.concurrencyLimit) {
|
||||
await marqs?.updateQueueConcurrency(env, taskQueue.name, taskQueue.concurrencyLimit);
|
||||
}
|
||||
}
|
||||
|
||||
const [date, buildNumber] = latestVersion.split(".");
|
||||
|
||||
if (date === todayFormatted) {
|
||||
const nextBuildNumber = parseInt(buildNumber, 10) + 1;
|
||||
return `${date}.${nextBuildNumber}`;
|
||||
}
|
||||
|
||||
return `${todayFormatted}.1`;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
|
||||
import type { BackgroundWorker } from "@trigger.dev/database";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { createBackgroundTasks } from "./createBackgroundWorker.server";
|
||||
|
||||
export class CreateDeployedBackgroundWorkerService extends BaseService {
|
||||
public async call(
|
||||
projectRef: string,
|
||||
environment: AuthenticatedEnvironment,
|
||||
deploymentId: string,
|
||||
body: CreateBackgroundWorkerRequestBody
|
||||
): Promise<BackgroundWorker | undefined> {
|
||||
return this.traceWithEnv("call", environment, async (span) => {
|
||||
span.setAttribute("projectRef", projectRef);
|
||||
|
||||
const deployment = await this._prisma.workerDeployment.findUnique({
|
||||
where: {
|
||||
friendlyId: deploymentId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundWorker = await this._prisma.backgroundWorker.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("worker"),
|
||||
version: deployment.version,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
metadata: body.metadata,
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
},
|
||||
});
|
||||
|
||||
await createBackgroundTasks(body.metadata.tasks, backgroundWorker, environment, this._prisma);
|
||||
|
||||
// Link the deployment with the background worker
|
||||
await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
},
|
||||
data: {
|
||||
workerId: backgroundWorker.id,
|
||||
status: "DEPLOYED",
|
||||
deployedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return backgroundWorker;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3";
|
||||
import type { ImageDetails } from "@trigger.dev/database";
|
||||
import { PrismaClient, prisma } from "~/db.server";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { env } from "~/env.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
|
||||
function escapeStringForRegex(rawString: string) {
|
||||
return rawString.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&").replace(/-/g, "\\x2d");
|
||||
}
|
||||
|
||||
export class CreateImageDetailsService {
|
||||
#prismaClient: PrismaClient;
|
||||
|
||||
constructor(prismaClient: PrismaClient = prisma) {
|
||||
this.#prismaClient = prismaClient;
|
||||
}
|
||||
|
||||
public async call(
|
||||
projectRef: string,
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: CreateImageDetailsRequestBody
|
||||
): Promise<ImageDetails> {
|
||||
const allowedTagPrefix = escapeStringForRegex(`${env.IMAGE_REGISTRY}/${env.IMAGE_REPO}:`);
|
||||
|
||||
if (!body.metadata.imageTag.match(`^${allowedTagPrefix}`)) {
|
||||
if (env.NODE_ENV !== "development") {
|
||||
throw new Error("Forbidden image tag");
|
||||
}
|
||||
}
|
||||
|
||||
const project = await this.#prismaClient.project.findUniqueOrThrow({
|
||||
where: {
|
||||
externalRef: projectRef,
|
||||
environments: {
|
||||
some: {
|
||||
id: environment.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
logger.debug(`Creating image details`, {
|
||||
imageTag: body.metadata.imageTag,
|
||||
});
|
||||
|
||||
const imageDetails = await this.#prismaClient.imageDetails.upsert({
|
||||
where: {
|
||||
projectId_runtimeEnvironmentId_contentHash: {
|
||||
contentHash: body.metadata.contentHash,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
contentHash: body.metadata.contentHash,
|
||||
friendlyId: generateFriendlyId("image"),
|
||||
tag: body.metadata.imageTag,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
metadata: body.metadata,
|
||||
},
|
||||
update: {
|
||||
tag: body.metadata.imageTag,
|
||||
metadata: body.metadata,
|
||||
},
|
||||
});
|
||||
|
||||
await workerQueue.enqueue("indexTasks", { id: imageDetails.id });
|
||||
|
||||
return imageDetails;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { env } from "~/env.server";
|
||||
|
||||
export class IndexDeploymentService extends BaseService {
|
||||
public async call(id: string) {
|
||||
const deployment = await this._prisma.workerDeployment.findUnique({
|
||||
where: {
|
||||
id,
|
||||
},
|
||||
include: {
|
||||
environment: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
logger.error(`No worker deployment with this ID: ${id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!deployment.imageReference) {
|
||||
logger.error(`No image reference for worker deployment: ${id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (deployment.workerId) {
|
||||
logger.debug(
|
||||
`Deployment have already been indexed for ${deployment.friendlyId}. Refreshing worker timestamp.`
|
||||
);
|
||||
|
||||
await this._prisma.backgroundWorker.update({
|
||||
where: {
|
||||
id: deployment.workerId,
|
||||
},
|
||||
data: {
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// just broadcast for now - there should only ever be one provider connected
|
||||
socketIo.providerNamespace.emit("INDEX", {
|
||||
version: "v1",
|
||||
contentHash: deployment.contentHash,
|
||||
imageTag: deployment.imageReference,
|
||||
envId: deployment.environmentId,
|
||||
apiKey: deployment.environment.apiKey,
|
||||
apiUrl: env.APP_ORIGIN,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
import { PrismaClient, prisma } from "~/db.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
|
||||
export type IndexTasksServiceOptions = {
|
||||
idempotencyKey?: string;
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, string | undefined>;
|
||||
};
|
||||
|
||||
export class IndexTasksService {
|
||||
#prismaClient: PrismaClient;
|
||||
|
||||
constructor(prismaClient: PrismaClient = prisma) {
|
||||
this.#prismaClient = prismaClient;
|
||||
}
|
||||
|
||||
public async call(imageDetailsId: string) {
|
||||
const imageDetails = await this.#prismaClient.imageDetails.findUnique({
|
||||
where: {
|
||||
id: imageDetailsId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!imageDetails) {
|
||||
logger.error(`No image details with this ID: ${imageDetailsId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (imageDetails.backgroundWorkerId) {
|
||||
logger.debug(
|
||||
`Image details have already been indexed for ${imageDetails.friendlyId}. Refreshing worker timestamp.`
|
||||
);
|
||||
await this.#prismaClient.backgroundWorker.update({
|
||||
where: {
|
||||
id: imageDetails.backgroundWorkerId,
|
||||
},
|
||||
data: {
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// just broadcast for now - there should only ever be one provider connected
|
||||
socketIo.providerNamespace.emit("INDEX", {
|
||||
version: "v1",
|
||||
contentHash: imageDetails.contentHash,
|
||||
imageTag: imageDetails.tag,
|
||||
envId: imageDetails.runtimeEnvironmentId,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { customAlphabet } from "nanoid";
|
||||
import { createRemoteImageBuild } from "../remoteImageBuilder.server";
|
||||
import { InitializeDeploymentRequestBody } from "@trigger.dev/core/v3";
|
||||
|
||||
const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 8);
|
||||
|
||||
export class InitializeDeploymentService extends BaseService {
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
payload: InitializeDeploymentRequestBody
|
||||
) {
|
||||
return this.traceWithEnv("call", environment, async (span) => {
|
||||
const latestDeployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
environmentId: environment.id,
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
take: 1,
|
||||
});
|
||||
|
||||
const nextVersion = calculateNextBuildVersion(latestDeployment?.version);
|
||||
|
||||
// Try and create a depot build and get back the external build data
|
||||
const externalBuildData = await createRemoteImageBuild();
|
||||
|
||||
const deployment = await this._prisma.workerDeployment.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("deployment"),
|
||||
contentHash: payload.contentHash,
|
||||
shortCode: nanoid(8),
|
||||
version: nextVersion,
|
||||
status: "BUILDING",
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
externalBuildData,
|
||||
},
|
||||
});
|
||||
|
||||
const imageTag = `trigger/${environment.project.externalRef}:${deployment.version}.${environment.slug}`;
|
||||
|
||||
return { deployment, imageTag };
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { StartDeploymentIndexingRequestBody } from "@trigger.dev/core/v3";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { registryProxy } from "../registryProxy.server";
|
||||
|
||||
export class StartDeploymentIndexing extends BaseService {
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
deploymentId: string,
|
||||
body: StartDeploymentIndexingRequestBody
|
||||
) {
|
||||
const deployment = await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
friendlyId: deploymentId,
|
||||
},
|
||||
data: {
|
||||
imageReference: registryProxy
|
||||
? registryProxy.rewriteImageReference(body.imageReference)
|
||||
: body.imageReference,
|
||||
status: "DEPLOYING",
|
||||
},
|
||||
});
|
||||
|
||||
await workerQueue.enqueue("v3.indexDeployment", { id: deployment.id });
|
||||
|
||||
return deployment;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Calculate next build version based on the previous version
|
||||
// Version formats are YYYYMMDD.1, YYYYMMDD.2, etc.
|
||||
// If there is no previous version, start at Todays date and .1
|
||||
export function calculateNextBuildVersion(latestVersion?: string | null): string {
|
||||
const today = new Date();
|
||||
const year = today.getFullYear();
|
||||
const month = today.getMonth() + 1;
|
||||
const day = today.getDate();
|
||||
const todayFormatted = `${year}${month < 10 ? "0" : ""}${month}${day < 10 ? "0" : ""}${day}`;
|
||||
|
||||
if (!latestVersion) {
|
||||
return `${todayFormatted}.1`;
|
||||
}
|
||||
|
||||
const [date, buildNumber] = latestVersion.split(".");
|
||||
|
||||
if (date === todayFormatted) {
|
||||
const nextBuildNumber = parseInt(buildNumber, 10) + 1;
|
||||
return `${date}.${nextBuildNumber}`;
|
||||
}
|
||||
|
||||
return `${todayFormatted}.1`;
|
||||
}
|
||||
@@ -41,6 +41,7 @@
|
||||
"@codemirror/view": "^6.5.0",
|
||||
"@conform-to/react": "^0.6.1",
|
||||
"@conform-to/zod": "^0.6.1",
|
||||
"@depot/sdk-node": "^0.5.0",
|
||||
"@headlessui/react": "^1.7.8",
|
||||
"@heroicons/react": "^2.0.12",
|
||||
"@highlight-run/node": "^3.1.0",
|
||||
@@ -112,6 +113,7 @@
|
||||
"ioredis": "^5.3.2",
|
||||
"isbot": "^3.6.5",
|
||||
"jsonpointer": "^5.0.1",
|
||||
"jwt-decode": "^4.0.0",
|
||||
"lodash.omit": "^4.5.0",
|
||||
"lucide-react": "^0.229.0",
|
||||
"marked": "^4.0.18",
|
||||
|
||||
+40
-22
@@ -7,23 +7,10 @@ import { WebSocketServer } from "ws";
|
||||
import { broadcastDevReady, logDevReady } from "@remix-run/server-runtime";
|
||||
import type { Server as IoServer } from "socket.io";
|
||||
import type { Server as EngineServer } from "engine.io";
|
||||
import { RegistryProxy } from "~/v3/registryProxy.server";
|
||||
|
||||
const app = express();
|
||||
|
||||
app.use((req, res, next) => {
|
||||
// helpful headers:
|
||||
res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`);
|
||||
|
||||
// /clean-urls/ -> /clean-urls
|
||||
if (req.path.endsWith("/") && req.path.length > 1) {
|
||||
const query = req.url.slice(req.path.length);
|
||||
const safepath = req.path.slice(0, -1).replace(/\/+/g, "/");
|
||||
res.redirect(301, safepath + query);
|
||||
return;
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
if (process.env.DISABLE_COMPRESSION !== "1") {
|
||||
app.use(compression());
|
||||
}
|
||||
@@ -40,23 +27,54 @@ app.use(express.static("public", { maxAge: "1h" }));
|
||||
|
||||
app.use(morgan("tiny"));
|
||||
|
||||
process.title = "node webapp-server";
|
||||
|
||||
const MODE = process.env.NODE_ENV;
|
||||
const BUILD_DIR = path.join(process.cwd(), "build");
|
||||
const build = require(BUILD_DIR);
|
||||
|
||||
app.all(
|
||||
"*",
|
||||
createRequestHandler({
|
||||
build,
|
||||
mode: MODE,
|
||||
})
|
||||
);
|
||||
|
||||
const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
|
||||
|
||||
if (process.env.HTTP_SERVER_DISABLED !== "true") {
|
||||
const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
|
||||
const wss: WebSocketServer | undefined = build.entry.module.wss;
|
||||
const registryProxy: RegistryProxy | undefined = build.entry.module.registryProxy;
|
||||
|
||||
if (registryProxy && process.env.ENABLE_REGISTRY_PROXY === "true") {
|
||||
console.log(`🐳 Enabling container registry proxy to ${registryProxy.origin}`);
|
||||
|
||||
// Adjusted to match /v2 and any subpath under /v2
|
||||
app.all("/v2/*", async (req, res) => {
|
||||
await registryProxy.call(req, res);
|
||||
});
|
||||
|
||||
// This might also be necessary if you need to explicitly match /v2 as well
|
||||
app.all("/v2", async (req, res) => {
|
||||
await registryProxy.call(req, res);
|
||||
});
|
||||
}
|
||||
|
||||
app.use((req, res, next) => {
|
||||
// helpful headers:
|
||||
res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`);
|
||||
|
||||
// /clean-urls/ -> /clean-urls
|
||||
if (req.path.endsWith("/") && req.path.length > 1) {
|
||||
const query = req.url.slice(req.path.length);
|
||||
const safepath = req.path.slice(0, -1).replace(/\/+/g, "/");
|
||||
res.redirect(301, safepath + query);
|
||||
return;
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
app.all(
|
||||
"*",
|
||||
createRequestHandler({
|
||||
build,
|
||||
mode: MODE,
|
||||
})
|
||||
);
|
||||
|
||||
const server = app.listen(port, () => {
|
||||
console.log(`✅ app ready: http://localhost:${port} [NODE_ENV: ${MODE}]`);
|
||||
|
||||
@@ -64,15 +64,11 @@
|
||||
"typecheck": "tsc -p tsconfig.check.json",
|
||||
"build": "npm run clean && run-p build:**",
|
||||
"build:main": "tsup",
|
||||
"build:facade": "tsup --config tsup.facade.config.ts",
|
||||
"build:prod-facade": "tsup --config tsup.prod-facade.config.ts",
|
||||
"build:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts",
|
||||
"build:workers": "tsup --config tsup.workers.config.ts",
|
||||
"build:prod-containerfile": "cpy --flat src/Containerfile.prod dist/",
|
||||
"dev": "npm run clean && run-p dev:**",
|
||||
"dev:main": "tsup --watch",
|
||||
"dev:facade": "tsup --config tsup.facade.config.ts --watch",
|
||||
"dev:prod-facade": "tsup --config tsup.prod-facade.config.ts --watch",
|
||||
"dev:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts --watch",
|
||||
"dev:workers": "tsup --config tsup.workers.config.ts --watch",
|
||||
"dev:prod-containerfile": "npm-watch",
|
||||
"clean": "rimraf dist",
|
||||
"start": "node dist/index.js",
|
||||
@@ -80,6 +76,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^0.7.0",
|
||||
"@depot/cli": "0.0.1-cli.2.55.0",
|
||||
"@opentelemetry/api": "^1.7.0",
|
||||
"@opentelemetry/api-logs": "^0.48.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.40.3",
|
||||
@@ -131,6 +128,7 @@
|
||||
"source-map-support": "^0.5.21",
|
||||
"supports-color": "^9.4.0",
|
||||
"terminal-link": "^3.0.0",
|
||||
"tiny-invariant": "^1.2.0",
|
||||
"update-check": "^1.5.4",
|
||||
"url": "^0.11.1",
|
||||
"ws": "^8.12.0",
|
||||
|
||||
@@ -2,17 +2,32 @@ FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dd
|
||||
|
||||
RUN apk add --no-cache dumb-init
|
||||
|
||||
FROM base
|
||||
|
||||
ENV TRIGGER_CONTENT_HASH=__CONTENT_HASH__
|
||||
ENV TRIGGER_PROJECT_DIR=__PROJECT_DIR__
|
||||
ENV TRIGGER_PROJECT_REF=__PROJECT_REF__
|
||||
ENV TRIGGER_CLI_PACKAGE_VERSION=__CLI_PACKAGE_VERSION__
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies
|
||||
COPY --chown=node:node package.json package-lock.json ./
|
||||
RUN npm ci --no-fund --no-audit && npm cache clean --force
|
||||
|
||||
# Development or production stage builds upon the base stage
|
||||
FROM base AS final
|
||||
|
||||
# Copy the rest of the application
|
||||
COPY --chown=node:node . .
|
||||
|
||||
# Use ARG for build-time variables
|
||||
ARG TRIGGER_PROJECT_ID
|
||||
ARG TRIGGER_DEPLOYMENT_ID
|
||||
ARG TRIGGER_DEPLOYMENT_VERSION
|
||||
ARG TRIGGER_CONTENT_HASH
|
||||
ARG TRIGGER_PROJECT_REF
|
||||
|
||||
ENV TRIGGER_PROJECT_ID=${TRIGGER_PROJECT_ID} \
|
||||
TRIGGER_DEPLOYMENT_ID=${TRIGGER_DEPLOYMENT_ID} \
|
||||
TRIGGER_DEPLOYMENT_VERSION=${TRIGGER_DEPLOYMENT_VERSION} \
|
||||
TRIGGER_CONTENT_HASH=${TRIGGER_CONTENT_HASH} \
|
||||
TRIGGER_PROJECT_REF=${TRIGGER_PROJECT_REF} \
|
||||
NODE_ENV=production
|
||||
|
||||
USER node
|
||||
|
||||
CMD [ "dumb-init", "node", "index.mjs" ]
|
||||
CMD [ "dumb-init", "node", "index.js" ]
|
||||
@@ -5,10 +5,13 @@ import {
|
||||
WhoAmIResponseSchema,
|
||||
CreateBackgroundWorkerRequestBody,
|
||||
CreateBackgroundWorkerResponse,
|
||||
CreateImageDetailsRequestBody,
|
||||
CreateImageDetailsResponse,
|
||||
StartDeploymentIndexingResponseBody,
|
||||
GetProjectDevResponse,
|
||||
GetEnvironmentVariablesResponseBody,
|
||||
InitializeDeploymentResponseBody,
|
||||
InitializeDeploymentRequestBody,
|
||||
StartDeploymentIndexingRequestBody,
|
||||
GetDeploymentResponseBody,
|
||||
} from "@trigger.dev/core/v3";
|
||||
|
||||
export class CliApiClient {
|
||||
@@ -72,25 +75,6 @@ export class CliApiClient {
|
||||
);
|
||||
}
|
||||
|
||||
async createImageDetails(projectRef: string, body: CreateImageDetailsRequestBody) {
|
||||
if (!this.accessToken) {
|
||||
throw new Error("createImageDetails: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
CreateImageDetailsResponse,
|
||||
`${this.apiURL}/api/v1/projects/${projectRef}/image-details`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async getProjectDevEnv({ projectRef }: { projectRef: string }) {
|
||||
if (!this.accessToken) {
|
||||
throw new Error("getProjectDevEnv: No access token");
|
||||
@@ -133,6 +117,57 @@ export class CliApiClient {
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async initializeDeployment(body: InitializeDeploymentRequestBody) {
|
||||
if (!this.accessToken) {
|
||||
throw new Error("initializeDeployment: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(InitializeDeploymentResponseBody, `${this.apiURL}/api/v1/deployments`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
async startDeploymentIndexing(deploymentId: string, body: StartDeploymentIndexingRequestBody) {
|
||||
if (!this.accessToken) {
|
||||
throw new Error("startDeploymentIndexing: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
StartDeploymentIndexingResponseBody,
|
||||
`${this.apiURL}/api/v1/deployments/${deploymentId}/start-indexing`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async getDeployment(deploymentId: string) {
|
||||
if (!this.accessToken) {
|
||||
throw new Error("getDeployment: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
GetDeploymentResponseBody,
|
||||
`${this.apiURL}/api/v1/deployments/${deploymentId}`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
type ApiResult<TSuccessResult> =
|
||||
|
||||
@@ -9,6 +9,7 @@ import { COMMAND_NAME } from "../consts.js";
|
||||
import { getVersion } from "../utilities/getVersion.js";
|
||||
import { printInitialBanner } from "../utilities/initialBanner.js";
|
||||
import { configureBuildCommand } from "../commands/build.js";
|
||||
import { configureDeployCommand } from "../commands/deploy.js";
|
||||
|
||||
export const program = new Command();
|
||||
|
||||
@@ -56,9 +57,9 @@ program
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
configureBuildCommand(program);
|
||||
configureDevCommand(program);
|
||||
configureDeployCommand(program);
|
||||
|
||||
program
|
||||
.command("update")
|
||||
|
||||
@@ -98,13 +98,7 @@ async function startBuild(
|
||||
apiKey: prodEnv.data.apiKey,
|
||||
});
|
||||
|
||||
const envClient = new CliApiClient(authorization.apiUrl, prodEnv.data.apiKey);
|
||||
await envClient.createImageDetails(config.project, {
|
||||
metadata: {
|
||||
contentHash: buildResult.contentHash,
|
||||
imageTag: buildResult.imageTag,
|
||||
},
|
||||
});
|
||||
logger.log(`⎔ Finished building ${buildResult.imageTag}`);
|
||||
} catch (e) {
|
||||
throw e;
|
||||
}
|
||||
@@ -142,7 +136,7 @@ async function runBuild(
|
||||
logger.log(chalk.green(`Typecheck succeeded.\n`));
|
||||
}
|
||||
|
||||
logger.log(chalk.dim("⎔ Bundling tasks..."));
|
||||
logger.log(chalk.dim("⎔ Building tasks..."));
|
||||
|
||||
const result = await build({
|
||||
stdin: {
|
||||
@@ -150,7 +144,7 @@ async function runBuild(
|
||||
resolveDir: process.cwd(),
|
||||
sourcefile: "__entryPoint.ts",
|
||||
},
|
||||
bundle: true,
|
||||
bundle: false,
|
||||
metafile: true,
|
||||
write: false,
|
||||
minify: false,
|
||||
@@ -160,9 +154,6 @@ async function runBuild(
|
||||
format: "esm",
|
||||
target: ["node18", "es2020"],
|
||||
outdir: "out",
|
||||
banner: {
|
||||
js: "import { createRequire } from 'module';const require = createRequire(import.meta.url);",
|
||||
},
|
||||
});
|
||||
|
||||
if (result.errors.length > 0) {
|
||||
|
||||
@@ -0,0 +1,571 @@
|
||||
import { intro, spinner } from "@clack/prompts";
|
||||
import { depot } from "@depot/cli";
|
||||
import { ResolvedConfig } from "@trigger.dev/core/v3";
|
||||
import { Command } from "commander";
|
||||
import { Metafile, build } from "esbuild";
|
||||
import { execa } from "execa";
|
||||
import { resolve as importResolve } from "import-meta-resolve";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { copyFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { exit } from "node:process";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import invariant from "tiny-invariant";
|
||||
import { z } from "zod";
|
||||
import * as packageJson from "../../package.json";
|
||||
import { CliApiClient } from "../apiClient";
|
||||
import { CommonCommandOptions } from "../cli/common.js";
|
||||
import { getConfigPath, readConfig } from "../utilities/configFiles.js";
|
||||
import { createTempDir, readJSONFile, writeJSONFile } from "../utilities/fileSystem";
|
||||
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
|
||||
import { detectPackageNameFromImportPath } from "../utilities/installPackages";
|
||||
import { logger } from "../utilities/logger.js";
|
||||
import { isLoggedIn } from "../utilities/session.js";
|
||||
import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
|
||||
|
||||
const DeployCommandOptions = CommonCommandOptions.extend({
|
||||
skipTypecheck: z.boolean().optional(),
|
||||
});
|
||||
|
||||
type DeployCommandOptions = z.infer<typeof DeployCommandOptions>;
|
||||
|
||||
export function configureDeployCommand(program: Command) {
|
||||
program
|
||||
.command("deploy")
|
||||
.description("Deploy your Trigger.dev v3 project to the cloud.")
|
||||
.argument("[path]", "The path to the project", ".")
|
||||
.option("-T, --skip-typecheck", "Whether to skip the pre-build typecheck")
|
||||
.option(
|
||||
"-l, --log-level <level>",
|
||||
"The log level to use (debug, info, log, warn, error, none)",
|
||||
"log"
|
||||
)
|
||||
.action(async (path, options) => {
|
||||
try {
|
||||
await deployCommand(path, options);
|
||||
} catch (e) {
|
||||
//todo error reporting
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function deployCommand(dir: string, anyOptions: unknown) {
|
||||
const options = DeployCommandOptions.safeParse(anyOptions);
|
||||
|
||||
if (!options.success) {
|
||||
throw new Error(`Invalid options: ${options.error}`);
|
||||
}
|
||||
|
||||
const authorization = await isLoggedIn();
|
||||
|
||||
if (!authorization.ok) {
|
||||
if (authorization.error === "fetch failed") {
|
||||
logger.error(
|
||||
`Failed to connect to ${authorization.config?.apiUrl}. Are you sure it's the correct URL?`
|
||||
);
|
||||
} else {
|
||||
logger.error("You must login first. Use `trigger.dev login` to login.");
|
||||
}
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.data.logLevel) {
|
||||
logger.loggerLevel = options.data.logLevel;
|
||||
}
|
||||
|
||||
await printStandloneInitialBanner(true);
|
||||
|
||||
const configPath = await getConfigPath(dir);
|
||||
const config = await readConfig(configPath);
|
||||
|
||||
const apiClient = new CliApiClient(authorization.config.apiUrl, authorization.config.accessToken);
|
||||
|
||||
const prodEnv = await apiClient.getProjectProdEnv({ projectRef: config.project });
|
||||
|
||||
if (!prodEnv.success) {
|
||||
throw new Error(prodEnv.error);
|
||||
}
|
||||
|
||||
const environmentClient = new CliApiClient(authorization.config.apiUrl, prodEnv.data.apiKey);
|
||||
|
||||
intro(`Preparing to deploy "${prodEnv.data.name}" (${config.project})`);
|
||||
|
||||
// Step 1: Build the project into a temporary directory
|
||||
const compilation = await compileProject(config, options.data);
|
||||
|
||||
const deploymentSpinner = spinner();
|
||||
|
||||
deploymentSpinner.start("Initializing deployment");
|
||||
|
||||
// Step 2: Initialize a deployment on the server (response will have everything we need to build an image)
|
||||
const deploymentResponse = await environmentClient.initializeDeployment({
|
||||
contentHash: compilation.contentHash,
|
||||
});
|
||||
|
||||
if (!deploymentResponse.success) {
|
||||
deploymentSpinner.stop(`Failed to initialize deployment: ${deploymentResponse.error}`);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
deploymentSpinner.message(`Deploying version ${deploymentResponse.data.version}`);
|
||||
|
||||
// If the deployment doesn't have any externalBuildData, then we can't use the remote image builder
|
||||
// TODO: handle this and allow the user to the build and push the image themselves
|
||||
if (!deploymentResponse.data.externalBuildData) {
|
||||
deploymentSpinner.stop(
|
||||
`Failed to initialize deployment. The deployment does not have any external build data. Support for local building coming soon.`
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const registryHost = new URL(prodEnv.data.apiUrl).host;
|
||||
|
||||
const image = await buildAndPushImage({
|
||||
registryHost,
|
||||
auth: authorization.config.accessToken,
|
||||
imageTag: deploymentResponse.data.imageTag,
|
||||
buildId: deploymentResponse.data.externalBuildData.buildId,
|
||||
buildToken: deploymentResponse.data.externalBuildData.buildToken,
|
||||
buildProjectId: deploymentResponse.data.externalBuildData.projectId,
|
||||
cwd: compilation.path,
|
||||
projectId: config.project,
|
||||
deploymentId: deploymentResponse.data.id,
|
||||
deploymentVersion: deploymentResponse.data.version,
|
||||
contentHash: deploymentResponse.data.contentHash,
|
||||
projectRef: config.project,
|
||||
});
|
||||
|
||||
if (!image.ok) {
|
||||
deploymentSpinner.stop(`Failed to build and push image: ${image.error}`);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const imageReference = `${registryHost}/${image.image}${image.digest ? `@${image.digest}` : ""}`;
|
||||
|
||||
deploymentSpinner.message(
|
||||
`${deploymentResponse.data.version} image uploaded, starting indexing process`
|
||||
);
|
||||
|
||||
logger.debug(`Image built and pushed: ${imageReference}`);
|
||||
|
||||
// Need to update the deployment with the image and start the deployment (indexing)
|
||||
// registry.digitalocean.com/trigger/yubjwjsfkxnylobaqvqz:20240306.41.prod@sha256:8b48dd2866bc8878644d2880bbe35a27e66cf6ff78aa1e489d7fdde5e228faf1
|
||||
// Step 5: Update the deployment with the image and start the deployment (indexing)
|
||||
const startIndexingResponse = await environmentClient.startDeploymentIndexing(
|
||||
deploymentResponse.data.id,
|
||||
{
|
||||
imageReference,
|
||||
}
|
||||
);
|
||||
|
||||
if (!startIndexingResponse.success) {
|
||||
deploymentSpinner.stop(`Failed to start indexing: ${startIndexingResponse.error}`);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Step 6: Wait for the deployment to finish and print the result
|
||||
const completedDeployment = await waitForDeploymentToComplete(
|
||||
deploymentResponse.data.id,
|
||||
environmentClient
|
||||
);
|
||||
|
||||
if (!completedDeployment) {
|
||||
deploymentSpinner.stop(`Deployment failed to complete`);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
deploymentSpinner.stop(`Deployment completed successfully, you can now use this version`);
|
||||
}
|
||||
|
||||
// Poll every 1 second for the deployment to complete
|
||||
async function waitForDeploymentToComplete(
|
||||
deploymentId: string,
|
||||
client: CliApiClient,
|
||||
timeoutInSeconds: number = 60
|
||||
) {
|
||||
const start = Date.now();
|
||||
|
||||
while (true) {
|
||||
if (Date.now() - start > timeoutInSeconds * 1000) {
|
||||
return;
|
||||
}
|
||||
|
||||
const deployment = await client.getDeployment(deploymentId);
|
||||
|
||||
if (!deployment.success) {
|
||||
throw new Error(deployment.error);
|
||||
}
|
||||
|
||||
logger.debug(`Deployment status: ${deployment.data.status}`);
|
||||
|
||||
if (deployment.data.status === "DEPLOYED") {
|
||||
return deployment.data;
|
||||
}
|
||||
|
||||
await setTimeout(1000);
|
||||
}
|
||||
}
|
||||
|
||||
type BuildAndPushImageOptions = {
|
||||
registryHost: string;
|
||||
auth: string;
|
||||
imageTag: string;
|
||||
buildId: string;
|
||||
buildToken: string;
|
||||
buildProjectId: string;
|
||||
cwd: string;
|
||||
projectId: string;
|
||||
deploymentId: string;
|
||||
deploymentVersion: string;
|
||||
contentHash: string;
|
||||
projectRef: string;
|
||||
};
|
||||
|
||||
type BuildAndPushImageResults =
|
||||
| {
|
||||
ok: true;
|
||||
image: string;
|
||||
digest?: string;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
async function buildAndPushImage(
|
||||
options: BuildAndPushImageOptions
|
||||
): Promise<BuildAndPushImageResults> {
|
||||
// Step 3: Ensure we are "logged in" to our registry by writing to $HOME/.docker/config.json
|
||||
// TODO: make sure this works on windows
|
||||
const dockerConfigDir = await ensureLoggedIntoDockerRegistry(options.registryHost, {
|
||||
username: "trigger",
|
||||
password: options.auth,
|
||||
});
|
||||
|
||||
const args = [
|
||||
"build",
|
||||
"-f",
|
||||
"Containerfile",
|
||||
"--platform",
|
||||
"linux/amd64",
|
||||
"--provenance",
|
||||
"false",
|
||||
"--build-arg",
|
||||
`TRIGGER_PROJECT_ID=${options.projectId}`,
|
||||
"--build-arg",
|
||||
`TRIGGER_DEPLOYMENT_ID=${options.deploymentId}`,
|
||||
"--build-arg",
|
||||
`TRIGGER_DEPLOYMENT_VERSION=${options.deploymentVersion}`,
|
||||
"--build-arg",
|
||||
`TRIGGER_CONTENT_HASH=${options.contentHash}`,
|
||||
"--build-arg",
|
||||
`TRIGGER_PROJECT_REF=${options.projectRef}`,
|
||||
"-t",
|
||||
`${options.registryHost}/${options.imageTag}`,
|
||||
"--push",
|
||||
".",
|
||||
];
|
||||
|
||||
logger.debug(`depot ${args.join(" ")}`);
|
||||
|
||||
// Step 4: Build and push the image
|
||||
const childProcess = depot(args, {
|
||||
cwd: options.cwd,
|
||||
env: {
|
||||
DEPOT_BUILD_ID: options.buildId,
|
||||
DEPOT_TOKEN: options.buildToken,
|
||||
DEPOT_PROJECT_ID: options.buildProjectId,
|
||||
DEPOT_NO_SUMMARY_LINK: "1",
|
||||
DOCKER_CONFIG: dockerConfigDir,
|
||||
},
|
||||
});
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
try {
|
||||
await new Promise<void>((res, rej) => {
|
||||
// For some reason everything is output on stderr, not stdout
|
||||
childProcess.stderr?.on("data", (data: Buffer) => {
|
||||
const text = data.toString();
|
||||
|
||||
errors.push(text);
|
||||
});
|
||||
|
||||
childProcess.on("error", (e) => rej(e));
|
||||
childProcess.on("close", () => res());
|
||||
});
|
||||
|
||||
const digest = extractImageDigest(errors);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
image: options.imageTag,
|
||||
digest,
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false,
|
||||
error: e instanceof Error ? e.message : JSON.stringify(e),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function extractImageDigest(outputs: string[]) {
|
||||
const imageDigestRegex = /sha256:[a-f0-9]{64}/;
|
||||
|
||||
for (const line of outputs) {
|
||||
if (line.includes("pushing manifest")) {
|
||||
const imageDigestMatch = line.match(imageDigestRegex);
|
||||
if (imageDigestMatch) {
|
||||
return imageDigestMatch[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function compileProject(config: ResolvedConfig, options: DeployCommandOptions) {
|
||||
if (!options.skipTypecheck) {
|
||||
await typecheckProject(config, options);
|
||||
}
|
||||
|
||||
const createAuthCodeSpinner = spinner();
|
||||
createAuthCodeSpinner.start(`Compiling project "${config.project}" in "${config.projectDir}"`);
|
||||
|
||||
const taskFiles = await gatherTaskFiles(config);
|
||||
const workerFacade = readFileSync(
|
||||
new URL(importResolve("./workers/prod/worker-facade.js", import.meta.url)).href.replace(
|
||||
"file://",
|
||||
""
|
||||
),
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
const registerTracingPath = new URL(
|
||||
importResolve("./workers/common/register-tracing.js", import.meta.url)
|
||||
).href.replace("file://", "");
|
||||
|
||||
const workerContents = workerFacade
|
||||
.replace("__TASKS__", createTaskFileImports(taskFiles))
|
||||
.replace("__REGISTER_TRACING__", `import { tracingSDK } from "${registerTracingPath}";`);
|
||||
|
||||
const result = await build({
|
||||
stdin: {
|
||||
contents: workerContents,
|
||||
resolveDir: process.cwd(),
|
||||
sourcefile: "__entryPoint.ts",
|
||||
},
|
||||
bundle: true,
|
||||
metafile: true,
|
||||
write: false,
|
||||
minify: false,
|
||||
sourcemap: "external", // does not set the //# sourceMappingURL= comment in the file, we handle it ourselves
|
||||
packages: "external", // https://esbuild.github.io/api/#packages
|
||||
logLevel: "error",
|
||||
platform: "node",
|
||||
format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching
|
||||
target: ["node18", "es2020"],
|
||||
outdir: "out",
|
||||
define: {
|
||||
TRIGGER_API_URL: `"${config.triggerUrl}"`,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.errors.length > 0) {
|
||||
createAuthCodeSpinner.stop("Build failed");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const entryPointContents = readFileSync(
|
||||
new URL(importResolve("./workers/prod/entry-point.js", import.meta.url)).href.replace(
|
||||
"file://",
|
||||
""
|
||||
),
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
const entryPointResult = await build({
|
||||
stdin: {
|
||||
contents: entryPointContents,
|
||||
resolveDir: process.cwd(),
|
||||
sourcefile: "index.ts",
|
||||
},
|
||||
bundle: true,
|
||||
metafile: true,
|
||||
write: false,
|
||||
minify: false,
|
||||
sourcemap: false,
|
||||
packages: "external", // https://esbuild.github.io/api/#packages
|
||||
logLevel: "error",
|
||||
platform: "node",
|
||||
format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching
|
||||
target: ["node18", "es2020"],
|
||||
outdir: "out",
|
||||
});
|
||||
|
||||
if (entryPointResult.errors.length > 0) {
|
||||
createAuthCodeSpinner.stop("Build failed");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Create a tmp directory to store the build
|
||||
const tempDir = await createTempDir();
|
||||
|
||||
logger.debug(`Writing compiled files to ${tempDir}`);
|
||||
|
||||
// Get the metaOutput for the result build
|
||||
const metaOutput = result.metafile!.outputs[join("out", "stdin.js")];
|
||||
|
||||
invariant(metaOutput, "Meta output for the result build is missing");
|
||||
|
||||
// Get the metaOutput for the entryPoint build
|
||||
const entryPointMetaOutput = entryPointResult.metafile!.outputs[join("out", "stdin.js")];
|
||||
|
||||
invariant(entryPointMetaOutput, "Meta output for the entryPoint build is missing");
|
||||
|
||||
// Get the outputFile and the sourceMapFile for the result build
|
||||
const workerOutputFile = result.outputFiles.find(
|
||||
(file) => file.path === join(config.projectDir, "out", "stdin.js")
|
||||
);
|
||||
|
||||
invariant(workerOutputFile, "Output file for the result build is missing");
|
||||
|
||||
const workerSourcemapFile = result.outputFiles.find(
|
||||
(file) => file.path === join(config.projectDir, "out", "stdin.js.map")
|
||||
);
|
||||
|
||||
invariant(workerSourcemapFile, "Sourcemap file for the result build is missing");
|
||||
|
||||
// Get the outputFile for the entryPoint build
|
||||
|
||||
const entryPointOutputFile = entryPointResult.outputFiles.find(
|
||||
(file) => file.path === join(config.projectDir, "out", "stdin.js")
|
||||
);
|
||||
|
||||
invariant(entryPointOutputFile, "Output file for the entryPoint build is missing");
|
||||
|
||||
// Save the result outputFile to /tmp/dir/worker.js (and make sure to map the sourceMap to the correct location in the file)
|
||||
await writeFile(
|
||||
join(tempDir, "worker.js"),
|
||||
`${workerOutputFile.text}\n//# sourceMappingURL=worker.js.map`
|
||||
);
|
||||
// Save the sourceMapFile to /tmp/dir/worker.js.map
|
||||
await writeFile(join(tempDir, "worker.js.map"), workerSourcemapFile.text);
|
||||
// Save the entryPoint outputFile to /tmp/dir/index.js
|
||||
await writeFile(join(tempDir, "index.js"), entryPointOutputFile.text);
|
||||
|
||||
// Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json
|
||||
const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports];
|
||||
const projectPackageJson = await readJSONFile(join(config.projectDir, "package.json"));
|
||||
const dependencies = gatherRequiredDependencies(allImports, projectPackageJson);
|
||||
|
||||
await writeJSONFile(join(tempDir, "package.json"), {
|
||||
name: "trigger-worker",
|
||||
version: "0.0.0",
|
||||
description: "",
|
||||
dependencies,
|
||||
});
|
||||
|
||||
createAuthCodeSpinner.stop(`Project "${config.project}" compiled successfully`);
|
||||
|
||||
// Run npm install --package-lock-only in /tmp/dir to produce a package-lock.json
|
||||
const resolvingDepsSpinner = spinner();
|
||||
|
||||
resolvingDepsSpinner.start("Resolving dependencies");
|
||||
|
||||
await execa("npm", ["install", "--package-lock-only"], {
|
||||
cwd: tempDir,
|
||||
});
|
||||
|
||||
resolvingDepsSpinner.stop("Dependencies resolved");
|
||||
// Write the Containerfile to /tmp/dir/Containerfile
|
||||
const containerFilePath = new URL(
|
||||
importResolve("./Containerfile.prod", import.meta.url)
|
||||
).href.replace("file://", "");
|
||||
// Copy the Containerfile to /tmp/dir/Containerfile
|
||||
await copyFile(containerFilePath, join(tempDir, "Containerfile"));
|
||||
|
||||
const contentHasher = createHash("sha256");
|
||||
contentHasher.update(Buffer.from(entryPointOutputFile.text));
|
||||
contentHasher.update(Buffer.from(workerOutputFile.text));
|
||||
// Sort the dependencies by key to ensure consistent hashing
|
||||
const sortedDependencies = Object.fromEntries(
|
||||
Object.entries(dependencies).sort(([a], [b]) => a.localeCompare(b))
|
||||
);
|
||||
|
||||
contentHasher.update(Buffer.from(JSON.stringify(sortedDependencies)));
|
||||
|
||||
const contentHash = contentHasher.digest("hex");
|
||||
|
||||
return { path: tempDir, contentHash };
|
||||
}
|
||||
|
||||
async function typecheckProject(config: ResolvedConfig, options: DeployCommandOptions) {
|
||||
const createAuthCodeSpinner = spinner();
|
||||
createAuthCodeSpinner.start("Typechecking project");
|
||||
|
||||
await setTimeout(2000);
|
||||
|
||||
createAuthCodeSpinner.stop(`Project typechecked with 0 errors`);
|
||||
}
|
||||
|
||||
// Returns the dependencies that are required by the output that are found in output and the CLI package dependencies
|
||||
// Returns the dependency names and the version to use (taken from the CLI deps package.json)
|
||||
function gatherRequiredDependencies(
|
||||
imports: Metafile["outputs"][string]["imports"],
|
||||
externalPackageJson?: { dependencies: Record<string, string> }
|
||||
) {
|
||||
const dependencies: Record<string, string> = {};
|
||||
|
||||
for (const file of imports) {
|
||||
if (file.kind !== "require-call" || !file.external) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const packageName = detectPackageNameFromImportPath(file.path);
|
||||
|
||||
if (dependencies[packageName]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const externalDependencyVersion = (externalPackageJson?.dependencies ?? {})[packageName];
|
||||
|
||||
if (externalDependencyVersion) {
|
||||
dependencies[packageName] = externalDependencyVersion;
|
||||
continue;
|
||||
}
|
||||
|
||||
const internalDependencyVersion = (packageJson.dependencies as Record<string, string>)[
|
||||
packageName
|
||||
];
|
||||
|
||||
if (internalDependencyVersion) {
|
||||
dependencies[packageName] = internalDependencyVersion;
|
||||
}
|
||||
}
|
||||
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
async function ensureLoggedIntoDockerRegistry(
|
||||
registryHost: string,
|
||||
auth: { username: string; password: string }
|
||||
) {
|
||||
const tmpDir = await createTempDir();
|
||||
// Read the current docker config
|
||||
const dockerConfigPath = join(tmpDir, "config.json");
|
||||
|
||||
await writeJSONFile(dockerConfigPath, {
|
||||
auths: {
|
||||
[registryHost]: {
|
||||
auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
logger.debug(`Writing docker config to ${dockerConfigPath}`);
|
||||
|
||||
return tmpDir;
|
||||
}
|
||||
@@ -25,12 +25,13 @@ import { z } from "zod";
|
||||
import * as packageJson from "../../package.json";
|
||||
import { CliApiClient } from "../apiClient";
|
||||
import { CommonCommandOptions } from "../cli/common.js";
|
||||
import { BackgroundWorker, BackgroundWorkerCoordinator } from "../dev/backgroundWorker.js";
|
||||
import { BackgroundWorker, BackgroundWorkerCoordinator } from "../workers/dev/backgroundWorker.js";
|
||||
import { getConfigPath, readConfig } from "../utilities/configFiles";
|
||||
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
|
||||
import { logger } from "../utilities/logger.js";
|
||||
import { isLoggedIn } from "../utilities/session.js";
|
||||
import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
|
||||
import { detectPackageNameFromImportPath } from "../utilities/installPackages";
|
||||
|
||||
let apiClient: CliApiClient | undefined;
|
||||
|
||||
@@ -643,20 +644,3 @@ function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) {
|
||||
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
// Expects path to be in the format:
|
||||
// - source-map-support/register.js
|
||||
// - @opentelemetry/api
|
||||
// - zod
|
||||
//
|
||||
// With the result being:
|
||||
// - source-map-support
|
||||
// - @opentelemetry/api
|
||||
// - zod
|
||||
function detectPackageNameFromImportPath(path: string): string {
|
||||
if (path.startsWith("@")) {
|
||||
return path.split("/").slice(0, 2).join("/");
|
||||
} else {
|
||||
return path.split("/")[0] as string;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import fsSync from "fs";
|
||||
import fsModule, { writeFile } from "fs/promises";
|
||||
import fs from "node:fs";
|
||||
import pathModule from "path";
|
||||
import { tmpdir } from "node:os";
|
||||
import pathModule from "node:path";
|
||||
|
||||
// Creates a file at the given path, if the directory doesn't exist it will be created
|
||||
export async function createFile(path: string, contents: string): Promise<string> {
|
||||
@@ -52,8 +53,22 @@ export async function readJSONFile(path: string) {
|
||||
return JSON.parse(fileContents);
|
||||
}
|
||||
|
||||
export async function safeFeadJSONFile(path: string) {
|
||||
try {
|
||||
const fileExists = await pathExists(path);
|
||||
|
||||
if (!fileExists) return;
|
||||
|
||||
const fileContents = await readFile(path);
|
||||
|
||||
return JSON.parse(fileContents);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeJSONFile(path: string, json: any) {
|
||||
await writeFile(path, JSON.stringify(json, null, 2));
|
||||
await writeFile(path, JSON.stringify(json), "utf8");
|
||||
}
|
||||
|
||||
export function readJSONFileSync(path: string) {
|
||||
@@ -69,3 +84,14 @@ export function safeDeleteFileSync(path: string) {
|
||||
// ignore error
|
||||
}
|
||||
}
|
||||
|
||||
// Create a temporary directory within the OS's temp directory
|
||||
export async function createTempDir(): Promise<string> {
|
||||
// Generate a unique temp directory path
|
||||
const tempDirPath: string = pathModule.join(tmpdir(), "trigger-");
|
||||
|
||||
// Create the temp directory synchronously and return the path
|
||||
const directory = await fsModule.mkdtemp(tempDirPath);
|
||||
|
||||
return directory;
|
||||
}
|
||||
|
||||
@@ -90,3 +90,20 @@ async function getPackageVersion(path: string) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Expects path to be in the format:
|
||||
// - source-map-support/register.js
|
||||
// - @opentelemetry/api
|
||||
// - zod
|
||||
//
|
||||
// With the result being:
|
||||
// - source-map-support
|
||||
// - @opentelemetry/api
|
||||
// - zod
|
||||
export function detectPackageNameFromImportPath(path: string): string {
|
||||
if (path.startsWith("@")) {
|
||||
return path.split("/").slice(0, 2).join("/");
|
||||
} else {
|
||||
return path.split("/")[0] as string;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,10 @@ export async function isLoggedIn() {
|
||||
return {
|
||||
ok: false as const,
|
||||
error: userData.error,
|
||||
config: {
|
||||
apiUrl: config.apiUrl,
|
||||
accessToken: config.accessToken,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import { Evt } from "evt";
|
||||
import { ChildProcess, fork } from "node:child_process";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import terminalLink from "terminal-link";
|
||||
import { safeDeleteFileSync } from "../utilities/fileSystem.js";
|
||||
import { installPackages } from "../utilities/installPackages.js";
|
||||
import { logger } from "../utilities/logger.js";
|
||||
import { safeDeleteFileSync } from "../../utilities/fileSystem.js";
|
||||
import { installPackages } from "../../utilities/installPackages.js";
|
||||
import { logger } from "../../utilities/logger.js";
|
||||
|
||||
export type CurrentWorkers = BackgroundWorkerCoordinator["currentWorkers"];
|
||||
export class BackgroundWorkerCoordinator {
|
||||
+2
-3
@@ -31,13 +31,12 @@ import {
|
||||
taskContextManager,
|
||||
workerToChildMessages,
|
||||
type BackgroundWorkerProperties,
|
||||
type TracingDiagnosticLogLevel,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import * as packageJson from "../../package.json";
|
||||
import * as packageJson from "../../../package.json";
|
||||
|
||||
import { Resource } from "@opentelemetry/resources";
|
||||
import { flattenAttributes } from "@trigger.dev/core/v3";
|
||||
import { TaskMetadataWithFunctions } from "../types.js";
|
||||
import { TaskMetadataWithFunctions } from "../../types.js";
|
||||
|
||||
const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
|
||||
const consoleInterceptor = new ConsoleInterceptor(otelLogger);
|
||||
+3
-2
@@ -17,7 +17,7 @@ import {
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { Evt } from "evt";
|
||||
import { ChildProcess, fork } from "node:child_process";
|
||||
import { safeDeleteFileSync } from "../utilities/fileSystem";
|
||||
import { safeDeleteFileSync } from "../../utilities/fileSystem";
|
||||
|
||||
class UnexpectedExitError extends Error {
|
||||
constructor(public code: number) {
|
||||
@@ -88,7 +88,7 @@ export class ProdBackgroundWorker {
|
||||
safeDeleteFileSync(`${this.path}.map`);
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
async initialize(options?: { env?: Record<string, string> }) {
|
||||
if (this._initialized) {
|
||||
throw new Error("Worker already initialized");
|
||||
}
|
||||
@@ -100,6 +100,7 @@ export class ProdBackgroundWorker {
|
||||
stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
|
||||
env: {
|
||||
...this.params.env,
|
||||
...options?.env,
|
||||
},
|
||||
});
|
||||
|
||||
+40
-9
@@ -6,7 +6,7 @@ import {
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { HttpReply, getTextBody, SimpleLogger, getRandomPortNumber } from "@trigger.dev/core-apps";
|
||||
import { createServer } from "node:http";
|
||||
import { ProdBackgroundWorker } from "./prod/backgroundWorker";
|
||||
import { ProdBackgroundWorker } from "./backgroundWorker";
|
||||
|
||||
const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
||||
const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
||||
@@ -24,8 +24,8 @@ class ProdWorker {
|
||||
private projectDir = process.env.TRIGGER_PROJECT_DIR!;
|
||||
private projectRef = process.env.TRIGGER_PROJECT_REF!;
|
||||
private envId = process.env.TRIGGER_ENV_ID!;
|
||||
private cliPackageVersion = process.env.TRIGGER_CLI_PACKAGE_VERSION!;
|
||||
private attemptId = process.env.TRIGGER_ATTEMPT_ID || "index-only";
|
||||
private deploymentId = process.env.TRIGGER_DEPLOYMENT_ID!;
|
||||
|
||||
private executing = false;
|
||||
private completed = false;
|
||||
@@ -44,7 +44,7 @@ class ProdWorker {
|
||||
) {
|
||||
this.#coordinatorSocket = this.#createCoordinatorSocket();
|
||||
|
||||
this.#backgroundWorker = new ProdBackgroundWorker(this.#getWorkerEntryPath(this.contentHash), {
|
||||
this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
|
||||
projectDir: this.projectDir,
|
||||
env: {
|
||||
TRIGGER_API_URL: this.apiUrl,
|
||||
@@ -73,6 +73,18 @@ class ProdWorker {
|
||||
}
|
||||
|
||||
#createCoordinatorSocket() {
|
||||
logger.log("connecting to coordinator", {
|
||||
host: COORDINATOR_HOST,
|
||||
port: COORDINATOR_PORT,
|
||||
deploymentId: this.deploymentId,
|
||||
podName: POD_NAME,
|
||||
machineName: MACHINE_NAME,
|
||||
contentHash: this.contentHash,
|
||||
projectRef: this.projectRef,
|
||||
attemptId: this.attemptId,
|
||||
envId: this.envId,
|
||||
});
|
||||
|
||||
const coordinatorConnection = new ZodSocketConnection({
|
||||
namespace: "prod-worker",
|
||||
host: COORDINATOR_HOST,
|
||||
@@ -83,10 +95,10 @@ class ProdWorker {
|
||||
"x-machine-name": MACHINE_NAME,
|
||||
"x-pod-name": POD_NAME,
|
||||
"x-trigger-content-hash": this.contentHash,
|
||||
"x-trigger-cli-package-version": this.cliPackageVersion,
|
||||
"x-trigger-project-ref": this.projectRef,
|
||||
"x-trigger-attempt-id": this.attemptId,
|
||||
"x-trigger-env-id": this.envId,
|
||||
"x-trigger-deployment-id": this.deploymentId,
|
||||
},
|
||||
handlers: {
|
||||
RESUME: async (message) => {
|
||||
@@ -131,6 +143,7 @@ class ProdWorker {
|
||||
|
||||
const { success } = await socket.emitWithAck("INDEX_TASKS", {
|
||||
version: "v1",
|
||||
deploymentId: this.deploymentId,
|
||||
...taskResources,
|
||||
});
|
||||
|
||||
@@ -259,12 +272,13 @@ class ProdWorker {
|
||||
return httpServer;
|
||||
}
|
||||
|
||||
#getWorkerEntryPath(contentHash: string) {
|
||||
return `${contentHash}.mjs`;
|
||||
}
|
||||
|
||||
async #initializeWorker() {
|
||||
await this.#backgroundWorker.initialize();
|
||||
// Make an API call for the env vars
|
||||
// Don't use ApiClient again
|
||||
// Pass those into this.#backgroundWorker.initialize()
|
||||
const envVars = await this.#fetchEnvironmentVariables();
|
||||
|
||||
await this.#backgroundWorker.initialize({ env: envVars });
|
||||
|
||||
let packageVersion: string | undefined;
|
||||
|
||||
@@ -294,6 +308,23 @@ class ProdWorker {
|
||||
};
|
||||
}
|
||||
|
||||
async #fetchEnvironmentVariables(): Promise<Record<string, string>> {
|
||||
const response = await fetch(`${this.apiUrl}/api/v1/projects/${this.projectRef}/envvars`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
return data?.variables ?? {};
|
||||
}
|
||||
|
||||
start() {
|
||||
this.#httpServer.listen(this.#httpPort, this.host);
|
||||
}
|
||||
+8
-21
@@ -1,20 +1,9 @@
|
||||
// import "source-map-support/register";
|
||||
import { TracingSDK } from "@trigger.dev/core/v3";
|
||||
// import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
|
||||
import { type TracingSDK } from "@trigger.dev/core/v3";
|
||||
import "source-map-support/register.js";
|
||||
|
||||
// IMPORTANT: this needs to be the first import to work properly
|
||||
// WARNING: [WARNING] Constructing "ImportInTheMiddle" will crash at run-time because it's an import namespace object, not a constructor [call-import-namespace]
|
||||
// TODO: https://github.com/open-telemetry/opentelemetry-js/issues/3954
|
||||
const tracingSDK = new TracingSDK({
|
||||
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
|
||||
resource: new Resource({
|
||||
[SemanticInternalAttributes.CLI_VERSION]: packageJson.version,
|
||||
}),
|
||||
instrumentations: [
|
||||
// new OpenAIInstrumentation(),
|
||||
],
|
||||
diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
|
||||
});
|
||||
__REGISTER_TRACING__;
|
||||
declare const __REGISTER_TRACING__: unknown;
|
||||
declare const tracingSDK: TracingSDK;
|
||||
|
||||
const otelTracer = tracingSDK.getTracer("trigger-prod-worker", packageJson.version);
|
||||
const otelLogger = tracingSDK.getLogger("trigger-prod-worker", packageJson.version);
|
||||
@@ -22,8 +11,8 @@ const otelLogger = tracingSDK.getLogger("trigger-prod-worker", packageJson.versi
|
||||
import { SpanKind } from "@opentelemetry/api";
|
||||
import {
|
||||
ConsoleInterceptor,
|
||||
ProdRuntimeManager,
|
||||
OtelTaskLogger,
|
||||
ProdRuntimeManager,
|
||||
SemanticInternalAttributes,
|
||||
TaskMetadataWithFilePath,
|
||||
TaskRunContext,
|
||||
@@ -43,12 +32,10 @@ import {
|
||||
workerToChildMessages,
|
||||
type BackgroundWorkerProperties,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import * as packageJson from "../package.json";
|
||||
import * as packageJson from "../../../package.json";
|
||||
|
||||
import { Resource } from "@opentelemetry/resources";
|
||||
import { flattenAttributes } from "@trigger.dev/core/v3";
|
||||
import { TaskMetadataWithFunctions } from "./types";
|
||||
import { TracingDiagnosticLogLevel } from "@trigger.dev/core/v3";
|
||||
import { TaskMetadataWithFunctions } from "../../types";
|
||||
|
||||
const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
|
||||
const consoleInterceptor = new ConsoleInterceptor(otelLogger);
|
||||
@@ -1,16 +0,0 @@
|
||||
import { defineConfig } from "tsup";
|
||||
|
||||
export default defineConfig({
|
||||
clean: false,
|
||||
dts: true,
|
||||
tsconfig: "tsconfig.json",
|
||||
splitting: false,
|
||||
entry: ["src/prod-facade.ts"],
|
||||
format: ["esm"],
|
||||
minify: false,
|
||||
metafile: false,
|
||||
sourcemap: true,
|
||||
target: "esnext",
|
||||
outDir: "dist",
|
||||
noExternal: ["zod", /traceloop/, /opentelemetry/],
|
||||
});
|
||||
@@ -5,11 +5,16 @@ export default defineConfig({
|
||||
dts: false,
|
||||
tsconfig: "tsconfig.json",
|
||||
splitting: false,
|
||||
entry: ["src/dev/worker-facade.ts", "src/dev/register-tracing.ts"],
|
||||
entry: [
|
||||
"src/workers/dev/worker-facade.ts",
|
||||
"src/workers/common/register-tracing.ts",
|
||||
"src/workers/prod/worker-facade.ts",
|
||||
"src/workers/prod/entry-point.ts",
|
||||
],
|
||||
format: ["esm"],
|
||||
minify: false,
|
||||
metafile: false,
|
||||
sourcemap: false,
|
||||
target: "esnext",
|
||||
outDir: "dist",
|
||||
outDir: "dist/workers",
|
||||
});
|
||||
@@ -135,6 +135,8 @@ export class ProviderShell implements Provider {
|
||||
contentHash: message.contentHash,
|
||||
imageTag: message.imageTag,
|
||||
envId: message.envId,
|
||||
apiKey: message.apiKey,
|
||||
apiUrl: message.apiUrl,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("task index failed", error);
|
||||
|
||||
@@ -12,6 +12,7 @@ export type WhoAmIResponse = z.infer<typeof WhoAmIResponseSchema>;
|
||||
export const GetProjectDevResponse = z.object({
|
||||
apiKey: z.string(),
|
||||
name: z.string(),
|
||||
apiUrl: z.string(),
|
||||
});
|
||||
|
||||
export type GetProjectDevResponse = z.infer<typeof GetProjectDevResponse>;
|
||||
@@ -89,15 +90,67 @@ export type GetEnvironmentVariablesResponseBody = z.infer<
|
||||
typeof GetEnvironmentVariablesResponseBody
|
||||
>;
|
||||
|
||||
export const CreateImageDetailsRequestBody = z.object({
|
||||
metadata: ImageDetailsMetadata,
|
||||
export const StartDeploymentIndexingRequestBody = z.object({
|
||||
imageReference: z.string(),
|
||||
});
|
||||
|
||||
export type CreateImageDetailsRequestBody = z.infer<typeof CreateImageDetailsRequestBody>;
|
||||
export type StartDeploymentIndexingRequestBody = z.infer<typeof StartDeploymentIndexingRequestBody>;
|
||||
|
||||
export const CreateImageDetailsResponse = z.object({
|
||||
export const StartDeploymentIndexingResponseBody = z.object({
|
||||
id: z.string(),
|
||||
contentHash: z.string(),
|
||||
});
|
||||
|
||||
export type CreateImageDetailsResponse = z.infer<typeof CreateImageDetailsResponse>;
|
||||
export type StartDeploymentIndexingResponseBody = z.infer<
|
||||
typeof StartDeploymentIndexingResponseBody
|
||||
>;
|
||||
|
||||
export const ExternalBuildData = z.object({
|
||||
buildId: z.string(),
|
||||
buildToken: z.string(),
|
||||
projectId: z.string(),
|
||||
});
|
||||
|
||||
export type ExternalBuildData = z.infer<typeof ExternalBuildData>;
|
||||
|
||||
export const InitializeDeploymentResponseBody = z.object({
|
||||
id: z.string(),
|
||||
contentHash: z.string(),
|
||||
shortCode: z.string(),
|
||||
version: z.string(),
|
||||
imageTag: z.string(),
|
||||
externalBuildData: ExternalBuildData.optional(),
|
||||
});
|
||||
|
||||
export type InitializeDeploymentResponseBody = z.infer<typeof InitializeDeploymentResponseBody>;
|
||||
|
||||
export const InitializeDeploymentRequestBody = z.object({
|
||||
contentHash: z.string(),
|
||||
});
|
||||
|
||||
export type InitializeDeploymentRequestBody = z.infer<typeof InitializeDeploymentRequestBody>;
|
||||
|
||||
export const GetDeploymentResponseBody = z.object({
|
||||
id: z.string(),
|
||||
status: z.enum(["PENDING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED"]),
|
||||
contentHash: z.string(),
|
||||
shortCode: z.string(),
|
||||
version: z.string(),
|
||||
imageReference: z.string().optional(),
|
||||
worker: z
|
||||
.object({
|
||||
id: z.string(),
|
||||
version: z.string(),
|
||||
tasks: z.array(
|
||||
z.object({
|
||||
id: z.string(),
|
||||
slug: z.string(),
|
||||
filePath: z.string(),
|
||||
exportName: z.string(),
|
||||
})
|
||||
),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export type GetDeploymentResponseBody = z.infer<typeof GetDeploymentResponseBody>;
|
||||
|
||||
@@ -14,7 +14,7 @@ export type TaskResource = z.infer<typeof TaskResource>;
|
||||
export const BackgroundWorkerMetadata = z.object({
|
||||
packageVersion: z.string(),
|
||||
contentHash: z.string(),
|
||||
cliPackageVersion: z.string(),
|
||||
cliPackageVersion: z.string().optional(),
|
||||
tasks: z.array(TaskResource),
|
||||
});
|
||||
|
||||
|
||||
@@ -62,6 +62,8 @@ export const PlatformToProviderMessages = {
|
||||
imageTag: z.string(),
|
||||
contentHash: z.string(),
|
||||
envId: z.string(),
|
||||
apiKey: z.string(),
|
||||
apiUrl: z.string(),
|
||||
}),
|
||||
},
|
||||
INVOKE: {
|
||||
@@ -111,8 +113,9 @@ export const CoordinatorToPlatformMessages = {
|
||||
version: z.literal("v1").default("v1"),
|
||||
projectRef: z.string(),
|
||||
envId: z.string(),
|
||||
deploymentId: z.string(),
|
||||
metadata: z.object({
|
||||
cliPackageVersion: z.string(),
|
||||
cliPackageVersion: z.string().optional(),
|
||||
contentHash: z.string(),
|
||||
packageVersion: z.string(),
|
||||
tasks: TaskResource.array(),
|
||||
@@ -227,6 +230,7 @@ export const ProdWorkerToCoordinatorMessages = {
|
||||
INDEX_TASKS: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
deploymentId: z.string(),
|
||||
tasks: TaskResource.array(),
|
||||
packageVersion: z.string(),
|
||||
}),
|
||||
@@ -308,10 +312,10 @@ export const CoordinatorToProdWorkerMessages = {
|
||||
};
|
||||
|
||||
export const ProdWorkerSocketData = z.object({
|
||||
cliPackageVersion: z.string(),
|
||||
contentHash: z.string(),
|
||||
projectRef: z.string(),
|
||||
envId: z.string(),
|
||||
attemptId: z.string(),
|
||||
podName: z.string(),
|
||||
deploymentId: z.string(),
|
||||
});
|
||||
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- A unique constraint covering the columns `[backgroundWorkerId]` on the table `ImageDetails` will be added. If there are existing duplicate values, this will fail.
|
||||
|
||||
*/
|
||||
-- CreateEnum
|
||||
CREATE TYPE "WorkerDeploymentStatus" AS ENUM ('PENDING', 'DEPLOYING', 'DEPLOYED', 'FAILED', 'CANCELED');
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "WorkerDeployment" (
|
||||
"id" TEXT NOT NULL,
|
||||
"friendlyId" TEXT NOT NULL,
|
||||
"shortCode" TEXT NOT NULL,
|
||||
"version" TEXT NOT NULL,
|
||||
"buildId" TEXT NOT NULL,
|
||||
"buildToken" TEXT NOT NULL,
|
||||
"status" "WorkerDeploymentStatus" NOT NULL DEFAULT 'PENDING',
|
||||
"projectId" TEXT NOT NULL,
|
||||
"environmentId" TEXT NOT NULL,
|
||||
"workerId" TEXT,
|
||||
"deployedAt" TIMESTAMP(3),
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "WorkerDeployment_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "WorkerDeploymentPromotion" (
|
||||
"id" TEXT NOT NULL,
|
||||
"label" TEXT NOT NULL,
|
||||
"deploymentId" TEXT NOT NULL,
|
||||
"environmentId" TEXT NOT NULL,
|
||||
|
||||
CONSTRAINT "WorkerDeploymentPromotion_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "WorkerDeployment_friendlyId_key" ON "WorkerDeployment"("friendlyId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "WorkerDeployment_workerId_key" ON "WorkerDeployment"("workerId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "WorkerDeployment_projectId_shortCode_key" ON "WorkerDeployment"("projectId", "shortCode");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "WorkerDeployment_environmentId_version_key" ON "WorkerDeployment"("environmentId", "version");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "WorkerDeploymentPromotion_environmentId_label_key" ON "WorkerDeploymentPromotion"("environmentId", "label");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "ImageDetails_backgroundWorkerId_key" ON "ImageDetails"("backgroundWorkerId");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "WorkerDeployment" ADD CONSTRAINT "WorkerDeployment_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "WorkerDeployment" ADD CONSTRAINT "WorkerDeployment_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "WorkerDeployment" ADD CONSTRAINT "WorkerDeployment_workerId_fkey" FOREIGN KEY ("workerId") REFERENCES "BackgroundWorker"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "WorkerDeploymentPromotion" ADD CONSTRAINT "WorkerDeploymentPromotion_deploymentId_fkey" FOREIGN KEY ("deploymentId") REFERENCES "WorkerDeployment"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "WorkerDeploymentPromotion" ADD CONSTRAINT "WorkerDeploymentPromotion_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `buildId` on the `WorkerDeployment` table. All the data in the column will be lost.
|
||||
- You are about to drop the column `buildToken` on the `WorkerDeployment` table. All the data in the column will be lost.
|
||||
|
||||
*/
|
||||
-- AlterEnum
|
||||
ALTER TYPE "WorkerDeploymentStatus" ADD VALUE 'BUILDING';
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "WorkerDeployment" DROP COLUMN "buildId",
|
||||
DROP COLUMN "buildToken",
|
||||
ADD COLUMN "externalBuildData" JSONB;
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- Added the required column `contentHash` to the `WorkerDeployment` table without a default value. This is not possible if the table is not empty.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "WorkerDeployment" ADD COLUMN "contentHash" TEXT NOT NULL;
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the `ImageDetails` table. If the table is not empty, all the data it contains will be lost.
|
||||
|
||||
*/
|
||||
-- DropForeignKey
|
||||
ALTER TABLE "ImageDetails" DROP CONSTRAINT "ImageDetails_backgroundWorkerId_fkey";
|
||||
|
||||
-- DropForeignKey
|
||||
ALTER TABLE "ImageDetails" DROP CONSTRAINT "ImageDetails_projectId_fkey";
|
||||
|
||||
-- DropForeignKey
|
||||
ALTER TABLE "ImageDetails" DROP CONSTRAINT "ImageDetails_runtimeEnvironmentId_fkey";
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "WorkerDeployment" ADD COLUMN "imageReference" TEXT;
|
||||
|
||||
-- DropTable
|
||||
DROP TABLE "ImageDetails";
|
||||
@@ -367,30 +367,31 @@ model RuntimeEnvironment {
|
||||
|
||||
tunnelId String?
|
||||
|
||||
endpoints Endpoint[]
|
||||
jobVersions JobVersion[]
|
||||
events EventRecord[]
|
||||
jobRuns JobRun[]
|
||||
requestDeliveries HttpSourceRequestDelivery[]
|
||||
jobAliases JobAlias[]
|
||||
JobQueue JobQueue[]
|
||||
sources TriggerSource[]
|
||||
eventDispatchers EventDispatcher[]
|
||||
scheduleSources ScheduleSource[]
|
||||
ExternalAccount ExternalAccount[]
|
||||
httpEndpointEnvironments TriggerHttpEndpointEnvironment[]
|
||||
concurrencyLimitGroups ConcurrencyLimitGroup[]
|
||||
keyValueItems KeyValueItem[]
|
||||
webhookEnvironments WebhookEnvironment[]
|
||||
webhookRequestDeliveries WebhookRequestDelivery[]
|
||||
backgroundWorkers BackgroundWorker[]
|
||||
backgroundWorkerTasks BackgroundWorkerTask[]
|
||||
taskRuns TaskRun[]
|
||||
taskQueues TaskQueue[]
|
||||
batchTaskRuns BatchTaskRun[]
|
||||
environmentVariableValues EnvironmentVariableValue[]
|
||||
imageDetails ImageDetails[]
|
||||
checkpoints Checkpoint[]
|
||||
endpoints Endpoint[]
|
||||
jobVersions JobVersion[]
|
||||
events EventRecord[]
|
||||
jobRuns JobRun[]
|
||||
requestDeliveries HttpSourceRequestDelivery[]
|
||||
jobAliases JobAlias[]
|
||||
JobQueue JobQueue[]
|
||||
sources TriggerSource[]
|
||||
eventDispatchers EventDispatcher[]
|
||||
scheduleSources ScheduleSource[]
|
||||
ExternalAccount ExternalAccount[]
|
||||
httpEndpointEnvironments TriggerHttpEndpointEnvironment[]
|
||||
concurrencyLimitGroups ConcurrencyLimitGroup[]
|
||||
keyValueItems KeyValueItem[]
|
||||
webhookEnvironments WebhookEnvironment[]
|
||||
webhookRequestDeliveries WebhookRequestDelivery[]
|
||||
backgroundWorkers BackgroundWorker[]
|
||||
backgroundWorkerTasks BackgroundWorkerTask[]
|
||||
taskRuns TaskRun[]
|
||||
taskQueues TaskQueue[]
|
||||
batchTaskRuns BatchTaskRun[]
|
||||
environmentVariableValues EnvironmentVariableValue[]
|
||||
checkpoints Checkpoint[]
|
||||
workerDeployments WorkerDeployment[]
|
||||
workerDeploymentPromotions WorkerDeploymentPromotion[]
|
||||
|
||||
@@unique([projectId, slug, orgMemberId])
|
||||
@@unique([projectId, shortcode])
|
||||
@@ -434,8 +435,8 @@ model Project {
|
||||
taskTags TaskTag[]
|
||||
taskQueues TaskQueue[]
|
||||
environmentVariables EnvironmentVariable[]
|
||||
imageDetails ImageDetails[]
|
||||
checkpoints Checkpoint[]
|
||||
WorkerDeployment WorkerDeployment[]
|
||||
}
|
||||
|
||||
enum ProjectVersion {
|
||||
@@ -1517,10 +1518,11 @@ model BackgroundWorker {
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
tasks BackgroundWorkerTask[]
|
||||
attempts TaskRunAttempt[]
|
||||
lockedRuns TaskRun[]
|
||||
imageDetails ImageDetails[]
|
||||
tasks BackgroundWorkerTask[]
|
||||
attempts TaskRunAttempt[]
|
||||
lockedRuns TaskRun[]
|
||||
|
||||
deployment WorkerDeployment?
|
||||
|
||||
@@unique([projectId, runtimeEnvironmentId, version])
|
||||
}
|
||||
@@ -1892,31 +1894,6 @@ model EnvironmentVariableValue {
|
||||
@@unique([variableId, environmentId])
|
||||
}
|
||||
|
||||
model ImageDetails {
|
||||
id String @id @default(cuid())
|
||||
|
||||
friendlyId String @unique
|
||||
|
||||
contentHash String
|
||||
tag String
|
||||
|
||||
backgroundWorker BackgroundWorker? @relation(fields: [backgroundWorkerId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
backgroundWorkerId String?
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
projectId String
|
||||
|
||||
runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
runtimeEnvironmentId String
|
||||
|
||||
metadata Json
|
||||
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@unique([projectId, runtimeEnvironmentId, contentHash])
|
||||
}
|
||||
|
||||
model Checkpoint {
|
||||
id String @id @default(cuid())
|
||||
|
||||
@@ -1943,3 +1920,65 @@ enum CheckpointType {
|
||||
DOCKER
|
||||
KUBERNETES
|
||||
}
|
||||
|
||||
model WorkerDeployment {
|
||||
id String @id @default(cuid())
|
||||
|
||||
contentHash String
|
||||
friendlyId String @unique
|
||||
shortCode String
|
||||
version String
|
||||
|
||||
imageReference String?
|
||||
|
||||
externalBuildData Json?
|
||||
|
||||
status WorkerDeploymentStatus @default(PENDING)
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
projectId String
|
||||
|
||||
environment RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
environmentId String
|
||||
|
||||
worker BackgroundWorker? @relation(fields: [workerId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
workerId String? @unique
|
||||
|
||||
deployedAt DateTime?
|
||||
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
promotions WorkerDeploymentPromotion[]
|
||||
|
||||
@@unique([projectId, shortCode])
|
||||
@@unique([environmentId, version])
|
||||
}
|
||||
|
||||
enum WorkerDeploymentStatus {
|
||||
PENDING
|
||||
/// This is the status when the image is being built
|
||||
BUILDING
|
||||
/// This is the status when the image is built and we are waiting for the indexing to finish
|
||||
DEPLOYING
|
||||
/// This is the status when the image is built and indexed, meaning we have everything we need to deploy
|
||||
DEPLOYED
|
||||
FAILED
|
||||
CANCELED
|
||||
}
|
||||
|
||||
model WorkerDeploymentPromotion {
|
||||
id String @id @default(cuid())
|
||||
|
||||
/// This is the promotion label, e.g. "current"
|
||||
label String
|
||||
|
||||
deployment WorkerDeployment @relation(fields: [deploymentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
deploymentId String
|
||||
|
||||
environment RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
|
||||
environmentId String
|
||||
|
||||
// Only one promotion per environment can be active at a time
|
||||
@@unique([environmentId, label])
|
||||
}
|
||||
|
||||
Generated
+149
-7
@@ -153,6 +153,7 @@ importers:
|
||||
'@codemirror/view': ^6.5.0
|
||||
'@conform-to/react': ^0.6.1
|
||||
'@conform-to/zod': ^0.6.1
|
||||
'@depot/sdk-node': ^0.5.0
|
||||
'@headlessui/react': ^1.7.8
|
||||
'@heroicons/react': ^2.0.12
|
||||
'@highlight-run/node': ^3.1.0
|
||||
@@ -275,6 +276,7 @@ importers:
|
||||
ioredis: ^5.3.2
|
||||
isbot: ^3.6.5
|
||||
jsonpointer: ^5.0.1
|
||||
jwt-decode: ^4.0.0
|
||||
lodash.omit: ^4.5.0
|
||||
lucide-react: ^0.229.0
|
||||
marked: ^4.0.18
|
||||
@@ -348,6 +350,7 @@ importers:
|
||||
'@codemirror/view': 6.7.2
|
||||
'@conform-to/react': 0.6.1_react@18.2.0
|
||||
'@conform-to/zod': 0.6.1_zod@3.22.3
|
||||
'@depot/sdk-node': 0.5.0
|
||||
'@headlessui/react': 1.7.8_biqbaboplfbrettd7655fr4n2y
|
||||
'@heroicons/react': 2.0.13_react@18.2.0
|
||||
'@highlight-run/node': 3.1.0
|
||||
@@ -419,6 +422,7 @@ importers:
|
||||
ioredis: 5.3.2
|
||||
isbot: 3.6.5
|
||||
jsonpointer: 5.0.1
|
||||
jwt-decode: 4.0.0
|
||||
lodash.omit: 4.5.0
|
||||
lucide-react: 0.229.0_react@18.2.0
|
||||
marked: 4.2.5
|
||||
@@ -553,7 +557,7 @@ importers:
|
||||
devDependencies:
|
||||
eslint: 8.31.0
|
||||
eslint-config-prettier: 8.6.0_eslint@8.31.0
|
||||
eslint-config-turbo: 1.12.4_eslint@8.31.0
|
||||
eslint-config-turbo: 1.12.5_eslint@8.31.0
|
||||
eslint-plugin-react: 7.31.8_eslint@8.31.0
|
||||
typescript: 4.9.4
|
||||
|
||||
@@ -1028,6 +1032,7 @@ importers:
|
||||
packages/cli-v3:
|
||||
specifiers:
|
||||
'@clack/prompts': ^0.7.0
|
||||
'@depot/cli': 0.0.1-cli.2.55.0
|
||||
'@opentelemetry/api': ^1.7.0
|
||||
'@opentelemetry/api-logs': ^0.48.0
|
||||
'@opentelemetry/auto-instrumentations-node': ^0.40.3
|
||||
@@ -1094,6 +1099,7 @@ importers:
|
||||
source-map-support: ^0.5.21
|
||||
supports-color: ^9.4.0
|
||||
terminal-link: ^3.0.0
|
||||
tiny-invariant: ^1.2.0
|
||||
tsup: ^8.0.1
|
||||
type-fest: ^3.6.0
|
||||
typescript: ^5.3.3
|
||||
@@ -1105,6 +1111,7 @@ importers:
|
||||
zod: 3.22.3
|
||||
dependencies:
|
||||
'@clack/prompts': 0.7.0
|
||||
'@depot/cli': 0.0.1-cli.2.55.0
|
||||
'@opentelemetry/api': 1.7.0
|
||||
'@opentelemetry/api-logs': 0.48.0
|
||||
'@opentelemetry/auto-instrumentations-node': 0.40.3_rr4fxqkzz7nhz3auplkqn4p6em
|
||||
@@ -1156,6 +1163,7 @@ importers:
|
||||
source-map-support: 0.5.21
|
||||
supports-color: 9.4.0
|
||||
terminal-link: 3.0.0
|
||||
tiny-invariant: 1.3.1
|
||||
update-check: 1.5.4
|
||||
url: 0.11.1
|
||||
ws: 8.12.0
|
||||
@@ -5582,6 +5590,10 @@ packages:
|
||||
resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}
|
||||
dev: true
|
||||
|
||||
/@bufbuild/protobuf/1.7.2:
|
||||
resolution: {integrity: sha512-i5GE2Dk5ekdlK1TR7SugY4LWRrKSfb5T1Qn4unpIMbfxoeGKERKQ59HG3iYewacGD10SR7UzevfPnh6my4tNmQ==}
|
||||
dev: false
|
||||
|
||||
/@bundled-es-modules/cookie/2.0.0:
|
||||
resolution: {integrity: sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==}
|
||||
dependencies:
|
||||
@@ -6003,12 +6015,137 @@ packages:
|
||||
zod: 3.22.3
|
||||
dev: false
|
||||
|
||||
/@connectrpc/connect-node/0.13.2_@bufbuild+protobuf@1.7.2:
|
||||
resolution: {integrity: sha512-dAoBuQ+fYFw22KYgZXPw1t19i6TRwNmk6c6XsOYLhBl1Y5vzXoMfMX2jm7lmkmLQFIcM9Xc2EYoUKtjNXH5bjw==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
peerDependencies:
|
||||
'@bufbuild/protobuf': ^1.2.1
|
||||
dependencies:
|
||||
'@bufbuild/protobuf': 1.7.2
|
||||
'@connectrpc/connect': 0.13.2_@bufbuild+protobuf@1.7.2
|
||||
undici: 5.25.4
|
||||
dev: false
|
||||
|
||||
/@connectrpc/connect/0.13.2_@bufbuild+protobuf@1.7.2:
|
||||
resolution: {integrity: sha512-KZg6EH8gYnQZm/d2IXXMVB2mom/A1dCD8+7JScm2tKN9OQyQSeCJOLqtv/M4M5XVS0Y9JM2/VCbiUwfhl9Rqmg==}
|
||||
peerDependencies:
|
||||
'@bufbuild/protobuf': ^1.2.1
|
||||
dependencies:
|
||||
'@bufbuild/protobuf': 1.7.2
|
||||
dev: false
|
||||
|
||||
/@cspotcode/source-map-support/0.8.1:
|
||||
resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
|
||||
engines: {node: '>=12'}
|
||||
dependencies:
|
||||
'@jridgewell/trace-mapping': 0.3.9
|
||||
|
||||
/@depot/cli-darwin-arm64/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-1uRGmCqd+kuHK2fub0oVt+SctP3mwNX15y17oMilIOpvvJtwMvjwBI73qt2aN0jbx0e3xrDiv5GD4GMzsqNGEw==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-darwin-x64/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-3Z3fczSJ1WsxhgS0Ui//9CIk6yJS2+JKlwmZ44EbQIPQ4ZyR+ZZDoJa8pmZdLDehRxVLjeXkUdsRPD2K6yrCqA==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-linux-arm/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-T89KJJjk9By50L6pCrDDAmJFgM3V8DzppMmM7cA1g3biLTjeJZLYgwTrBXu3CfpWMqqLWWOvZ/je8myx7PwBOA==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-linux-arm64/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-GsnItZk9jawX4f6xNI7d0FveSLNIWNVo7udzmtYmPkGo0hsiNfOxzQixR8cusJKnTOo9wYiHx4pfYNB3Hjs7Ng==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-linux-ia32/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-Eunct4bkpN1TVhMt8xu9XtURzipz9yEfXidQoX/vijMLZemsvwcMVnuCmUkfyq/m5DJ1ZN8APHvj+9NMrERfaQ==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [ia32]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-linux-x64/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-G93+XCpzGHYfH5LeT+3A0J3SF7GGQGnjMVoH8w7m5b6GtTu7SFGfat3a2/M07SOtGXa5tz3O56wrgR1jDPc1Fg==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-win32-arm64/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-y7K48ngTqvSCyHItxW0kapVfYyqtWNYZVeEJ344tSuvq6H3bt8mLWfjH+WF9fT14OAmR1jOt0+ykpta45ehKrA==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-win32-ia32/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-hsO3NKWZMeiFgtFAkEsj1vrHG49N0fLjkhEEJ7DrUV0OBMUifkScE66E+9+pWJoxWq3B3Bmc/9OPc0XVMfYekQ==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [ia32]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli-win32-x64/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-QKhpSpa1QwWTf79iMNxUR17EFbe7hT/Bs9mhTcR6NiyOB0GeWBpX3ejULpQcfHpVjBBHIXTW/TJ/Unx4N80cIA==}
|
||||
engines: {node: '>=14'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/@depot/cli/0.0.1-cli.2.55.0:
|
||||
resolution: {integrity: sha512-Y5hlyMxJ8wKeXQKQaYOMCVVzLO42qFHaybcgmCjiZC4CSXkX0+nOL8daoH3Sy0M4U0RcbUijNMJ51oGFq2As9g==}
|
||||
engines: {node: '>=14'}
|
||||
hasBin: true
|
||||
requiresBuild: true
|
||||
optionalDependencies:
|
||||
'@depot/cli-darwin-arm64': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-darwin-x64': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-linux-arm': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-linux-arm64': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-linux-ia32': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-linux-x64': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-win32-arm64': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-win32-ia32': 0.0.1-cli.2.55.0
|
||||
'@depot/cli-win32-x64': 0.0.1-cli.2.55.0
|
||||
dev: false
|
||||
|
||||
/@depot/sdk-node/0.5.0:
|
||||
resolution: {integrity: sha512-Pl9Yji00B6uQpMm5xZU4/eZbzaBo72N6c534ZqLI6nbOEIPH0VX27TVF9lFF3Fl3GHFZ+6X9laJtDn2DrLS2DQ==}
|
||||
dependencies:
|
||||
'@bufbuild/protobuf': 1.7.2
|
||||
'@connectrpc/connect': 0.13.2_@bufbuild+protobuf@1.7.2
|
||||
'@connectrpc/connect-node': 0.13.2_@bufbuild+protobuf@1.7.2
|
||||
dev: false
|
||||
|
||||
/@discoveryjs/json-ext/0.5.7:
|
||||
resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
@@ -23839,13 +23976,13 @@ packages:
|
||||
eslint: 8.45.0
|
||||
dev: true
|
||||
|
||||
/eslint-config-turbo/1.12.4_eslint@8.31.0:
|
||||
resolution: {integrity: sha512-5hqEaV6PNmAYLL4RTmq74OcCt8pgzOLnfDVPG/7PUXpQ0Mpz0gr926oCSFukywKKXjdum3VHD84S7Z9A/DqTAw==}
|
||||
/eslint-config-turbo/1.12.5_eslint@8.31.0:
|
||||
resolution: {integrity: sha512-wXytbX+vTzQ6rwgM6sIr447tjYJBlRj5V/eBFNGNXw5Xs1R715ppPYhbmxaFbkrWNQSGJsWRrYGAlyq0sT/OsQ==}
|
||||
peerDependencies:
|
||||
eslint: '>6.6.0'
|
||||
dependencies:
|
||||
eslint: 8.31.0
|
||||
eslint-plugin-turbo: 1.12.4_eslint@8.31.0
|
||||
eslint-plugin-turbo: 1.12.5_eslint@8.31.0
|
||||
dev: true
|
||||
|
||||
/eslint-doc-generator/1.4.3_eslint@8.45.0:
|
||||
@@ -24548,8 +24685,8 @@ packages:
|
||||
- typescript
|
||||
dev: true
|
||||
|
||||
/eslint-plugin-turbo/1.12.4_eslint@8.31.0:
|
||||
resolution: {integrity: sha512-3AGmXvH7E4i/XTWqBrcgu+G7YKZJV/8FrEn79kTd50ilNsv+U3nS2IlcCrQB6Xm2m9avGD9cadLzKDR1/UF2+g==}
|
||||
/eslint-plugin-turbo/1.12.5_eslint@8.31.0:
|
||||
resolution: {integrity: sha512-cXy7mCzAdngBTJIWH4DASXHy0vQpujWDBqRTu0YYqCN/QEGsi3HWM+STZEbPYELdjtm5EsN2HshOSSqWnjdRHg==}
|
||||
peerDependencies:
|
||||
eslint: '>6.6.0'
|
||||
dependencies:
|
||||
@@ -28760,6 +28897,11 @@ packages:
|
||||
safe-buffer: 5.2.1
|
||||
dev: false
|
||||
|
||||
/jwt-decode/4.0.0:
|
||||
resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==}
|
||||
engines: {node: '>=18'}
|
||||
dev: false
|
||||
|
||||
/keyv/3.1.0:
|
||||
resolution: {integrity: sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==}
|
||||
dependencies:
|
||||
@@ -36854,7 +36996,7 @@ packages:
|
||||
dependencies:
|
||||
bs-logger: 0.2.6
|
||||
fast-json-stable-stringify: 2.1.0
|
||||
jest: 29.6.2_@types+node@18.17.1
|
||||
jest: 29.6.2_@types+node@18.15.13
|
||||
jest-util: 29.6.2
|
||||
json5: 2.2.3
|
||||
lodash.memoize: 4.1.2
|
||||
|
||||
@@ -9,7 +9,9 @@
|
||||
"@trigger.dev/sdk": ["../../packages/trigger-sdk/src/index"],
|
||||
"@trigger.dev/sdk/*": ["../../packages/trigger-sdk/src/*"],
|
||||
"@trigger.dev/core": ["../../packages/core/src/index"],
|
||||
"@trigger.dev/core/*": ["../../packages/core/src/*"]
|
||||
"@trigger.dev/core/*": ["../../packages/core/src/*"],
|
||||
"@trigger.dev/core-apps": ["../../packages/core-apps/src/index"],
|
||||
"@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user