From 9945980ab03efd5bfce4a921e5f1c65bf37a64a4 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 18 Mar 2024 18:45:00 +0000 Subject: [PATCH] update kubernetes provider and fix builds again --- apps/coordinator/Containerfile | 1 - apps/coordinator/package.json | 2 +- apps/docker-provider/.env.example | 1 + apps/docker-provider/src/index.ts | 52 ++++---- apps/kubernetes-provider/.env.example | 7 +- apps/kubernetes-provider/Containerfile | 1 - apps/kubernetes-provider/src/index.ts | 116 +++++++++--------- .../v3/services/createCheckpoint.server.ts | 19 ++- .../v3/services/restoreCheckpoint.server.ts | 1 + packages/core-apps/package.json | 7 +- packages/core-apps/src/provider.ts | 42 ++++++- packages/core-apps/tsconfig.build.json | 11 -- packages/core-apps/tsup.config.ts | 9 -- packages/core/src/v3/schemas/schemas.ts | 1 + .../migration.sql | 8 ++ packages/database/prisma/schema.prisma | 1 + pnpm-lock.yaml | 47 +------ 17 files changed, 155 insertions(+), 171 deletions(-) delete mode 100644 packages/core-apps/tsconfig.build.json delete mode 100644 packages/core-apps/tsup.config.ts create mode 100644 packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql diff --git a/apps/coordinator/Containerfile b/apps/coordinator/Containerfile index f2ca3b3e6..0d695d6f8 100644 --- a/apps/coordinator/Containerfile +++ b/apps/coordinator/Containerfile @@ -35,7 +35,6 @@ COPY --from=pruner --chown=node:node /app/out/full/ . COPY --from=dev-deps --chown=node:node /app/ . COPY --chown=node:node turbo.json turbo.json -RUN pnpm run -r --filter '@trigger.dev/core*' build RUN pnpm run -r --filter coordinator build:bundle FROM alpine AS cri-tools diff --git a/apps/coordinator/package.json b/apps/coordinator/package.json index 4cf789c16..b93ee4605 100644 --- a/apps/coordinator/package.json +++ b/apps/coordinator/package.json @@ -6,7 +6,7 @@ "main": "dist/index.cjs", "scripts": { "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"const require = createRequire(import.meta.url);\"", + "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", "build:image": "docker build -f Containerfile . -t coordinator", "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", "start": "tsx src/index.ts", diff --git a/apps/docker-provider/.env.example b/apps/docker-provider/.env.example index cdf3cf9cc..075b6823a 100644 --- a/apps/docker-provider/.env.example +++ b/apps/docker-provider/.env.example @@ -2,6 +2,7 @@ HTTP_SERVER_PORT=8050 PLATFORM_WS_PORT=3030 PLATFORM_SECRET=provider-secret + # Use this if you are on macOS # COORDINATOR_HOST="host.docker.internal" # OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts index 610a03cab..d88d37ac1 100644 --- a/apps/docker-provider/src/index.ts +++ b/apps/docker-provider/src/index.ts @@ -1,6 +1,12 @@ import { $, type ExecaChildProcess, execa } from "execa"; -import { Machine } from "@trigger.dev/core/v3"; -import { SimpleLogger, TaskOperations, ProviderShell } from "@trigger.dev/core-apps"; +import { + SimpleLogger, + TaskOperations, + ProviderShell, + TaskOperationsRestoreOptions, + TaskOperationsCreateOptions, + TaskOperationsIndexOptions, +} from "@trigger.dev/core-apps"; const MACHINE_NAME = process.env.MACHINE_NAME || "local"; const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020; @@ -72,18 +78,12 @@ class DockerTaskOperations implements TaskOperations { }; } - async index(opts: { - contentHash: string; - imageTag: string; - envId: string; - apiKey: string; - apiUrl: string; - }) { + async index(opts: TaskOperationsIndexOptions) { await this.#initialize(); const containerName = this.#getIndexContainerName(opts.contentHash); - logger.log(`Indexing task ${opts.imageTag}`, { + logger.log(`Indexing task ${opts.imageRef}`, { host: COORDINATOR_HOST, port: COORDINATOR_PORT, }); @@ -94,15 +94,16 @@ class DockerTaskOperations implements TaskOperations { "run", "--network=host", "--rm", + `--env=INDEX_TASKS=true`, `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`, `--env=TRIGGER_API_URL=${opts.apiUrl}`, + `--env=TRIGGER_ENV_ID=${opts.envId}`, + `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, + `--env=POD_NAME=${containerName}`, `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--env=POD_NAME=${containerName}`, - `--env=TRIGGER_ENV_ID=${opts.envId}`, - `--env=INDEX_TASKS=true`, `--name=${containerName}`, - `${opts.imageTag}`, + `${opts.imageRef}`, ]) ); } catch (error: any) { @@ -122,13 +123,7 @@ class DockerTaskOperations implements TaskOperations { } } - async create(opts: { - runId: string; - attemptId: string; - image: string; - machine: Machine; - envId: string; - }) { + async create(opts: TaskOperationsCreateOptions) { await this.#initialize(); const containerName = this.#getRunContainerName(opts.attemptId); @@ -139,13 +134,13 @@ class DockerTaskOperations implements TaskOperations { "run", "--network=host", "--detach", - `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, - `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, - `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--env=POD_NAME=${containerName}`, `--env=TRIGGER_ENV_ID=${opts.envId}`, `--env=TRIGGER_RUN_ID=${opts.runId}`, `--env=TRIGGER_ATTEMPT_ID=${opts.attemptId}`, + `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, + `--env=POD_NAME=${containerName}`, + `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, + `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, `--name=${containerName}`, `${opts.image}`, ]) @@ -167,12 +162,7 @@ class DockerTaskOperations implements TaskOperations { } } - async restore(opts: { - runId: string; - attemptId: string; - checkpointRef: string; - machine: Machine; - }) { + async restore(opts: TaskOperationsRestoreOptions) { await this.#initialize(); const containerName = this.#getRunContainerName(opts.attemptId); diff --git a/apps/kubernetes-provider/.env.example b/apps/kubernetes-provider/.env.example index 352a6ca8e..10949a807 100644 --- a/apps/kubernetes-provider/.env.example +++ b/apps/kubernetes-provider/.env.example @@ -1,7 +1,8 @@ HTTP_SERVER_PORT=8060 -PLATFORM_WS_PORT=8003 +PLATFORM_WS_PORT=3030 PLATFORM_SECRET=provider-secret -REGISTRY_FQDN=docker.io -REPO_NAME=task \ No newline at end of file +# Use this if you are on macOS +# COORDINATOR_HOST="host.docker.internal" +# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/kubernetes-provider/Containerfile b/apps/kubernetes-provider/Containerfile index d9630c5a3..f9fad85f4 100644 --- a/apps/kubernetes-provider/Containerfile +++ b/apps/kubernetes-provider/Containerfile @@ -31,7 +31,6 @@ COPY --from=pruner --chown=node:node /app/out/full/ . COPY --from=dev-deps --chown=node:node /app/ . COPY --chown=node:node turbo.json turbo.json -RUN pnpm run -r --filter '@trigger.dev/core*' build RUN pnpm run -r --filter kubernetes-provider build:bundle FROM base AS runner diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index df328011a..66ea4c510 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -1,16 +1,18 @@ -import { randomUUID } from "node:crypto"; import * as k8s from "@kubernetes/client-node"; -import { Machine } from "@trigger.dev/core/v3"; -import { ProviderShell, SimpleLogger, TaskOperations } from "@trigger.dev/core-apps"; +import { + ProviderShell, + SimpleLogger, + TaskOperations, + TaskOperationsCreateOptions, + TaskOperationsIndexOptions, + TaskOperationsRestoreOptions, +} from "@trigger.dev/core-apps"; const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; const NODE_NAME = process.env.NODE_NAME || "some-node"; const OTEL_EXPORTER_OTLP_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"; -const REGISTRY_FQDN = process.env.REGISTRY_FQDN || "localhost:5000"; -const REPO_NAME = process.env.REPO_NAME || "test"; - const logger = new SimpleLogger(`[${NODE_NAME}]`); type Namespace = { @@ -36,11 +38,11 @@ class KubernetesTaskOperations implements TaskOperations { this.#k8sApi = this.#createK8sApi(); } - async index(opts: { contentHash: string; imageTag: string; envId: string }) { + async index(opts: TaskOperationsIndexOptions) { await this.#createJob( { metadata: { - name: `task-index-${opts.contentHash}`, + name: this.#getIndexContainerName(opts.contentHash), namespace: this.#namespace.metadata.name, }, spec: { @@ -62,7 +64,7 @@ class KubernetesTaskOperations implements TaskOperations { containers: [ { name: opts.contentHash, - image: opts.imageTag, + image: opts.imageRef, ports: [ { containerPort: 8000, @@ -83,6 +85,14 @@ class KubernetesTaskOperations implements TaskOperations { name: "INDEX_TASKS", value: "true", }, + { + name: "TRIGGER_SECRET_KEY", + value: opts.apiKey, + }, + { + name: "TRIGGER_API_URL", + value: opts.apiUrl, + }, { name: "TRIGGER_ENV_ID", value: opts.envId, @@ -130,11 +140,11 @@ class KubernetesTaskOperations implements TaskOperations { ); } - async create(opts: { attemptId: string; image: string; machine: Machine; envId: string }) { + async create(opts: TaskOperationsCreateOptions) { await this.#createPod( { metadata: { - name: `task-run-${opts.attemptId}-${randomUUID().slice(0, 5)}`, + name: this.#getRunContainerName(opts.attemptId), namespace: this.#namespace.metadata.name, labels: { app: "task-run", @@ -209,18 +219,11 @@ class KubernetesTaskOperations implements TaskOperations { ); } - async restore(opts: { - attemptId: string; - runId: string; - image: string; - name: string; - checkpointId: string; - machine: Machine; - }) { + async restore(opts: TaskOperationsRestoreOptions) { await this.#createPod( { metadata: { - name: opts.name, + name: this.#getRunContainerName(opts.attemptId), namespace: this.#namespace.metadata.name, }, spec: { @@ -232,14 +235,14 @@ class KubernetesTaskOperations implements TaskOperations { initContainers: [ { name: "pull-base-image", - image: this.#getRestoreImage(opts.runId, opts.checkpointId), + image: opts.imageRef, command: ["sleep", "0"], }, ], containers: [ { name: opts.runId, - image: this.#getImageFromRunId(opts.runId), + image: opts.checkpointRef, ports: [ { containerPort: 8000, @@ -256,36 +259,37 @@ class KubernetesTaskOperations implements TaskOperations { }, }, }, - env: [ - { - name: "DEBUG", - value: "true", - }, - { - name: "POD_NAME", - valueFrom: { - fieldRef: { - fieldPath: "metadata.name", - }, - }, - }, - { - name: "COORDINATOR_HOST", - valueFrom: { - fieldRef: { - fieldPath: "status.hostIP", - }, - }, - }, - { - name: "NODE_NAME", - valueFrom: { - fieldRef: { - fieldPath: "spec.nodeName", - }, - }, - }, - ], + // TODO: check we definitely don't need to specify these again + // env: [ + // { + // name: "DEBUG", + // value: "true", + // }, + // { + // name: "POD_NAME", + // valueFrom: { + // fieldRef: { + // fieldPath: "metadata.name", + // }, + // }, + // }, + // { + // name: "COORDINATOR_HOST", + // valueFrom: { + // fieldRef: { + // fieldPath: "status.hostIP", + // }, + // }, + // }, + // { + // name: "NODE_NAME", + // valueFrom: { + // fieldRef: { + // fieldPath: "spec.nodeName", + // }, + // }, + // }, + // ], }, ], }, @@ -305,12 +309,12 @@ class KubernetesTaskOperations implements TaskOperations { await this.#getPod(opts.runId, this.#namespace); } - #getImageFromRunId(runId: string) { - return `${REGISTRY_FQDN}/${REPO_NAME}:${runId}`; + #getIndexContainerName(contentHash: string) { + return `task-index-${contentHash}`; } - #getRestoreImage(runId: string, checkpointId: string) { - return `${REGISTRY_FQDN}/${REPO_NAME}:${checkpointId}`; + #getRunContainerName(attemptId: string) { + return `task-run-${attemptId}`; } #createK8sApi() { diff --git a/apps/webapp/app/v3/services/createCheckpoint.server.ts b/apps/webapp/app/v3/services/createCheckpoint.server.ts index f59092fdf..c0f34f18b 100644 --- a/apps/webapp/app/v3/services/createCheckpoint.server.ts +++ b/apps/webapp/app/v3/services/createCheckpoint.server.ts @@ -15,7 +15,7 @@ export class CreateCheckpointService { public async call( params: InferSocketMessageSchema - ): Promise { + ): Promise { logger.debug(`Creating checkpoint`, params); const attempt = await this.#prismaClient.taskRunAttempt.findUniqueOrThrow({ @@ -24,9 +24,25 @@ export class CreateCheckpointService { }, include: { taskRun: true, + backgroundWorker: { + select: { + deployment: { + select: { + imageReference: true, + }, + }, + }, + }, }, }); + const imageRef = attempt.backgroundWorker.deployment?.imageReference; + + if (!imageRef) { + logger.error("No image ref", { attemptId: params.attemptId }); + return; + } + const checkpoint = await this.#prismaClient.checkpoint.create({ data: { friendlyId: generateFriendlyId("checkpoint"), @@ -38,6 +54,7 @@ export class CreateCheckpointService { type: params.docker ? "DOCKER" : "KUBERNETES", reason: params.reason.type, metadata: JSON.stringify(params.reason), + imageRef, }, }); diff --git a/apps/webapp/app/v3/services/restoreCheckpoint.server.ts b/apps/webapp/app/v3/services/restoreCheckpoint.server.ts index 3704cb4e5..cd8676b45 100644 --- a/apps/webapp/app/v3/services/restoreCheckpoint.server.ts +++ b/apps/webapp/app/v3/services/restoreCheckpoint.server.ts @@ -31,6 +31,7 @@ export class RestoreCheckpointService { type: checkpoint.type, location: checkpoint.location, reason: checkpoint.reason ?? undefined, + imageRef: checkpoint.imageRef, }); return checkpoint; diff --git a/packages/core-apps/package.json b/packages/core-apps/package.json index bebf33384..5eeb914b6 100644 --- a/packages/core-apps/package.json +++ b/packages/core-apps/package.json @@ -21,17 +21,12 @@ "./package.json": "./package.json" }, "scripts": { - "clean": "rimraf dist", - "build": "npm run clean && npm run build:tsup", - "build:tsup": "tsup --dts-resolve", "typecheck": "tsc --noEmit" }, "devDependencies": { + "@trigger.dev/core": "workspace:*", "@trigger.dev/tsconfig": "workspace:*", - "@trigger.dev/tsup": "workspace:*", "@types/node": "18", - "rimraf": "^3.0.2", - "tsup": "^8.0.1", "typescript": "^5.3.0" }, "engines": { diff --git a/packages/core-apps/src/provider.ts b/packages/core-apps/src/provider.ts index 1b120664b..28447d876 100644 --- a/packages/core-apps/src/provider.ts +++ b/packages/core-apps/src/provider.ts @@ -2,6 +2,7 @@ import { createServer } from "node:http"; import { ClientToSharedQueueMessages, clientWebsocketMessages, + Machine, PlatformToProviderMessages, ProviderToPlatformMessages, SharedQueueToClientMessages, @@ -20,12 +21,37 @@ const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret"; const logger = new SimpleLogger(`[${MACHINE_NAME}]`); +export interface TaskOperationsIndexOptions { + contentHash: string; + imageRef: string; + envId: string; + apiKey: string; + apiUrl: string; +} + +export interface TaskOperationsCreateOptions { + runId: string; + attemptId: string; + image: string; + machine: Machine; + envId: string; +} + +export interface TaskOperationsRestoreOptions { + runId: string; + attemptId: string; + imageRef: string; + checkpointRef: string; + machine: Machine; +} + export interface TaskOperations { - create: (...args: any[]) => Promise; - restore: (...args: any[]) => Promise; + index: (opts: TaskOperationsIndexOptions) => Promise; + create: (opts: TaskOperationsCreateOptions) => Promise; + restore: (opts: TaskOperationsRestoreOptions) => Promise; + delete: (...args: any[]) => Promise; get: (...args: any[]) => Promise; - index: (...args: any[]) => Promise; } type ProviderShellOptions = { @@ -135,7 +161,7 @@ export class ProviderShell implements Provider { try { await this.tasks.index({ contentHash: message.contentHash, - imageTag: message.imageTag, + imageRef: message.imageTag, envId: message.envId, apiKey: message.apiKey, apiUrl: message.apiUrl, @@ -180,8 +206,11 @@ export class ProviderShell implements Provider { runId: message.runId, attemptId: message.attemptId, checkpointRef: message.location, - // TODO - // machine: message.machine, + machine: { + cpu: "1", + memory: "100Mi", + }, + imageRef: message.imageRef, }); } catch (error) { logger.error("restore failed", error); @@ -228,6 +257,7 @@ export class ProviderShell implements Provider { cpu: "1", memory: "100Mi", }, + runId: "", }); return reply.text(`sent restore request: ${body}`); diff --git a/packages/core-apps/tsconfig.build.json b/packages/core-apps/tsconfig.build.json deleted file mode 100644 index f444efc19..000000000 --- a/packages/core-apps/tsconfig.build.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "@trigger.dev/tsconfig/node18.json", - "include": ["src/globals.d.ts", "./src/**/*.ts", "tsup.config.ts"], - "compilerOptions": { - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "declaration": false, - "declarationMap": false - }, - "exclude": ["node_modules"] -} diff --git a/packages/core-apps/tsup.config.ts b/packages/core-apps/tsup.config.ts deleted file mode 100644 index 5a0b9c86a..000000000 --- a/packages/core-apps/tsup.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { packageOptions, defineConfig } from "@trigger.dev/tsup"; - -export default defineConfig({ - ...packageOptions, - config: "tsconfig.build.json", - banner: { - js: "import { createRequire } from 'module';const require = createRequire(import.meta.url);", - }, -}); diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index ce243d199..3a055aca7 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -102,6 +102,7 @@ export const PlatformToProviderMessages = { type: z.enum(["DOCKER", "KUBERNETES"]), location: z.string(), reason: z.string().optional(), + imageRef: z.string(), }), }, DELETE: { diff --git a/packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql b/packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql new file mode 100644 index 000000000..1f12a8b59 --- /dev/null +++ b/packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Added the required column `imageRef` to the `Checkpoint` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "Checkpoint" ADD COLUMN "imageRef" TEXT NOT NULL; diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 8a59cdf9c..982e05e86 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -1956,6 +1956,7 @@ model Checkpoint { type CheckpointType location String + imageRef String reason String? metadata String? diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index efa350912..a4ae0da1d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1263,18 +1263,14 @@ importers: packages/core-apps: specifiers: + '@trigger.dev/core': workspace:* '@trigger.dev/tsconfig': workspace:* - '@trigger.dev/tsup': workspace:* '@types/node': '18' - rimraf: ^3.0.2 - tsup: ^8.0.1 typescript: ^5.3.0 devDependencies: + '@trigger.dev/core': link:../core '@trigger.dev/tsconfig': link:../../config-packages/tsconfig - '@trigger.dev/tsup': link:../../config-packages/tsup '@types/node': 18.17.1 - rimraf: 3.0.2 - tsup: 8.0.2_typescript@5.3.3 typescript: 5.3.3 packages/core-backend: @@ -37606,45 +37602,6 @@ packages: - ts-node dev: true - /tsup/8.0.2_typescript@5.3.3: - resolution: {integrity: sha512-NY8xtQXdH7hDUAZwcQdY/Vzlw9johQsaqf7iwZ6g1DOUlFYQ5/AtVAjTvihhEyeRlGo4dLRVHtrRaL35M1daqQ==} - engines: {node: '>=18'} - hasBin: true - peerDependencies: - '@microsoft/api-extractor': ^7.36.0 - '@swc/core': ^1 - postcss: ^8.4.12 - typescript: '>=4.5.0' - peerDependenciesMeta: - '@microsoft/api-extractor': - optional: true - '@swc/core': - optional: true - postcss: - optional: true - typescript: - optional: true - dependencies: - bundle-require: 4.0.1_esbuild@0.19.11 - cac: 6.7.14 - chokidar: 3.5.3 - debug: 4.3.4 - esbuild: 0.19.11 - execa: 5.1.1 - globby: 11.1.0 - joycon: 3.1.1 - postcss-load-config: 4.0.1 - resolve-from: 5.0.0 - rollup: 4.6.1 - source-map: 0.8.0-beta.0 - sucrase: 3.32.0 - tree-kill: 1.2.2 - typescript: 5.3.3 - transitivePeerDependencies: - - supports-color - - ts-node - dev: true - /tsutils/3.21.0: resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'}