github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

update kubernetes provider and fix builds again

Browse Source
This commit is contained in:
nicktrn
2024-03-18 18:45:00 +00:00
parent ce9c5d22fe
commit 9945980ab0
17 changed files with 155 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/coordinator/Containerfile
-1
View File
@@ -35,7 +35,6 @@ COPY --from=pruner --chown=node:node /app/out/full/ .
COPY --from=dev-deps --chown=node:node /app/ .
COPY --chown=node:node turbo.json turbo.json
RUN pnpm run -r --filter '@trigger.dev/core*' build
RUN pnpm run -r --filter coordinator build:bundle
FROM alpine AS cri-tools
apps/coordinator/package.json
+1 -1
View File
@@ -6,7 +6,7 @@
"main": "dist/index.cjs",
"scripts": {
"build": "npm run build:bundle",
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"const require = createRequire(import.meta.url);\"",
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
"build:image": "docker build -f Containerfile . -t coordinator",
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
"start": "tsx src/index.ts",
apps/docker-provider/.env.example
+1
View File
@@ -2,6 +2,7 @@ HTTP_SERVER_PORT=8050
PLATFORM_WS_PORT=3030
PLATFORM_SECRET=provider-secret
# Use this if you are on macOS
# COORDINATOR_HOST="host.docker.internal"
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
apps/docker-provider/src/index.ts
+21 -31
View File
@@ -1,6 +1,12 @@
import { $, type ExecaChildProcess, execa } from "execa";
import { Machine } from "@trigger.dev/core/v3";
import { SimpleLogger, TaskOperations, ProviderShell } from "@trigger.dev/core-apps";
import {
SimpleLogger,
TaskOperations,
ProviderShell,
TaskOperationsRestoreOptions,
TaskOperationsCreateOptions,
TaskOperationsIndexOptions,
} from "@trigger.dev/core-apps";
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
@@ -72,18 +78,12 @@ class DockerTaskOperations implements TaskOperations {
};
}
async index(opts: {
contentHash: string;
imageTag: string;
envId: string;
apiKey: string;
apiUrl: string;
}) {
async index(opts: TaskOperationsIndexOptions) {
await this.#initialize();
const containerName = this.#getIndexContainerName(opts.contentHash);
logger.log(`Indexing task ${opts.imageTag}`, {
logger.log(`Indexing task ${opts.imageRef}`, {
host: COORDINATOR_HOST,
port: COORDINATOR_PORT,
});
@@ -94,15 +94,16 @@ class DockerTaskOperations implements TaskOperations {
"run",
"--network=host",
"--rm",
`--env=INDEX_TASKS=true`,
`--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
`--env=TRIGGER_API_URL=${opts.apiUrl}`,
`--env=TRIGGER_ENV_ID=${opts.envId}`,
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
`--env=POD_NAME=${containerName}`,
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
`--env=POD_NAME=${containerName}`,
`--env=TRIGGER_ENV_ID=${opts.envId}`,
`--env=INDEX_TASKS=true`,
`--name=${containerName}`,
`${opts.imageTag}`,
`${opts.imageRef}`,
])
);
} catch (error: any) {
@@ -122,13 +123,7 @@ class DockerTaskOperations implements TaskOperations {
}
}
async create(opts: {
runId: string;
attemptId: string;
image: string;
machine: Machine;
envId: string;
}) {
async create(opts: TaskOperationsCreateOptions) {
await this.#initialize();
const containerName = this.#getRunContainerName(opts.attemptId);
@@ -139,13 +134,13 @@ class DockerTaskOperations implements TaskOperations {
"run",
"--network=host",
"--detach",
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
`--env=POD_NAME=${containerName}`,
`--env=TRIGGER_ENV_ID=${opts.envId}`,
`--env=TRIGGER_RUN_ID=${opts.runId}`,
`--env=TRIGGER_ATTEMPT_ID=${opts.attemptId}`,
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
`--env=POD_NAME=${containerName}`,
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
`--name=${containerName}`,
`${opts.image}`,
])
@@ -167,12 +162,7 @@ class DockerTaskOperations implements TaskOperations {
}
}
async restore(opts: {
runId: string;
attemptId: string;
checkpointRef: string;
machine: Machine;
}) {
async restore(opts: TaskOperationsRestoreOptions) {
await this.#initialize();
const containerName = this.#getRunContainerName(opts.attemptId);
apps/kubernetes-provider/.env.example
+4 -3
View File
@@ -1,7 +1,8 @@
HTTP_SERVER_PORT=8060
PLATFORM_WS_PORT=8003
PLATFORM_WS_PORT=3030
PLATFORM_SECRET=provider-secret
REGISTRY_FQDN=docker.io
REPO_NAME=task
# Use this if you are on macOS
# COORDINATOR_HOST="host.docker.internal"
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
apps/kubernetes-provider/Containerfile
-1
View File
@@ -31,7 +31,6 @@ COPY --from=pruner --chown=node:node /app/out/full/ .
COPY --from=dev-deps --chown=node:node /app/ .
COPY --chown=node:node turbo.json turbo.json
RUN pnpm run -r --filter '@trigger.dev/core*' build
RUN pnpm run -r --filter kubernetes-provider build:bundle
FROM base AS runner
apps/kubernetes-provider/src/index.ts
+60 -56
View File
@@ -1,16 +1,18 @@
import { randomUUID } from "node:crypto";
import * as k8s from "@kubernetes/client-node";
import { Machine } from "@trigger.dev/core/v3";
import { ProviderShell, SimpleLogger, TaskOperations } from "@trigger.dev/core-apps";
import {
ProviderShell,
SimpleLogger,
TaskOperations,
TaskOperationsCreateOptions,
TaskOperationsIndexOptions,
TaskOperationsRestoreOptions,
} from "@trigger.dev/core-apps";
const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
const NODE_NAME = process.env.NODE_NAME || "some-node";
const OTEL_EXPORTER_OTLP_ENDPOINT =
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
const REGISTRY_FQDN = process.env.REGISTRY_FQDN || "localhost:5000";
const REPO_NAME = process.env.REPO_NAME || "test";
const logger = new SimpleLogger(`[${NODE_NAME}]`);
type Namespace = {
@@ -36,11 +38,11 @@ class KubernetesTaskOperations implements TaskOperations {
this.#k8sApi = this.#createK8sApi();
}
async index(opts: { contentHash: string; imageTag: string; envId: string }) {
async index(opts: TaskOperationsIndexOptions) {
await this.#createJob(
{
metadata: {
name: `task-index-${opts.contentHash}`,
name: this.#getIndexContainerName(opts.contentHash),
namespace: this.#namespace.metadata.name,
},
spec: {
@@ -62,7 +64,7 @@ class KubernetesTaskOperations implements TaskOperations {
containers: [
{
name: opts.contentHash,
image: opts.imageTag,
image: opts.imageRef,
ports: [
{
containerPort: 8000,
@@ -83,6 +85,14 @@ class KubernetesTaskOperations implements TaskOperations {
name: "INDEX_TASKS",
value: "true",
},
{
name: "TRIGGER_SECRET_KEY",
value: opts.apiKey,
},
{
name: "TRIGGER_API_URL",
value: opts.apiUrl,
},
{
name: "TRIGGER_ENV_ID",
value: opts.envId,
@@ -130,11 +140,11 @@ class KubernetesTaskOperations implements TaskOperations {
);
}
async create(opts: { attemptId: string; image: string; machine: Machine; envId: string }) {
async create(opts: TaskOperationsCreateOptions) {
await this.#createPod(
{
metadata: {
name: `task-run-${opts.attemptId}-${randomUUID().slice(0, 5)}`,
name: this.#getRunContainerName(opts.attemptId),
namespace: this.#namespace.metadata.name,
labels: {
app: "task-run",
@@ -209,18 +219,11 @@ class KubernetesTaskOperations implements TaskOperations {
);
}
async restore(opts: {
attemptId: string;
runId: string;
image: string;
name: string;
checkpointId: string;
machine: Machine;
}) {
async restore(opts: TaskOperationsRestoreOptions) {
await this.#createPod(
{
metadata: {
name: opts.name,
name: this.#getRunContainerName(opts.attemptId),
namespace: this.#namespace.metadata.name,
},
spec: {
@@ -232,14 +235,14 @@ class KubernetesTaskOperations implements TaskOperations {
initContainers: [
{
name: "pull-base-image",
image: this.#getRestoreImage(opts.runId, opts.checkpointId),
image: opts.imageRef,
command: ["sleep", "0"],
},
],
containers: [
{
name: opts.runId,
image: this.#getImageFromRunId(opts.runId),
image: opts.checkpointRef,
ports: [
{
containerPort: 8000,
@@ -256,36 +259,37 @@ class KubernetesTaskOperations implements TaskOperations {
},
},
},
env: [
{
name: "DEBUG",
value: "true",
},
{
name: "POD_NAME",
valueFrom: {
fieldRef: {
fieldPath: "metadata.name",
},
},
},
{
name: "COORDINATOR_HOST",
valueFrom: {
fieldRef: {
fieldPath: "status.hostIP",
},
},
},
{
name: "NODE_NAME",
valueFrom: {
fieldRef: {
fieldPath: "spec.nodeName",
},
},
},
],
// TODO: check we definitely don't need to specify these again
// env: [
// {
// name: "DEBUG",
// value: "true",
// },
// {
// name: "POD_NAME",
// valueFrom: {
// fieldRef: {
// fieldPath: "metadata.name",
// },
// },
// },
// {
// name: "COORDINATOR_HOST",
// valueFrom: {
// fieldRef: {
// fieldPath: "status.hostIP",
// },
// },
// },
// {
// name: "NODE_NAME",
// valueFrom: {
// fieldRef: {
// fieldPath: "spec.nodeName",
// },
// },
// },
// ],
},
],
},
@@ -305,12 +309,12 @@ class KubernetesTaskOperations implements TaskOperations {
await this.#getPod(opts.runId, this.#namespace);
}
#getImageFromRunId(runId: string) {
return `${REGISTRY_FQDN}/${REPO_NAME}:${runId}`;
#getIndexContainerName(contentHash: string) {
return `task-index-${contentHash}`;
}
#getRestoreImage(runId: string, checkpointId: string) {
return `${REGISTRY_FQDN}/${REPO_NAME}:${checkpointId}`;
#getRunContainerName(attemptId: string) {
return `task-run-${attemptId}`;
}
#createK8sApi() {
apps/webapp/app/v3/services/createCheckpoint.server.ts
+18 -1
View File
@@ -15,7 +15,7 @@ export class CreateCheckpointService {
public async call(
params: InferSocketMessageSchema<typeof CoordinatorToPlatformMessages, "CHECKPOINT_CREATED">
): Promise<Checkpoint> {
): Promise<Checkpoint | undefined> {
logger.debug(`Creating checkpoint`, params);
const attempt = await this.#prismaClient.taskRunAttempt.findUniqueOrThrow({
@@ -24,9 +24,25 @@ export class CreateCheckpointService {
},
include: {
taskRun: true,
backgroundWorker: {
select: {
deployment: {
select: {
imageReference: true,
},
},
},
},
},
});
const imageRef = attempt.backgroundWorker.deployment?.imageReference;
if (!imageRef) {
logger.error("No image ref", { attemptId: params.attemptId });
return;
}
const checkpoint = await this.#prismaClient.checkpoint.create({
data: {
friendlyId: generateFriendlyId("checkpoint"),
@@ -38,6 +54,7 @@ export class CreateCheckpointService {
type: params.docker ? "DOCKER" : "KUBERNETES",
reason: params.reason.type,
metadata: JSON.stringify(params.reason),
imageRef,
},
});
apps/webapp/app/v3/services/restoreCheckpoint.server.ts
+1
View File
@@ -31,6 +31,7 @@ export class RestoreCheckpointService {
type: checkpoint.type,
location: checkpoint.location,
reason: checkpoint.reason ?? undefined,
imageRef: checkpoint.imageRef,
});
return checkpoint;
packages/core-apps/package.json
+1 -6
View File
@@ -21,17 +21,12 @@
"./package.json": "./package.json"
},
"scripts": {
"clean": "rimraf dist",
"build": "npm run clean && npm run build:tsup",
"build:tsup": "tsup --dts-resolve",
"typecheck": "tsc --noEmit"
},
"devDependencies": {
"@trigger.dev/core": "workspace:*",
"@trigger.dev/tsconfig": "workspace:*",
"@trigger.dev/tsup": "workspace:*",
"@types/node": "18",
"rimraf": "^3.0.2",
"tsup": "^8.0.1",
"typescript": "^5.3.0"
},
"engines": {
packages/core-apps/src/provider.ts
+36 -6
View File
@@ -2,6 +2,7 @@ import { createServer } from "node:http";
import {
ClientToSharedQueueMessages,
clientWebsocketMessages,
Machine,
PlatformToProviderMessages,
ProviderToPlatformMessages,
SharedQueueToClientMessages,
@@ -20,12 +21,37 @@ const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
export interface TaskOperationsIndexOptions {
contentHash: string;
imageRef: string;
envId: string;
apiKey: string;
apiUrl: string;
}
export interface TaskOperationsCreateOptions {
runId: string;
attemptId: string;
image: string;
machine: Machine;
envId: string;
}
export interface TaskOperationsRestoreOptions {
runId: string;
attemptId: string;
imageRef: string;
checkpointRef: string;
machine: Machine;
}
export interface TaskOperations {
create: (...args: any[]) => Promise<any>;
restore: (...args: any[]) => Promise<any>;
index: (opts: TaskOperationsIndexOptions) => Promise<any>;
create: (opts: TaskOperationsCreateOptions) => Promise<any>;
restore: (opts: TaskOperationsRestoreOptions) => Promise<any>;
delete: (...args: any[]) => Promise<any>;
get: (...args: any[]) => Promise<any>;
index: (...args: any[]) => Promise<any>;
}
type ProviderShellOptions = {
@@ -135,7 +161,7 @@ export class ProviderShell implements Provider {
try {
await this.tasks.index({
contentHash: message.contentHash,
imageTag: message.imageTag,
imageRef: message.imageTag,
envId: message.envId,
apiKey: message.apiKey,
apiUrl: message.apiUrl,
@@ -180,8 +206,11 @@ export class ProviderShell implements Provider {
runId: message.runId,
attemptId: message.attemptId,
checkpointRef: message.location,
// TODO
// machine: message.machine,
machine: {
cpu: "1",
memory: "100Mi",
},
imageRef: message.imageRef,
});
} catch (error) {
logger.error("restore failed", error);
@@ -228,6 +257,7 @@ export class ProviderShell implements Provider {
cpu: "1",
memory: "100Mi",
},
runId: "<missing>",
});
return reply.text(`sent restore request: ${body}`);
packages/core-apps/tsconfig.build.json
-11
View File
@@ -1,11 +0,0 @@
{
"extends": "@trigger.dev/tsconfig/node18.json",
"include": ["src/globals.d.ts", "./src/**/*.ts", "tsup.config.ts"],
"compilerOptions": {
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"declaration": false,
"declarationMap": false
},
"exclude": ["node_modules"]
}
packages/core-apps/tsup.config.ts
-9
View File
@@ -1,9 +0,0 @@
import { packageOptions, defineConfig } from "@trigger.dev/tsup";
export default defineConfig({
...packageOptions,
config: "tsconfig.build.json",
banner: {
js: "import { createRequire } from 'module';const require = createRequire(import.meta.url);",
},
});
packages/core/src/v3/schemas/schemas.ts
+1
View File
@@ -102,6 +102,7 @@ export const PlatformToProviderMessages = {
type: z.enum(["DOCKER", "KUBERNETES"]),
location: z.string(),
reason: z.string().optional(),
imageRef: z.string(),
}),
},
DELETE: {
packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql
+8
View File
@@ -0,0 +1,8 @@
/*
Warnings:
- Added the required column `imageRef` to the `Checkpoint` table without a default value. This is not possible if the table is not empty.
*/
-- AlterTable
ALTER TABLE "Checkpoint" ADD COLUMN "imageRef" TEXT NOT NULL;
packages/database/prisma/schema.prisma
+1
View File
@@ -1956,6 +1956,7 @@ model Checkpoint {
type CheckpointType
location String
imageRef String
reason String?
metadata String?
pnpm-lock.yaml
Generated
+2 -45
View File
@@ -1263,18 +1263,14 @@ importers:
packages/core-apps:
specifiers:
'@trigger.dev/core': workspace:*
'@trigger.dev/tsconfig': workspace:*
'@trigger.dev/tsup': workspace:*
'@types/node': '18'
rimraf: ^3.0.2
tsup: ^8.0.1
typescript: ^5.3.0
devDependencies:
'@trigger.dev/core': link:../core
'@trigger.dev/tsconfig': link:../../config-packages/tsconfig
'@trigger.dev/tsup': link:../../config-packages/tsup
'@types/node': 18.17.1
rimraf: 3.0.2
tsup: 8.0.2_typescript@5.3.3
typescript: 5.3.3
packages/core-backend:
@@ -37606,45 +37602,6 @@ packages:
- ts-node
dev: true
/tsup/8.0.2_typescript@5.3.3:
resolution: {integrity: sha512-NY8xtQXdH7hDUAZwcQdY/Vzlw9johQsaqf7iwZ6g1DOUlFYQ5/AtVAjTvihhEyeRlGo4dLRVHtrRaL35M1daqQ==}
engines: {node: '>=18'}
hasBin: true
peerDependencies:
'@microsoft/api-extractor': ^7.36.0
'@swc/core': ^1
postcss: ^8.4.12
typescript: '>=4.5.0'
peerDependenciesMeta:
'@microsoft/api-extractor':
optional: true
'@swc/core':
optional: true
postcss:
optional: true
typescript:
optional: true
dependencies:
bundle-require: 4.0.1_esbuild@0.19.11
cac: 6.7.14
chokidar: 3.5.3
debug: 4.3.4
esbuild: 0.19.11
execa: 5.1.1
globby: 11.1.0
joycon: 3.1.1
postcss-load-config: 4.0.1
resolve-from: 5.0.0
rollup: 4.6.1
source-map: 0.8.0-beta.0
sucrase: 3.32.0
tree-kill: 1.2.2
typescript: 5.3.3
transitivePeerDependencies:
- supports-color
- ts-node
dev: true
/tsutils/3.21.0:
resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==}
engines: {node: '>= 6'}