chore: remove end-of-life v3 execution components (#4194)

v3 (engine V1) is end-of-lifed and the v3 clusters are gone, so this
removes the dead v3 execution code from the monorepo. It's the first
pass of TRI-11824 - the webapp v3 code paths are deliberately left
untouched and gated for a follow-up.

## Apps

Deletes the three v3-only execution apps and their build wiring:

- `apps/coordinator`, `apps/kubernetes-provider`, `apps/docker-provider`
- `.github/workflows/publish-worker.yml` - it built only those three;
the v4 worker publish is a separate workflow
- Their references in `.changeset/config.json`, `.cursorignore`,
`CHANGESETS.md`, `CONTRIBUTING.md`, `.server-changes/README.md`
- `pnpm-lock.yaml` regenerated to prune the apps and their app-only
dependencies (`socket.io`, `@kubernetes/client-node`, `p-queue`,
`execa`, `prom-client`, `tinyexec`)

## Core

Removes the helpers in `@trigger.dev/core` that only those apps used -
`ProviderShell`, `SimpleLogger`, the `Exec`/process helpers,
`isExecaChildProcess`, `getTextBody`, and `testDockerCheckpoint`. Each
was verified to have no remaining consumers anywhere in the repo.

Kept the helpers still used elsewhere: `ExponentialBackoff` (warm-start
client), `HttpReply`/`getJsonBody` (serverOnly http server),
`SimpleStructuredLogger` (widely used), and
`ZodNamespace`/`ZodSocketConnection` (still referenced by legacy v3
webapp code, hence the follow-up pass).

The `./v3/apps` and `./v3/serverOnly` export subpaths remain - only dead
members were trimmed from their barrels, so no `package.json` exports
changed.

## Verification

`@trigger.dev/core` builds, and `typecheck` passes for core, supervisor,
cli-v3, run-engine, redis-worker, and webapp.

refs TRI-11824
This commit is contained in:
nicktrn
2026-07-08 18:47:20 +01:00
committed by GitHub
parent e0208f3a27
commit a6bd370e42
48 changed files with 14 additions and 6854 deletions
-3
View File
@@ -14,9 +14,6 @@
"updateInternalDependencies": "patch",
"ignore": [
"webapp",
"coordinator",
"docker-provider",
"kubernetes-provider",
"supervisor"
],
"___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
+5
View File
@@ -0,0 +1,5 @@
---
"@trigger.dev/core": patch
---
Removed internal helpers that were only used by the end-of-life v3 self-hosted compute providers.
+2 -2
View File
@@ -5,7 +5,7 @@ paths:
# Server App Changes
When modifying server apps (webapp, supervisor, coordinator, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset:
When modifying server apps (webapp, supervisor, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset:
```bash
cat > .server-changes/descriptive-name.md << 'EOF'
@@ -18,6 +18,6 @@ Brief description of what changed and why.
EOF
```
- **area**: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
- **area**: `webapp` | `supervisor`
- **type**: `feature` | `fix` | `improvement` | `breaking`
- If the PR also touches `packages/`, just the changeset is sufficient (no `.server-changes/` needed).
-3
View File
@@ -1,7 +1,4 @@
apps/docker-provider/
apps/kubernetes-provider/
apps/proxy/
apps/coordinator/
packages/rsc/
.changeset
.zed
-105
View File
@@ -1,105 +0,0 @@
name: "⚒️ Publish Worker"
on:
workflow_call:
inputs:
image_tag:
description: The image tag to publish
type: string
required: false
default: ""
image_registry:
description: The registry namespace to publish under (e.g. ghcr.io/<owner>)
type: string
required: false
default: ""
secrets:
DOCKERHUB_USERNAME:
required: false
DOCKERHUB_TOKEN:
required: false
push:
tags:
- "infra-dev-*"
- "infra-test-*"
- "infra-prod-*"
permissions:
packages: write
contents: read
jobs:
build:
strategy:
matrix:
package: [coordinator, docker-provider, kubernetes-provider]
runs-on: warp-ubuntu-latest-x64-8x
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
steps:
- name: ⬇️ Checkout git repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: 📦 Get image repo
id: get_repository
env:
PACKAGE: ${{ matrix.package }}
run: |
if [[ "$PACKAGE" == *-provider ]]; then
repo="provider/${PACKAGE%-provider}"
else
repo="$PACKAGE"
fi
echo "repo=${repo}" >> "$GITHUB_OUTPUT"
- id: get_tag
uses: ./.github/actions/get-image-tag
with:
tag: ${{ inputs.image_tag }}
- name: 🐋 Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
# ..to avoid rate limits when pulling images
- name: 🐳 Login to DockerHub
if: ${{ env.DOCKERHUB_USERNAME }}
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: 🚢 Build Container Image
run: |
docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
# ..to push image
- name: 🐙 Login to GitHub Container Registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: 🐙 Push to GitHub Container Registry
run: |
docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
env:
# Resolved by the caller when invoked from publish.yml; falls back to the
# IMAGE_REGISTRY repository variable (or ghcr.io/<owner>) for the direct
# push triggers above, so a fork publishes to its own namespace.
REGISTRY: ${{ inputs.image_registry || vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }}
REPOSITORY: ${{ steps.get_repository.outputs.repo }}
IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
# - name: 🐙 Push 'v3' tag to GitHub Container Registry
# if: steps.get_tag.outputs.is_semver == 'true'
# run: |
# docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
# docker push "$REGISTRY/$REPOSITORY:v3"
# env:
# REGISTRY: ghcr.io/triggerdotdev
# REPOSITORY: ${{ steps.get_repository.outputs.repo }}
-14
View File
@@ -30,7 +30,6 @@ on:
- ".github/workflows/unit-tests.yml"
- ".github/workflows/e2e.yml"
- ".github/workflows/publish-webapp.yml"
- ".github/workflows/publish-worker.yml"
- "packages/**"
- "!packages/**/*.md"
- "!packages/**/*.eslintrc"
@@ -80,19 +79,6 @@ jobs:
# to its own namespace; set the IMAGE_REGISTRY repository variable to override.
image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }}
publish-worker:
needs: [typecheck]
permissions:
contents: read
packages: write
uses: ./.github/workflows/publish-worker.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
with:
image_tag: ${{ inputs.image_tag }}
image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }}
publish-worker-v4:
needs: [typecheck]
permissions:
+3 -3
View File
@@ -1,10 +1,10 @@
# Server Changes
This directory tracks changes to server-only components (webapp, supervisor, coordinator, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented.
This directory tracks changes to server-only components (webapp, supervisor, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented.
## When to add a file
**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file.
**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file.
**Mixed PRs** (both packages and server): Just add a changeset as usual. No `.server-changes/` file needed — the changeset covers it.
@@ -31,7 +31,7 @@ Speed up batch queue processing by removing stalls and fixing retry race
### Fields
- **area** (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
- **area** (required): `webapp` | `supervisor`
- **type** (required): `feature` | `fix` | `improvement` | `breaking`
### Description
+1 -1
View File
@@ -21,7 +21,7 @@ Speed up batch queue processing by removing stalls and fixing retry race
EOF
```
- `area`: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
- `area`: `webapp` | `supervisor`
- `type`: `feature` | `fix` | `improvement` | `breaking`
For **mixed PRs** (both packages and server): just add a changeset. No `.server-changes/` file needed.
+2 -2
View File
@@ -265,7 +265,7 @@ Most of the time the changes you'll make are likely to be categorized as patch r
## Adding server changes
Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes.
Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes.
Create a markdown file with a descriptive name:
@@ -281,7 +281,7 @@ EOF
```
**Fields:**
- `area` (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
- `area` (required): `webapp` | `supervisor`
- `type` (required): `feature` | `fix` | `improvement` | `breaking`
The body text (below the frontmatter) is a one-line description of the change. Keep it concise — it will appear in release notes.
-4
View File
@@ -1,4 +0,0 @@
HTTP_SERVER_PORT=8020
PLATFORM_ENABLED=true
PLATFORM_WS_PORT=3030
SECURE_CONNECTION=false
-3
View File
@@ -1,3 +0,0 @@
dist/
node_modules/
.env
-60
View File
@@ -1,60 +0,0 @@
# syntax=docker/dockerfile:labs
FROM node:22.23.1-bookworm-slim@sha256:813a7480f28fdadac1f7f5c824bcdad435b5bc1322a5968bbbdef8d058f9dff4 AS node-22
WORKDIR /app
FROM node-22 AS pruner
COPY --chown=node:node . .
RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
FROM node-22 AS base
RUN apt-get update \
&& apt-get install -y buildah ca-certificates dumb-init docker.io busybox \
&& rm -rf /var/lib/apt/lists/*
COPY --chown=node:node .gitignore .gitignore
COPY --from=pruner --chown=node:node /app/out/json/ .
COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
FROM base AS dev-deps
RUN corepack enable
ENV NODE_ENV development
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
FROM base AS builder
RUN corepack enable
COPY --from=pruner --chown=node:node /app/out/full/ .
COPY --from=dev-deps --chown=node:node /app/ .
COPY --chown=node:node turbo.json turbo.json
RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter coordinator build:bundle
FROM alpine AS cri-tools
WORKDIR /cri-tools
ARG CRICTL_VERSION=v1.29.0
ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
ADD --checksum=${CRICTL_CHECKSUM} \
https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
FROM base AS runner
RUN corepack enable
ENV NODE_ENV production
COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs
EXPOSE 8000
CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]
-3
View File
@@ -1,3 +0,0 @@
# Coordinator
Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.
-30
View File
@@ -1,30 +0,0 @@
{
"name": "coordinator",
"private": true,
"version": "0.0.1",
"description": "",
"main": "dist/index.cjs",
"scripts": {
"build": "npm run build:bundle",
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
"build:image": "docker build -f Containerfile . -t coordinator",
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
"start": "tsx src/index.ts",
"typecheck": "tsc --noEmit"
},
"keywords": [],
"author": "",
"license": "MIT",
"dependencies": {
"@trigger.dev/core": "workspace:*",
"nanoid": "^5.0.6",
"prom-client": "^15.1.0",
"socket.io": "4.7.4",
"tinyexec": "^0.3.0"
},
"devDependencies": {
"dotenv": "^16.4.2",
"esbuild": "^0.19.11",
"tsx": "^4.7.0"
}
}
-88
View File
@@ -1,88 +0,0 @@
import { setTimeout as timeout } from "node:timers/promises";
class ChaosMonkeyError extends Error {
constructor(message: string) {
super(message);
this.name = "ChaosMonkeyError";
}
}
export class ChaosMonkey {
private chaosEventRate = 0.2;
private delayInSeconds = 45;
constructor(
private enabled = false,
private disableErrors = false,
private disableDelays = false
) {
if (this.enabled) {
console.log("🍌 Chaos monkey enabled");
}
}
static Error = ChaosMonkeyError;
enable() {
this.enabled = true;
console.log("🍌 Chaos monkey enabled");
}
disable() {
this.enabled = false;
console.log("🍌 Chaos monkey disabled");
}
async call({
throwErrors = !this.disableErrors,
addDelays = !this.disableDelays,
}: {
throwErrors?: boolean;
addDelays?: boolean;
} = {}) {
if (!this.enabled) {
return;
}
const random = Math.random();
if (random > this.chaosEventRate) {
// Don't interfere with normal operation
return;
}
const chaosEvents: Array<() => Promise<any>> = [];
if (addDelays) {
chaosEvents.push(async () => {
console.log("🍌 Chaos monkey: Add delay");
await timeout(this.delayInSeconds * 1000);
});
}
if (throwErrors) {
chaosEvents.push(async () => {
console.log("🍌 Chaos monkey: Throw error");
throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
});
}
if (chaosEvents.length === 0) {
console.error("🍌 Chaos monkey: No events selected");
return;
}
const randomIndex = Math.floor(Math.random() * chaosEvents.length);
const chaosEvent = chaosEvents[randomIndex];
if (!chaosEvent) {
console.error("🍌 Chaos monkey: No event found");
return;
}
await chaosEvent();
}
}
-709
View File
@@ -1,709 +0,0 @@
import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
import { nanoid } from "nanoid";
import fs from "node:fs/promises";
import { ChaosMonkey } from "./chaosMonkey";
import { Buildah, Crictl, Exec } from "./exec";
import { setTimeout } from "node:timers/promises";
import { TempFileCleaner } from "./cleaner";
import { numFromEnv, boolFromEnv } from "./util";
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
type CheckpointerInitializeReturn = {
canCheckpoint: boolean;
willSimulate: boolean;
};
type CheckpointAndPushOptions = {
runId: string;
leaveRunning?: boolean;
projectRef: string;
deploymentVersion: string;
shouldHeartbeat?: boolean;
attemptNumber?: number;
};
type CheckpointAndPushResult =
| { success: true; checkpoint: CheckpointData }
| {
success: false;
reason?: "CANCELED" | "ERROR" | "SKIP_RETRYING";
};
type CheckpointData = {
location: string;
docker: boolean;
};
type CheckpointerOptions = {
dockerMode: boolean;
forceSimulate: boolean;
heartbeat: (runId: string) => void;
registryHost?: string;
registryNamespace?: string;
registryTlsVerify?: boolean;
disableCheckpointSupport?: boolean;
checkpointPath?: string;
simulateCheckpointFailure?: boolean;
simulateCheckpointFailureSeconds?: number;
simulatePushFailure?: boolean;
simulatePushFailureSeconds?: number;
chaosMonkey?: ChaosMonkey;
};
async function getFileSize(filePath: string): Promise<number> {
try {
const stats = await fs.stat(filePath);
return stats.size;
} catch (error) {
console.error("Error getting file size:", error);
return -1;
}
}
async function getParsedFileSize(filePath: string) {
const sizeInBytes = await getFileSize(filePath);
let message = `Size in bytes: ${sizeInBytes}`;
if (sizeInBytes > 1024 * 1024) {
const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
message = `Size in MB (rounded): ${sizeInMB}`;
} else if (sizeInBytes > 1024) {
const sizeInKB = (sizeInBytes / 1024).toFixed(2);
message = `Size in KB (rounded): ${sizeInKB}`;
}
return {
path: filePath,
sizeInBytes,
message,
};
}
export class Checkpointer {
#initialized = false;
#canCheckpoint = false;
#dockerMode: boolean;
#logger = new SimpleStructuredLogger("checkpointer");
#failedCheckpoints = new Map<string, unknown>();
// Indexed by run ID
#runAbortControllers = new Map<
string,
{ signal: AbortSignal; abort: AbortController["abort"] }
>();
private registryHost: string;
private registryNamespace: string;
private registryTlsVerify: boolean;
private disableCheckpointSupport: boolean;
private simulateCheckpointFailure: boolean;
private simulateCheckpointFailureSeconds: number;
private simulatePushFailure: boolean;
private simulatePushFailureSeconds: number;
private chaosMonkey: ChaosMonkey;
private tmpCleaner?: TempFileCleaner;
constructor(private opts: CheckpointerOptions) {
this.#dockerMode = opts.dockerMode;
this.registryHost = opts.registryHost ?? "localhost:5000";
this.registryNamespace = opts.registryNamespace ?? "trigger";
this.registryTlsVerify = opts.registryTlsVerify ?? true;
this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
this.simulatePushFailure = opts.simulatePushFailure ?? false;
this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
this.tmpCleaner = this.#createTmpCleaner();
}
async init(): Promise<CheckpointerInitializeReturn> {
if (this.#initialized) {
return this.#getInitReturn(this.#canCheckpoint);
}
this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
if (this.#dockerMode) {
const testCheckpoint = await testDockerCheckpoint();
if (testCheckpoint.ok) {
return this.#getInitReturn(true);
}
this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error });
return this.#getInitReturn(false);
}
const canLogin = await Buildah.canLogin(this.registryHost);
if (!canLogin) {
this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
}
return this.#getInitReturn(canLogin);
}
#getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
this.#canCheckpoint = canCheckpoint;
if (canCheckpoint) {
if (!this.#initialized) {
this.#logger.log("Full checkpoint support!");
}
}
this.#initialized = true;
const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
if (willSimulate) {
this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
forceSimulate: this.opts.forceSimulate,
});
}
return {
canCheckpoint,
willSimulate,
};
}
#getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
}
#getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
if (this.#dockerMode) {
return basename;
} else {
return Crictl.getExportLocation(basename);
}
}
async checkpointAndPush(
opts: CheckpointAndPushOptions,
delayMs?: number
): Promise<CheckpointData | undefined> {
const start = performance.now();
this.#logger.log(`checkpointAndPush() start`, { start, opts });
const { runId } = opts;
let interval: NodeJS.Timer | undefined;
if (opts.shouldHeartbeat) {
interval = setInterval(() => {
this.#logger.log("Sending heartbeat", { runId });
this.opts.heartbeat(runId);
}, 20_000);
}
const controller = new AbortController();
const signal = controller.signal;
const abort = controller.abort.bind(controller);
const onAbort = () => {
this.#logger.error("Checkpoint aborted", { runId, options: opts });
};
signal.addEventListener("abort", onAbort, { once: true });
const removeCurrentAbortController = () => {
const controller = this.#runAbortControllers.get(runId);
// Ensure only the current controller is removed
if (controller && controller.signal === signal) {
this.#runAbortControllers.delete(runId);
}
// Remove the abort listener in case it hasn't fired
signal.removeEventListener("abort", onAbort);
};
if (!this.#dockerMode && !this.#canCheckpoint) {
this.#logger.error("No checkpoint support. Simulation requires docker.");
this.#failCheckpoint(runId, "NO_SUPPORT");
return;
}
if (this.#isRunCheckpointing(runId)) {
this.#logger.error("Checkpoint procedure already in progress", { options: opts });
this.#failCheckpoint(runId, "IN_PROGRESS");
return;
}
// This is a new checkpoint, clear any last failure for this run
this.#clearFailedCheckpoint(runId);
if (this.disableCheckpointSupport) {
this.#logger.error("Checkpoint support disabled", { options: opts });
this.#failCheckpoint(runId, "DISABLED");
return;
}
this.#runAbortControllers.set(runId, { signal, abort });
try {
const result = await this.#checkpointAndPushWithBackoff(opts, { delayMs, signal });
const end = performance.now();
this.#logger.log(`checkpointAndPush() end`, {
start,
end,
diff: end - start,
diffWithoutDelay: end - start - (delayMs ?? 0),
opts,
success: result.success,
delayMs,
});
if (!result.success) {
return;
}
return result.checkpoint;
} finally {
if (opts.shouldHeartbeat) {
// @ts-ignore - Some kind of node incompatible type issue
clearInterval(interval);
}
removeCurrentAbortController();
}
}
#isRunCheckpointing(runId: string) {
return this.#runAbortControllers.has(runId);
}
cancelAllCheckpointsForRun(runId: string): boolean {
this.#logger.log("cancelAllCheckpointsForRun: call", { runId });
// If the last checkpoint failed, pretend we canceled it
// This ensures tasks don't wait for external resume messages to continue
if (this.#hasFailedCheckpoint(runId)) {
this.#logger.log("cancelAllCheckpointsForRun: hasFailedCheckpoint", { runId });
this.#clearFailedCheckpoint(runId);
return true;
}
const controller = this.#runAbortControllers.get(runId);
if (!controller) {
this.#logger.debug("cancelAllCheckpointsForRun: no abort controller", { runId });
return false;
}
const { abort, signal } = controller;
if (signal.aborted) {
this.#logger.debug("cancelAllCheckpointsForRun: signal already aborted", { runId });
return false;
}
abort("cancelCheckpoint()");
this.#runAbortControllers.delete(runId);
return true;
}
async #checkpointAndPushWithBackoff(
{
runId,
leaveRunning = true, // This mirrors kubernetes behaviour more accurately
projectRef,
deploymentVersion,
attemptNumber,
}: CheckpointAndPushOptions,
{ delayMs, signal }: { delayMs?: number; signal: AbortSignal }
): Promise<CheckpointAndPushResult> {
if (delayMs && delayMs > 0) {
this.#logger.log("Delaying checkpoint", { runId, delayMs });
try {
await setTimeout(delayMs, undefined, { signal });
} catch (_error) {
this.#logger.log("Checkpoint canceled during initial delay", { runId });
return { success: false, reason: "CANCELED" };
}
}
this.#logger.log("Checkpointing with backoff", {
runId,
leaveRunning,
projectRef,
deploymentVersion,
});
const backoff = new ExponentialBackoff()
.type("EqualJitter")
.base(3)
.max(3 * 3600)
.maxElapsed(48 * 3600);
for await (const { delay, retry } of backoff) {
try {
if (retry > 0) {
this.#logger.error("Retrying checkpoint", {
runId,
retry,
delay,
});
try {
await setTimeout(delay.milliseconds, undefined, { signal });
} catch (_error) {
this.#logger.log("Checkpoint canceled during retry delay", { runId });
return { success: false, reason: "CANCELED" };
}
}
const result = await this.#checkpointAndPush(
{
runId,
leaveRunning,
projectRef,
deploymentVersion,
attemptNumber,
},
{ signal }
);
if (result.success) {
return result;
}
if (result.reason === "CANCELED") {
this.#logger.log("Checkpoint canceled, won't retry", { runId });
// Don't fail the checkpoint, as it was canceled
return result;
}
if (result.reason === "SKIP_RETRYING") {
this.#logger.log("Skipping retrying", { runId });
return result;
}
continue;
} catch (error) {
this.#logger.error("Checkpoint error", {
retry,
runId,
delay,
error: error instanceof Error ? error.message : error,
});
}
}
this.#logger.error(`Checkpoint failed after exponential backoff`, {
runId,
leaveRunning,
projectRef,
deploymentVersion,
});
this.#failCheckpoint(runId, "ERROR");
return { success: false, reason: "ERROR" };
}
async #checkpointAndPush(
{
runId,
leaveRunning = true, // This mirrors kubernetes behaviour more accurately
projectRef,
deploymentVersion,
attemptNumber,
}: CheckpointAndPushOptions,
{ signal }: { signal: AbortSignal }
): Promise<CheckpointAndPushResult> {
await this.init();
const options = {
runId,
leaveRunning,
projectRef,
deploymentVersion,
attemptNumber,
};
const shortCode = nanoid(8);
const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: signal });
const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: signal });
const cleanup = async () => {
const metadata = {
runId,
exportLocation,
imageRef,
};
if (this.#dockerMode) {
this.#logger.debug("Skipping cleanup in docker mode", metadata);
return;
}
this.#logger.log("Cleaning up", metadata);
try {
await buildah.cleanup();
await crictl.cleanup();
} catch (error) {
this.#logger.error("Error during cleanup", { ...metadata, error });
}
};
try {
await this.chaosMonkey.call();
this.#logger.log("checkpointAndPush: checkpointing", { options });
const containterName = this.#getRunContainerName(runId);
// Create checkpoint (docker)
if (this.#dockerMode) {
await this.#createDockerCheckpoint(
signal,
runId,
exportLocation,
leaveRunning,
attemptNumber
);
this.#logger.log("checkpointAndPush: checkpoint created", {
runId,
location: exportLocation,
});
return {
success: true,
checkpoint: {
location: exportLocation,
docker: true,
},
};
}
// Create checkpoint (CRI)
if (!this.#canCheckpoint) {
this.#logger.error("No checkpoint support in kubernetes mode.");
return { success: false, reason: "SKIP_RETRYING" };
}
const containerId = await crictl.ps(containterName, true);
if (!containerId.stdout) {
this.#logger.error("could not find container id", { options, containterName });
return { success: false, reason: "SKIP_RETRYING" };
}
const start = performance.now();
if (this.simulateCheckpointFailure) {
if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
this.#logger.error("Simulating checkpoint failure", { options });
throw new Error("SIMULATE_CHECKPOINT_FAILURE");
}
}
// Create checkpoint
await crictl.checkpoint(containerId.stdout, exportLocation);
const postCheckpoint = performance.now();
// Print checkpoint size
const size = await getParsedFileSize(exportLocation);
this.#logger.log("checkpoint archive created", { size, options });
// Create image from checkpoint
const workingContainer = await buildah.from("scratch");
const postFrom = performance.now();
await buildah.add(workingContainer.stdout, exportLocation, "/");
const postAdd = performance.now();
await buildah.config(workingContainer.stdout, [
`io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`,
]);
const postConfig = performance.now();
await buildah.commit(workingContainer.stdout, imageRef);
const postCommit = performance.now();
if (this.simulatePushFailure) {
if (performance.now() < this.simulatePushFailureSeconds * 1000) {
this.#logger.error("Simulating push failure", { options });
throw new Error("SIMULATE_PUSH_FAILURE");
}
}
// Push checkpoint image
await buildah.push(imageRef, this.registryTlsVerify);
const postPush = performance.now();
const perf = {
"crictl checkpoint": postCheckpoint - start,
"buildah from": postFrom - postCheckpoint,
"buildah add": postAdd - postFrom,
"buildah config": postConfig - postAdd,
"buildah commit": postCommit - postConfig,
"buildah push": postPush - postCommit,
};
this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
return {
success: true,
checkpoint: {
location: imageRef,
docker: false,
},
};
} catch (error) {
if (error instanceof Exec.Result) {
if (error.aborted) {
this.#logger.error("Checkpoint canceled: Exec", { options });
return { success: false, reason: "CANCELED" };
} else {
this.#logger.error("Checkpoint command error", { options, error });
return { success: false, reason: "ERROR" };
}
}
this.#logger.error("Unhandled checkpoint error", {
options,
error: error instanceof Error ? error.message : error,
});
return { success: false, reason: "ERROR" };
} finally {
await cleanup();
if (signal.aborted) {
this.#logger.error("Checkpoint canceled: Cleanup", { options });
// Overrides any prior return value (intentional use of return-in-finally)
// eslint-disable-next-line no-unsafe-finally
return { success: false, reason: "CANCELED" };
}
}
}
async unpause(runId: string, attemptNumber?: number): Promise<void> {
try {
const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
const exec = new Exec({ logger: this.#logger });
await exec.x("docker", ["unpause", containterNameWithAttempt]);
} catch (error) {
this.#logger.error("[Docker] Error during unpause", { runId, attemptNumber, error });
}
}
async #createDockerCheckpoint(
abortSignal: AbortSignal,
runId: string,
exportLocation: string,
leaveRunning: boolean,
attemptNumber?: number
) {
const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
const exec = new Exec({ logger: this.#logger, abortSignal });
try {
if (this.opts.forceSimulate || !this.#canCheckpoint) {
this.#logger.log("Simulating checkpoint");
await exec.x("docker", ["pause", containterNameWithAttempt]);
return;
}
if (this.simulateCheckpointFailure) {
if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
this.#logger.error("Simulating checkpoint failure", {
runId,
exportLocation,
leaveRunning,
attemptNumber,
});
throw new Error("SIMULATE_CHECKPOINT_FAILURE");
}
}
const args = ["checkpoint", "create"];
if (leaveRunning) {
args.push("--leave-running");
}
args.push(containterNameWithAttempt, exportLocation);
await exec.x("docker", args);
} catch (error) {
this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
throw error;
}
}
#failCheckpoint(runId: string, error: unknown) {
this.#failedCheckpoints.set(runId, error);
}
#clearFailedCheckpoint(runId: string) {
this.#failedCheckpoints.delete(runId);
}
#hasFailedCheckpoint(runId: string) {
return this.#failedCheckpoints.has(runId);
}
#getRunContainerName(suffix: string, attemptNumber?: number) {
return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
}
#createTmpCleaner() {
if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) {
return;
}
const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean);
const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? [];
const paths = pathsOverride.length ? pathsOverride : defaultPaths;
if (paths.length === 0) {
this.#logger.error("TempFileCleaner enabled but no paths to clean", {
defaultPaths,
pathsOverride,
TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE,
});
return;
}
const cleaner = new TempFileCleaner({
paths,
maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60),
intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300),
leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false),
});
cleaner.start();
return cleaner;
}
}
-106
View File
@@ -1,106 +0,0 @@
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
import { Exec } from "./exec";
import { setTimeout } from "timers/promises";
interface TempFileCleanerOptions {
paths: string[];
maxAgeMinutes: number;
intervalSeconds: number;
leadingEdge?: boolean;
}
export class TempFileCleaner {
private enabled = false;
private logger: SimpleStructuredLogger;
private exec: Exec;
constructor(private opts: TempFileCleanerOptions) {
this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts });
this.exec = new Exec({ logger: this.logger });
}
async start() {
this.logger.log("TempFileCleaner.start");
this.enabled = true;
if (!this.opts.leadingEdge) {
await this.wait();
}
while (this.enabled) {
try {
await this.clean();
} catch (error) {
this.logger.error("error during tick", { error });
}
await this.wait();
}
}
stop() {
this.logger.log("TempFileCleaner.stop");
this.enabled = false;
}
private wait() {
return setTimeout(this.opts.intervalSeconds * 1000);
}
private async clean() {
for (const path of this.opts.paths) {
try {
await this.cleanSingle(path);
} catch (error) {
this.logger.error("error while cleaning", { path, error });
}
}
}
private async cleanSingle(startingPoint: string) {
const maxAgeMinutes = this.opts.maxAgeMinutes;
const ignoreStartingPoint = ["!", "-path", startingPoint];
const onlyDirectDescendants = ["-maxdepth", "1"];
const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`];
const baseArgs = [
startingPoint,
...ignoreStartingPoint,
...onlyDirectDescendants,
...onlyOldFiles,
];
const duArgs = ["-exec", "du", "-ch", "{}", "+"];
const rmArgs = ["-exec", "rm", "-rf", "{}", "+"];
const du = this.x("find", [...baseArgs, ...duArgs]);
const duOutput = await du;
const duLines = duOutput.stdout.trim().split("\n");
const fileCount = duLines.length - 1; // last line is the total
const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0];
if (fileCount === 0) {
this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes });
return;
}
this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
const rm = this.x("find", [...baseArgs, ...rmArgs]);
const rmOutput = await rm;
if (rmOutput.stderr.length > 0) {
this.logger.error("delete unsuccessful", { rmOutput });
return;
}
this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
}
private get x() {
return this.exec.x.bind(this.exec);
}
}
-293
View File
@@ -1,293 +0,0 @@
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
import { randomUUID } from "crypto";
import { homedir } from "os";
import { type Result, x } from "tinyexec";
class TinyResult {
pid?: number;
exitCode?: number;
aborted: boolean;
killed: boolean;
constructor(result: Result) {
this.pid = result.pid;
this.exitCode = result.exitCode;
this.aborted = result.aborted;
this.killed = result.killed;
}
}
interface ExecOptions {
logger?: SimpleStructuredLogger;
abortSignal?: AbortSignal;
logOutput?: boolean;
trimArgs?: boolean;
neverThrow?: boolean;
}
export class Exec {
private logger: SimpleStructuredLogger;
private abortSignal: AbortSignal | undefined;
private logOutput: boolean;
private trimArgs: boolean;
private neverThrow: boolean;
constructor(opts: ExecOptions) {
this.logger = opts.logger ?? new SimpleStructuredLogger("exec");
this.abortSignal = opts.abortSignal;
this.logOutput = opts.logOutput ?? true;
this.trimArgs = opts.trimArgs ?? true;
this.neverThrow = opts.neverThrow ?? false;
}
async x(
command: string,
args?: string[],
opts?: { neverThrow?: boolean; ignoreAbort?: boolean }
) {
const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args;
const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`;
this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed });
const result = x(command, argsTrimmed, {
signal: opts?.ignoreAbort ? undefined : this.abortSignal,
// We don't use this as it doesn't cover killed and aborted processes
// throwOnError: true,
});
const output = await result;
const metadata = {
command,
argsRaw: args,
argsTrimmed,
globalOpts: {
trimArgs: this.trimArgs,
neverThrow: this.neverThrow,
hasAbortSignal: !!this.abortSignal,
},
localOpts: opts,
stdout: output.stdout,
stderr: output.stderr,
pid: result.pid,
exitCode: result.exitCode,
aborted: result.aborted,
killed: result.killed,
};
if (this.logOutput) {
this.logger.debug(`output: ${commandWithFirstArg}`, metadata);
}
if (this.neverThrow || opts?.neverThrow) {
return output;
}
if (result.aborted) {
this.logger.error(`aborted: ${commandWithFirstArg}`, metadata);
throw new TinyResult(result);
}
if (result.killed) {
this.logger.error(`killed: ${commandWithFirstArg}`, metadata);
throw new TinyResult(result);
}
if (result.exitCode !== 0) {
this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata);
throw new TinyResult(result);
}
return output;
}
static Result = TinyResult;
}
interface BuildahOptions {
id?: string;
abortSignal?: AbortSignal;
}
export class Buildah {
private id: string;
private logger: SimpleStructuredLogger;
private exec: Exec;
private containers = new Set<string>();
private images = new Set<string>();
constructor(opts: BuildahOptions) {
this.id = opts.id ?? randomUUID();
this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id });
this.exec = new Exec({
logger: this.logger,
abortSignal: opts.abortSignal,
});
this.logger.log("initiaized", { opts });
}
private get x() {
return this.exec.x.bind(this.exec);
}
async from(baseImage: string) {
const output = await this.x("buildah", ["from", baseImage]);
this.containers.add(output.stdout);
return output;
}
async add(container: string, src: string, dest: string) {
return await this.x("buildah", ["add", container, src, dest]);
}
async config(container: string, annotations: string[]) {
const args = ["config"];
for (const annotation of annotations) {
args.push(`--annotation=${annotation}`);
}
args.push(container);
return await this.x("buildah", args);
}
async commit(container: string, imageRef: string) {
const output = await this.x("buildah", ["commit", container, imageRef]);
this.images.add(output.stdout);
return output;
}
async push(imageRef: string, registryTlsVerify?: boolean) {
return await this.x("buildah", [
"push",
`--tls-verify=${String(!!registryTlsVerify)}`,
imageRef,
]);
}
async cleanup() {
if (this.containers.size > 0) {
try {
const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true });
this.containers.clear();
if (output.stderr.length > 0) {
this.logger.error("failed to remove some containers", { output });
}
} catch (error) {
this.logger.error("failed to clean up containers", { error, containers: this.containers });
}
} else {
this.logger.debug("no containers to clean up");
}
if (this.images.size > 0) {
try {
const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true });
this.images.clear();
if (output.stderr.length > 0) {
this.logger.error("failed to remove some images", { output });
}
} catch (error) {
this.logger.error("failed to clean up images", { error, images: this.images });
}
} else {
this.logger.debug("no images to clean up");
}
}
static async canLogin(registryHost: string) {
try {
await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true });
return true;
} catch (_error) {
return false;
}
}
static get tmpDir() {
return process.env.TMPDIR ?? "/var/tmp";
}
static get storageRootDir() {
return process.getuid?.() === 0
? "/var/lib/containers/storage"
: `${homedir()}/.local/share/containers/storage`;
}
}
interface CrictlOptions {
id?: string;
abortSignal?: AbortSignal;
}
export class Crictl {
private id: string;
private logger: SimpleStructuredLogger;
private exec: Exec;
private archives = new Set<string>();
constructor(opts: CrictlOptions) {
this.id = opts.id ?? randomUUID();
this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id });
this.exec = new Exec({
logger: this.logger,
abortSignal: opts.abortSignal,
});
this.logger.log("initiaized", { opts });
}
private get x() {
return this.exec.x.bind(this.exec);
}
async ps(containerName: string, quiet?: boolean) {
return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]);
}
async checkpoint(containerId: string, exportLocation: string) {
const output = await this.x("crictl", [
"checkpoint",
`--export=${exportLocation}`,
containerId,
]);
this.archives.add(exportLocation);
return output;
}
async cleanup() {
if (this.archives.size > 0) {
try {
const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true });
this.archives.clear();
if (output.stderr.length > 0) {
this.logger.error("failed to remove some archives", { output });
}
} catch (error) {
this.logger.error("failed to clean up archives", { error, archives: this.archives });
}
} else {
this.logger.debug("no archives to clean up");
}
}
static getExportLocation(identifier: string) {
return `${this.checkpointDir}/${identifier}.tar`;
}
static get checkpointDir() {
return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints";
}
}
File diff suppressed because it is too large Load Diff
-31
View File
@@ -1,31 +0,0 @@
export const boolFromEnv = (env: string, defaultValue: boolean): boolean => {
const value = process.env[env];
if (!value) {
return defaultValue;
}
return ["1", "true"].includes(value);
};
export const numFromEnv = (env: string, defaultValue: number): number => {
const value = process.env[env];
if (!value) {
return defaultValue;
}
return parseInt(value, 10);
};
export function safeJsonParse(json?: string): unknown {
if (!json) {
return;
}
try {
return JSON.parse(json);
} catch (_e) {
return null;
}
}
-15
View File
@@ -1,15 +0,0 @@
{
"compilerOptions": {
"target": "es2020",
"module": "commonjs",
"esModuleInterop": true,
"resolveJsonModule": true,
"forceConsistentCasingInFileNames": true,
"strict": true,
"skipLibCheck": true,
"paths": {
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
}
}
}
-11
View File
@@ -1,11 +0,0 @@
HTTP_SERVER_PORT=8050
PLATFORM_WS_PORT=3030
PLATFORM_SECRET=provider-secret
SECURE_CONNECTION=false
OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
# Use this if you are on macOS
# COORDINATOR_HOST="host.docker.internal"
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
-3
View File
@@ -1,3 +0,0 @@
dist/
node_modules/
.env
-47
View File
@@ -1,47 +0,0 @@
FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine
WORKDIR /app
FROM node-22-alpine AS pruner
COPY --chown=node:node . .
RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
FROM node-22-alpine AS base
RUN apk add --no-cache dumb-init docker
COPY --chown=node:node .gitignore .gitignore
COPY --from=pruner --chown=node:node /app/out/json/ .
COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
FROM base AS dev-deps
RUN corepack enable
ENV NODE_ENV development
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
FROM base AS builder
RUN corepack enable
COPY --from=pruner --chown=node:node /app/out/full/ .
COPY --from=dev-deps --chown=node:node /app/ .
COPY --chown=node:node turbo.json turbo.json
RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter docker-provider build:bundle
FROM base AS runner
RUN corepack enable
ENV NODE_ENV production
COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
EXPOSE 8000
USER node
CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]
-3
View File
@@ -1,3 +0,0 @@
# Docker provider
The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.
-27
View File
@@ -1,27 +0,0 @@
{
"name": "docker-provider",
"private": true,
"version": "0.0.1",
"description": "",
"main": "dist/index.cjs",
"scripts": {
"build": "npm run build:bundle",
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
"build:image": "docker build -f Containerfile . -t docker-provider",
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
"start": "tsx src/index.ts",
"typecheck": "tsc --noEmit"
},
"keywords": [],
"author": "",
"license": "MIT",
"dependencies": {
"@trigger.dev/core": "workspace:*",
"execa": "^8.0.1"
},
"devDependencies": {
"dotenv": "^16.4.2",
"esbuild": "^0.19.11",
"tsx": "^4.7.0"
}
}
-295
View File
@@ -1,295 +0,0 @@
import type { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
import type {
TaskOperations,
TaskOperationsCreateOptions,
TaskOperationsIndexOptions,
TaskOperationsRestoreOptions,
} from "@trigger.dev/core/v3/apps";
import { ProviderShell, SimpleLogger, isExecaChildProcess } from "@trigger.dev/core/v3/apps";
import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
import { $, type ExecaChildProcess, execa } from "execa";
import { setTimeout } from "node:timers/promises";
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host";
const OTEL_EXPORTER_OTLP_ENDPOINT =
process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
process.env.FORCE_CHECKPOINT_SIMULATION ?? "false"
);
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
type TaskOperationsInitReturn = {
canCheckpoint: boolean;
willSimulate: boolean;
};
class DockerTaskOperations implements TaskOperations {
#initialized = false;
#canCheckpoint = false;
constructor(private opts = { forceSimulate: false }) {}
async init(): Promise<TaskOperationsInitReturn> {
if (this.#initialized) {
return this.#getInitReturn(this.#canCheckpoint);
}
logger.log("Initializing task operations");
const testCheckpoint = await testDockerCheckpoint();
if (testCheckpoint.ok) {
return this.#getInitReturn(true);
}
logger.error(testCheckpoint.message, testCheckpoint.error);
return this.#getInitReturn(false);
}
#getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
this.#canCheckpoint = canCheckpoint;
if (canCheckpoint) {
if (!this.#initialized) {
logger.log("Full checkpoint support!");
}
}
this.#initialized = true;
const willSimulate = !canCheckpoint || this.opts.forceSimulate;
if (willSimulate) {
logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
forceSimulate: this.opts.forceSimulate,
});
}
return {
canCheckpoint,
willSimulate,
};
}
async index(opts: TaskOperationsIndexOptions) {
await this.init();
const containerName = this.#getIndexContainerName(opts.shortCode);
logger.log(`Indexing task ${opts.imageRef}`, {
host: COORDINATOR_HOST,
port: COORDINATOR_PORT,
});
logger.debug(
await execa("docker", [
"run",
`--network=${DOCKER_NETWORK}`,
"--rm",
`--env=INDEX_TASKS=true`,
`--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
`--env=TRIGGER_API_URL=${opts.apiUrl}`,
`--env=TRIGGER_ENV_ID=${opts.envId}`,
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
`--env=POD_NAME=${containerName}`,
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
`--name=${containerName}`,
`${opts.imageRef}`,
])
);
}
async create(opts: TaskOperationsCreateOptions) {
await this.init();
const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
const runArgs = [
"run",
`--network=${DOCKER_NETWORK}`,
"--detach",
`--env=TRIGGER_ENV_ID=${opts.envId}`,
`--env=TRIGGER_RUN_ID=${opts.runId}`,
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
`--env=POD_NAME=${containerName}`,
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
`--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
`--name=${containerName}`,
];
if (process.env.ENFORCE_MACHINE_PRESETS) {
runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
}
if (opts.dequeuedAt) {
runArgs.push(`--env=TRIGGER_RUN_DEQUEUED_AT_MS=${opts.dequeuedAt}`);
}
runArgs.push(`${opts.image}`);
try {
logger.debug(await execa("docker", runArgs));
} catch (error) {
if (!isExecaChildProcess(error)) {
throw error;
}
logger.error("Create failed:", {
opts,
exitCode: error.exitCode,
escapedCommand: error.escapedCommand,
stdout: error.stdout,
stderr: error.stderr,
});
}
}
async restore(opts: TaskOperationsRestoreOptions) {
await this.init();
const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
if (!this.#canCheckpoint || this.opts.forceSimulate) {
logger.log("Simulating restore");
const unpause = logger.debug(await $`docker unpause ${containerName}`);
if (unpause.exitCode !== 0) {
throw new Error("docker unpause command failed");
}
await this.#sendPostStart(containerName);
return;
}
const { exitCode } = logger.debug(
await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}`
);
if (exitCode !== 0) {
throw new Error("docker start command failed");
}
await this.#sendPostStart(containerName);
}
async delete(opts: { runId: string }) {
await this.init();
const containerName = this.#getRunContainerName(opts.runId);
await this.#sendPreStop(containerName);
logger.log("noop: delete");
}
async get(opts: { runId: string }) {
await this.init();
logger.log("noop: get");
}
#getIndexContainerName(suffix: string) {
return `task-index-${suffix}`;
}
#getRunContainerName(suffix: string, attemptNumber?: number) {
return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
}
async #sendPostStart(containerName: string): Promise<void> {
try {
const port = await this.#getHttpServerPort(containerName);
logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
} catch (error) {
logger.error("postStart error", { error });
throw new Error("postStart command failed");
}
}
async #sendPreStop(containerName: string): Promise<void> {
try {
const port = await this.#getHttpServerPort(containerName);
logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
} catch (error) {
logger.error("preStop error", { error });
throw new Error("preStop command failed");
}
}
async #getHttpServerPort(containerName: string): Promise<number> {
// We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
// FIXME: Skip this in prod
const logs = logger.debug(await $`docker logs ${containerName}`);
const matches = logs.stdout.match(/http server listening on port (?<port>[0-9]+)/);
const port = Number(matches?.groups?.port);
if (!port) {
throw new Error("failed to extract port from logs");
}
return port;
}
async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
containerName: string,
port: number,
type: THookType,
cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
retryCount = 0
): Promise<ExecaChildProcess> {
try {
return await execa("docker", [
"exec",
containerName,
"busybox",
"wget",
"-q",
"-O-",
`127.0.0.1:${port}/${type}?cause=${cause}`,
]);
} catch (error: any) {
if (type === "postStart" && retryCount < 6) {
logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
}
logger.error(`final ${type} error`, { message: error?.message });
throw new Error(`${type} command failed after ${retryCount - 1} retries`);
}
}
}
const provider = new ProviderShell({
tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
type: "docker",
});
provider.listen();
function exponentialBackoff(
retryCount: number,
exponential: number,
minDelay: number,
maxDelay: number,
jitter: number
): number {
// Calculate the delay using the exponential backoff formula
const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
// Calculate the jitter
const jitterValue = Math.random() * jitter;
// Return the calculated delay with jitter
return delay + jitterValue;
}
-15
View File
@@ -1,15 +0,0 @@
{
"compilerOptions": {
"target": "es2020",
"module": "commonjs",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"strict": true,
"skipLibCheck": true,
"paths": {
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
}
}
}
-9
View File
@@ -1,9 +0,0 @@
HTTP_SERVER_PORT=8060
PLATFORM_WS_PORT=3030
PLATFORM_SECRET=provider-secret
SECURE_CONNECTION=false
# Use this if you are on macOS
# COORDINATOR_HOST="host.docker.internal"
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
-3
View File
@@ -1,3 +0,0 @@
dist/
node_modules/
.env
-47
View File
@@ -1,47 +0,0 @@
FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine
WORKDIR /app
FROM node-22-alpine AS pruner
COPY --chown=node:node . .
RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
FROM node-22-alpine AS base
RUN apk add --no-cache dumb-init
COPY --chown=node:node .gitignore .gitignore
COPY --from=pruner --chown=node:node /app/out/json/ .
COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
FROM base AS dev-deps
RUN corepack enable
ENV NODE_ENV development
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
FROM base AS builder
RUN corepack enable
COPY --from=pruner --chown=node:node /app/out/full/ .
COPY --from=dev-deps --chown=node:node /app/ .
COPY --chown=node:node turbo.json turbo.json
RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter kubernetes-provider build:bundle
FROM base AS runner
RUN corepack enable
ENV NODE_ENV production
COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs
EXPOSE 8000
USER node
CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]
-3
View File
@@ -1,3 +0,0 @@
# Kubernetes provider
The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions.
-28
View File
@@ -1,28 +0,0 @@
{
"name": "kubernetes-provider",
"private": true,
"version": "0.0.1",
"description": "",
"main": "dist/index.cjs",
"scripts": {
"build": "npm run build:bundle",
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
"build:image": "docker build -f Containerfile . -t kubernetes-provider",
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
"start": "tsx src/index.ts",
"typecheck": "tsc --noEmit"
},
"keywords": [],
"author": "",
"license": "MIT",
"dependencies": {
"@kubernetes/client-node": "^0.20.0",
"@trigger.dev/core": "workspace:*",
"p-queue": "^8.0.1"
},
"devDependencies": {
"dotenv": "^16.4.2",
"esbuild": "^0.19.11",
"tsx": "^4.7.0"
}
}
-782
View File
@@ -1,782 +0,0 @@
import * as k8s from "@kubernetes/client-node";
import type {
EnvironmentType,
MachinePreset,
PostStartCauses,
PreStopCauses,
} from "@trigger.dev/core/v3";
import type {
TaskOperations,
TaskOperationsCreateOptions,
TaskOperationsIndexOptions,
TaskOperationsPrePullDeploymentOptions,
TaskOperationsRestoreOptions,
} from "@trigger.dev/core/v3/apps";
import { ProviderShell, SimpleLogger } from "@trigger.dev/core/v3/apps";
import { PodCleaner } from "./podCleaner";
import { TaskMonitor } from "./taskMonitor";
import { UptimeHeartbeat } from "./uptimeHeartbeat";
import { assertExhaustive } from "@trigger.dev/core";
import { CustomLabelHelper } from "./labelHelper";
const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
const NODE_NAME = process.env.NODE_NAME || "local";
const OTEL_EXPORTER_OTLP_ENDPOINT =
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
const COORDINATOR_HOST = process.env.COORDINATOR_HOST ?? undefined;
const COORDINATOR_PORT = process.env.COORDINATOR_PORT ?? undefined;
const KUBERNETES_NAMESPACE = process.env.KUBERNETES_NAMESPACE ?? "default";
const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi";
const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi";
// Image config
const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true";
const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS;
const PAUSE_IMAGE = process.env.PAUSE_IMAGE || "registry.k8s.io/pause:3.9";
const BUSYBOX_IMAGE = process.env.BUSYBOX_IMAGE || "registry.digitalocean.com/trigger/busybox";
const DEPLOYMENT_IMAGE_PREFIX = process.env.DEPLOYMENT_IMAGE_PREFIX;
const RESTORE_IMAGE_PREFIX = process.env.RESTORE_IMAGE_PREFIX;
const UTILITY_IMAGE_PREFIX = process.env.UTILITY_IMAGE_PREFIX;
const logger = new SimpleLogger(`[${NODE_NAME}]`);
logger.log(`running in ${RUNTIME_ENV} mode`);
type Namespace = {
metadata: {
name: string;
};
};
type ResourceQuantities = {
[K in "cpu" | "memory" | "ephemeral-storage"]?: string;
};
class KubernetesTaskOperations implements TaskOperations {
#namespace: Namespace = {
metadata: {
name: "default",
},
};
#k8sApi: {
core: k8s.CoreV1Api;
batch: k8s.BatchV1Api;
apps: k8s.AppsV1Api;
};
#labelHelper = new CustomLabelHelper();
constructor(opts: { namespace?: string } = {}) {
if (opts.namespace) {
this.#namespace.metadata.name = opts.namespace;
}
this.#k8sApi = this.#createK8sApi();
}
async init() {
// noop
}
async index(opts: TaskOperationsIndexOptions) {
await this.#createJob(
{
metadata: {
name: this.#getIndexContainerName(opts.shortCode),
namespace: this.#namespace.metadata.name,
},
spec: {
completions: 1,
backoffLimit: 0,
ttlSecondsAfterFinished: 300,
template: {
metadata: {
labels: {
...this.#getSharedLabels(opts),
app: "task-index",
"app.kubernetes.io/part-of": "trigger-worker",
"app.kubernetes.io/component": "index",
deployment: opts.deploymentId,
},
},
spec: {
...this.#defaultPodSpec,
containers: [
{
name: this.#getIndexContainerName(opts.shortCode),
image: getImageRef("deployment", opts.imageRef),
ports: [
{
containerPort: 8000,
},
],
resources: {
limits: {
cpu: "1",
memory: "2G",
"ephemeral-storage": "2Gi",
},
},
lifecycle: {
preStop: {
exec: {
command: this.#getLifecycleCommand("preStop", "terminate"),
},
},
},
env: [
...this.#getSharedEnv(opts.envId),
{
name: "INDEX_TASKS",
value: "true",
},
{
name: "TRIGGER_SECRET_KEY",
value: opts.apiKey,
},
{
name: "TRIGGER_API_URL",
value: opts.apiUrl,
},
],
},
],
},
},
},
},
this.#namespace
);
}
async create(opts: TaskOperationsCreateOptions) {
const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
await this.#createPod(
{
metadata: {
name: containerName,
namespace: this.#namespace.metadata.name,
labels: {
...this.#labelHelper.getAdditionalLabels("create"),
...this.#getSharedLabels(opts),
app: "task-run",
"app.kubernetes.io/part-of": "trigger-worker",
"app.kubernetes.io/component": "create",
run: opts.runId,
},
},
spec: {
...this.#defaultPodSpec,
terminationGracePeriodSeconds: 60 * 60,
containers: [
{
name: containerName,
image: getImageRef("deployment", opts.image),
ports: [
{
containerPort: 8000,
},
],
resources: this.#getResourcesForMachine(opts.machine),
lifecycle: {
preStop: {
exec: {
command: this.#getLifecycleCommand("preStop", "terminate"),
},
},
},
env: [
...this.#getSharedEnv(opts.envId),
{
name: "TRIGGER_RUN_ID",
value: opts.runId,
},
...(opts.dequeuedAt
? [{ name: "TRIGGER_RUN_DEQUEUED_AT_MS", value: String(opts.dequeuedAt) }]
: []),
],
volumeMounts: [
{
name: "taskinfo",
mountPath: "/etc/taskinfo",
},
],
},
],
volumes: [
{
name: "taskinfo",
emptyDir: {},
},
],
},
},
this.#namespace
);
}
async restore(opts: TaskOperationsRestoreOptions) {
await this.#createPod(
{
metadata: {
name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
namespace: this.#namespace.metadata.name,
labels: {
...this.#labelHelper.getAdditionalLabels("restore"),
...this.#getSharedLabels(opts),
app: "task-run",
"app.kubernetes.io/part-of": "trigger-worker",
"app.kubernetes.io/component": "restore",
run: opts.runId,
checkpoint: opts.checkpointId,
},
},
spec: {
...this.#defaultPodSpec,
initContainers: [
{
name: "pull-base-image",
image: getImageRef("deployment", opts.imageRef),
command: ["sleep", "0"],
},
{
name: "populate-taskinfo",
image: getImageRef("utility", BUSYBOX_IMAGE),
imagePullPolicy: "IfNotPresent",
command: ["/bin/sh", "-c"],
args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
env: this.#coordinatorEnvVars,
volumeMounts: [
{
name: "taskinfo",
mountPath: "/etc/taskinfo",
},
],
},
],
containers: [
{
name: this.#getRunContainerName(opts.runId),
image: getImageRef("restore", opts.checkpointRef),
ports: [
{
containerPort: 8000,
},
],
resources: this.#getResourcesForMachine(opts.machine),
lifecycle: {
postStart: {
exec: {
command: this.#getLifecycleCommand("postStart", "restore"),
},
},
preStop: {
exec: {
command: this.#getLifecycleCommand("preStop", "terminate"),
},
},
},
volumeMounts: [
{
name: "taskinfo",
mountPath: "/etc/taskinfo",
},
],
},
],
volumes: [
{
name: "taskinfo",
emptyDir: {},
},
],
},
},
this.#namespace
);
}
async delete(opts: { runId: string }) {
await this.#deletePod({
runId: opts.runId,
namespace: this.#namespace,
});
}
async get(opts: { runId: string }) {
await this.#getPod(opts.runId, this.#namespace);
}
async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
if (PRE_PULL_DISABLED) {
logger.debug("Pre-pull is disabled, skipping.", { opts });
return;
}
const metaName = this.#getPrePullContainerName(opts.shortCode);
const metaLabels = {
...this.#getSharedLabels(opts),
app: "task-prepull",
"app.kubernetes.io/part-of": "trigger-worker",
"app.kubernetes.io/component": "prepull",
deployment: opts.deploymentId,
name: metaName,
} satisfies k8s.V1ObjectMeta["labels"];
await this.#createDaemonSet(
{
metadata: {
name: metaName,
namespace: this.#namespace.metadata.name,
labels: metaLabels,
},
spec: {
selector: {
matchLabels: {
name: metaName,
},
},
template: {
metadata: {
labels: metaLabels,
},
spec: {
...this.#defaultPodSpec,
restartPolicy: "Always",
affinity: {
nodeAffinity: {
requiredDuringSchedulingIgnoredDuringExecution: {
nodeSelectorTerms: [
{
matchExpressions: [
{
key: "trigger.dev/pre-pull-disabled",
operator: "DoesNotExist",
},
],
},
],
},
},
},
initContainers: [
{
name: "prepull",
image: getImageRef("deployment", opts.imageRef),
command: ["/usr/bin/true"],
resources: {
limits: {
cpu: "0.25",
memory: "100Mi",
"ephemeral-storage": "1Gi",
},
},
},
],
containers: [
{
name: "pause",
image: getImageRef("utility", PAUSE_IMAGE),
resources: {
limits: {
cpu: "1m",
memory: "12Mi",
},
},
},
],
},
},
},
},
this.#namespace
);
}
#envTypeToLabelValue(type: EnvironmentType) {
switch (type) {
case "PRODUCTION":
return "prod";
case "STAGING":
return "stg";
case "DEVELOPMENT":
return "dev";
case "PREVIEW":
return "preview";
}
}
get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
const pullSecrets = ["registry-trigger", "registry-trigger-failover"];
if (ADDITIONAL_PULL_SECRETS) {
pullSecrets.push(...ADDITIONAL_PULL_SECRETS.split(","));
}
const imagePullSecrets = pullSecrets.map(
(name) => ({ name }) satisfies k8s.V1LocalObjectReference
);
return {
restartPolicy: "Never",
automountServiceAccountToken: false,
imagePullSecrets,
nodeSelector: {
nodetype: "worker",
},
};
}
get #defaultResourceRequests(): ResourceQuantities {
return {
"ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST,
};
}
get #defaultResourceLimits(): ResourceQuantities {
return {
"ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT,
};
}
get #coordinatorHostEnvVar(): k8s.V1EnvVar {
return COORDINATOR_HOST
? {
name: "COORDINATOR_HOST",
value: COORDINATOR_HOST,
}
: {
name: "COORDINATOR_HOST",
valueFrom: {
fieldRef: {
fieldPath: "status.hostIP",
},
},
};
}
get #coordinatorPortEnvVar(): k8s.V1EnvVar | undefined {
if (COORDINATOR_PORT) {
return {
name: "COORDINATOR_PORT",
value: COORDINATOR_PORT,
};
}
}
get #coordinatorEnvVars(): k8s.V1EnvVar[] {
const envVars = [this.#coordinatorHostEnvVar];
if (this.#coordinatorPortEnvVar) {
envVars.push(this.#coordinatorPortEnvVar);
}
return envVars;
}
#getSharedEnv(envId: string): k8s.V1EnvVar[] {
return [
{
name: "TRIGGER_ENV_ID",
value: envId,
},
{
name: "DEBUG",
value: process.env.DEBUG ? "1" : "0",
},
{
name: "HTTP_SERVER_PORT",
value: "8000",
},
{
name: "OTEL_EXPORTER_OTLP_ENDPOINT",
value: OTEL_EXPORTER_OTLP_ENDPOINT,
},
{
name: "POD_NAME",
valueFrom: {
fieldRef: {
fieldPath: "metadata.name",
},
},
},
{
name: "MACHINE_NAME",
valueFrom: {
fieldRef: {
fieldPath: "spec.nodeName",
},
},
},
{
name: "TRIGGER_POD_SCHEDULED_AT_MS",
value: Date.now().toString(),
},
...this.#coordinatorEnvVars,
];
}
#getSharedLabels(
opts:
| TaskOperationsIndexOptions
| TaskOperationsCreateOptions
| TaskOperationsRestoreOptions
| TaskOperationsPrePullDeploymentOptions
): Record<string, string> {
return {
env: opts.envId,
envtype: this.#envTypeToLabelValue(opts.envType),
org: opts.orgId,
project: opts.projectId,
};
}
#getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
return {
cpu: `${preset.cpu * 0.75}`,
memory: `${preset.memory}G`,
};
}
#getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
return {
cpu: `${preset.cpu}`,
memory: `${preset.memory}G`,
};
}
#getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements {
return {
requests: {
...this.#defaultResourceRequests,
...this.#getResourceRequestsForMachine(preset),
},
limits: {
...this.#defaultResourceLimits,
...this.#getResourceLimitsForMachine(preset),
},
};
}
#getLifecycleCommand<THookType extends "postStart" | "preStop">(
type: THookType,
cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
) {
const retries = 5;
// This will retry sending the lifecycle hook up to `retries` times
// The sleep is required as this may start running before the HTTP server is up
const exec = [
"/bin/sh",
"-c",
`for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
];
logger.debug("getLifecycleCommand()", { exec });
return exec;
}
#getIndexContainerName(suffix: string) {
return `task-index-${suffix}`;
}
#getRunContainerName(suffix: string, attemptNumber?: number) {
return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
}
#getPrePullContainerName(suffix: string) {
return `task-prepull-${suffix}`;
}
#createK8sApi() {
const kubeConfig = new k8s.KubeConfig();
if (RUNTIME_ENV === "local") {
kubeConfig.loadFromDefault();
} else if (RUNTIME_ENV === "kubernetes") {
kubeConfig.loadFromCluster();
} else {
throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`);
}
return {
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
};
}
async #createPod(pod: k8s.V1Pod, namespace: Namespace) {
try {
const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
logger.debug(res.body);
} catch (err: unknown) {
this.#handleK8sError(err);
}
}
async #deletePod(opts: { runId: string; namespace: Namespace }) {
try {
const res = await this.#k8sApi.core.deleteNamespacedPod(
opts.runId,
opts.namespace.metadata.name
);
logger.debug(res.body);
} catch (err: unknown) {
this.#handleK8sError(err);
}
}
async #getPod(runId: string, namespace: Namespace) {
try {
const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name);
logger.debug(res.body);
return res.body;
} catch (err: unknown) {
this.#handleK8sError(err);
}
}
async #createJob(job: k8s.V1Job, namespace: Namespace) {
try {
const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
logger.debug(res.body);
} catch (err: unknown) {
this.#handleK8sError(err);
}
}
async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
try {
const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
namespace.metadata.name,
daemonSet
);
logger.debug(res.body);
} catch (err: unknown) {
this.#handleK8sError(err);
}
}
#throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
if (typeof candidate !== "object" || candidate === null) {
throw candidate;
}
}
#handleK8sError(err: unknown) {
this.#throwUnlessRecord(err);
if ("body" in err && err.body) {
logger.error(err.body);
this.#throwUnlessRecord(err.body);
if (typeof err.body.message === "string") {
throw new Error(err.body?.message);
} else {
throw err.body;
}
} else {
logger.error(err);
throw err;
}
}
}
type ImageType = "deployment" | "restore" | "utility";
function getImagePrefix(type: ImageType) {
switch (type) {
case "deployment":
return DEPLOYMENT_IMAGE_PREFIX;
case "restore":
return RESTORE_IMAGE_PREFIX;
case "utility":
return UTILITY_IMAGE_PREFIX;
default:
assertExhaustive(type);
}
}
function getImageRef(type: ImageType, ref: string) {
const prefix = getImagePrefix(type);
return prefix ? `${prefix}/${ref}` : ref;
}
const provider = new ProviderShell({
tasks: new KubernetesTaskOperations({
namespace: KUBERNETES_NAMESPACE,
}),
type: "kubernetes",
});
provider.listen();
const taskMonitor = new TaskMonitor({
runtimeEnv: RUNTIME_ENV,
namespace: KUBERNETES_NAMESPACE,
onIndexFailure: async (deploymentId, details) => {
logger.log("Indexing failed", { deploymentId, details });
try {
provider.platformSocket.send("INDEXING_FAILED", {
deploymentId,
error: {
name: `Crashed with exit code ${details.exitCode}`,
message: details.reason,
stack: details.logs,
},
overrideCompletion: details.overrideCompletion,
});
} catch (error) {
logger.error(error);
}
},
onRunFailure: async (runId, details) => {
logger.log("Run failed:", { runId, details });
try {
provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
} catch (error) {
logger.error(error);
}
},
});
taskMonitor.start();
const podCleaner = new PodCleaner({
runtimeEnv: RUNTIME_ENV,
namespace: KUBERNETES_NAMESPACE,
intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
});
podCleaner.start();
if (UPTIME_HEARTBEAT_URL) {
const uptimeHeartbeat = new UptimeHeartbeat({
runtimeEnv: RUNTIME_ENV,
namespace: KUBERNETES_NAMESPACE,
intervalInSeconds: UPTIME_INTERVAL_SECONDS,
pingUrl: UPTIME_HEARTBEAT_URL,
maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
});
uptimeHeartbeat.start();
} else {
logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
}
-153
View File
@@ -1,153 +0,0 @@
import { assertExhaustive } from "@trigger.dev/core";
const CREATE_LABEL_ENV_VAR_PREFIX = "DEPLOYMENT_LABEL_";
const RESTORE_LABEL_ENV_VAR_PREFIX = "RESTORE_LABEL_";
const LABEL_SAMPLE_RATE_POSTFIX = "_SAMPLE_RATE";
type OperationType = "create" | "restore";
type CustomLabel = {
key: string;
value: string;
sampleRate: number;
};
export class CustomLabelHelper {
// Labels and sample rates are defined in environment variables so only need to be computed once
private createLabels?: CustomLabel[];
private restoreLabels?: CustomLabel[];
private getLabelPrefix(type: OperationType) {
const prefix = type === "create" ? CREATE_LABEL_ENV_VAR_PREFIX : RESTORE_LABEL_ENV_VAR_PREFIX;
return prefix.toLowerCase();
}
private getLabelSampleRatePostfix() {
return LABEL_SAMPLE_RATE_POSTFIX.toLowerCase();
}
// Can only range from 0 to 1
private fractionFromPercent(percent: number) {
return Math.min(1, Math.max(0, percent / 100));
}
private isLabelSampleRateEnvVar(key: string) {
return key.toLowerCase().endsWith(this.getLabelSampleRatePostfix());
}
private isLabelEnvVar(type: OperationType, key: string) {
const prefix = this.getLabelPrefix(type);
return key.toLowerCase().startsWith(prefix) && !this.isLabelSampleRateEnvVar(key);
}
private getSampleRateEnvVarKey(type: OperationType, envKey: string) {
return `${envKey.toLowerCase()}${this.getLabelSampleRatePostfix()}`;
}
private getLabelNameFromEnvVarKey(type: OperationType, key: string) {
return key
.slice(this.getLabelPrefix(type).length)
.toLowerCase()
.replace(/___/g, ".")
.replace(/__/g, "/")
.replace(/_/g, "-");
}
private getCaseInsensitiveEnvValue(key: string) {
for (const [envKey, value] of Object.entries(process.env)) {
if (envKey.toLowerCase() === key.toLowerCase()) {
return value;
}
}
}
/** Returns the sample rate for a given label as fraction of 100 */
private getSampleRateFromEnvVarKey(type: OperationType, envKey: string) {
// Apply default: always sample
const DEFAULT_SAMPLE_RATE_PERCENT = 100;
const defaultSampleRateFraction = this.fractionFromPercent(DEFAULT_SAMPLE_RATE_PERCENT);
const value = this.getCaseInsensitiveEnvValue(this.getSampleRateEnvVarKey(type, envKey));
if (!value) {
return defaultSampleRateFraction;
}
const sampleRatePercent = parseFloat(value || String(DEFAULT_SAMPLE_RATE_PERCENT));
if (isNaN(sampleRatePercent)) {
return defaultSampleRateFraction;
}
const fractionalSampleRate = this.fractionFromPercent(sampleRatePercent);
return fractionalSampleRate;
}
private getCustomLabels(type: OperationType): CustomLabel[] {
switch (type) {
case "create":
if (this.createLabels) {
return this.createLabels;
}
break;
case "restore":
if (this.restoreLabels) {
return this.restoreLabels;
}
break;
default:
assertExhaustive(type);
}
const customLabels: CustomLabel[] = [];
for (const [envKey, value] of Object.entries(process.env)) {
const key = envKey.toLowerCase();
// Only process env vars that start with the expected prefix
if (!this.isLabelEnvVar(type, key)) {
continue;
}
// Skip sample rates - deal with them separately
if (this.isLabelSampleRateEnvVar(key)) {
continue;
}
const labelName = this.getLabelNameFromEnvVarKey(type, key);
const sampleRate = this.getSampleRateFromEnvVarKey(type, key);
const label = {
key: labelName,
value: value || "",
sampleRate,
} satisfies CustomLabel;
customLabels.push(label);
}
return customLabels;
}
getAdditionalLabels(type: OperationType): Record<string, string> {
const labels = this.getCustomLabels(type);
const additionalLabels: Record<string, string> = {};
for (const { key, value, sampleRate } of labels) {
// Always apply label if sample rate is 1
if (sampleRate === 1) {
additionalLabels[key] = value;
continue;
}
if (Math.random() <= sampleRate) {
additionalLabels[key] = value;
continue;
}
}
return additionalLabels;
}
}
-251
View File
@@ -1,251 +0,0 @@
import * as k8s from "@kubernetes/client-node";
import { SimpleLogger } from "@trigger.dev/core/v3/apps";
type PodCleanerOptions = {
runtimeEnv: "local" | "kubernetes";
namespace?: string;
intervalInSeconds?: number;
};
export class PodCleaner {
private enabled = false;
private namespace = "default";
private intervalInSeconds = 300;
private logger = new SimpleLogger("[PodCleaner]");
private k8sClient: {
core: k8s.CoreV1Api;
apps: k8s.AppsV1Api;
kubeConfig: k8s.KubeConfig;
};
constructor(private opts: PodCleanerOptions) {
if (opts.namespace) {
this.namespace = opts.namespace;
}
if (opts.intervalInSeconds) {
this.intervalInSeconds = opts.intervalInSeconds;
}
this.k8sClient = this.#createK8sClient();
}
#createK8sClient() {
const kubeConfig = new k8s.KubeConfig();
if (this.opts.runtimeEnv === "local") {
kubeConfig.loadFromDefault();
} else if (this.opts.runtimeEnv === "kubernetes") {
kubeConfig.loadFromCluster();
} else {
throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
}
return {
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
kubeConfig: kubeConfig,
};
}
#isRecord(candidate: unknown): candidate is Record<string, unknown> {
if (typeof candidate !== "object" || candidate === null) {
return false;
} else {
return true;
}
}
#logK8sError(err: unknown, debugOnly = false) {
if (debugOnly) {
this.logger.debug("K8s API Error", err);
} else {
this.logger.error("K8s API Error", err);
}
}
#handleK8sError(err: unknown) {
if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
this.#logK8sError(err);
return;
}
this.#logK8sError(err, true);
if (typeof err.body.message === "string") {
this.#logK8sError({ message: err.body.message });
return;
}
this.#logK8sError({ body: err.body });
}
async #deletePods(opts: {
namespace: string;
dryRun?: boolean;
fieldSelector?: string;
labelSelector?: string;
}) {
return await this.k8sClient.core
.deleteCollectionNamespacedPod(
opts.namespace,
undefined, // pretty
undefined, // continue
opts.dryRun ? "All" : undefined,
opts.fieldSelector,
undefined, // gracePeriodSeconds
opts.labelSelector
)
.catch(this.#handleK8sError.bind(this));
}
async #deleteDaemonSets(opts: {
namespace: string;
dryRun?: boolean;
fieldSelector?: string;
labelSelector?: string;
}) {
return await this.k8sClient.apps
.deleteCollectionNamespacedDaemonSet(
opts.namespace,
undefined, // pretty
undefined, // continue
opts.dryRun ? "All" : undefined,
opts.fieldSelector,
undefined, // gracePeriodSeconds
opts.labelSelector
)
.catch(this.#handleK8sError.bind(this));
}
async #deleteCompletedRuns() {
this.logger.log("Deleting completed runs");
const start = Date.now();
const result = await this.#deletePods({
namespace: this.namespace,
fieldSelector: "status.phase=Succeeded",
labelSelector: "app=task-run",
});
const elapsedMs = Date.now() - start;
if (!result) {
this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
return;
}
const total = (result.response as any)?.body?.items?.length ?? 0;
this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
}
async #deleteFailedRuns() {
this.logger.log("Deleting failed runs");
const start = Date.now();
const result = await this.#deletePods({
namespace: this.namespace,
fieldSelector: "status.phase=Failed",
labelSelector: "app=task-run",
});
const elapsedMs = Date.now() - start;
if (!result) {
this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
return;
}
const total = (result.response as any)?.body?.items?.length ?? 0;
this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
}
async #deleteCompletedPrePulls() {
this.logger.log("Deleting completed pre-pulls");
const start = Date.now();
const result = await this.#deleteDaemonSets({
namespace: this.namespace,
labelSelector: "app=task-prepull",
});
const elapsedMs = Date.now() - start;
if (!result) {
this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
return;
}
const total = (result.response as any)?.body?.items?.length ?? 0;
this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
}
async start() {
this.enabled = true;
this.logger.log("Starting");
const completedInterval = setInterval(async () => {
if (!this.enabled) {
clearInterval(completedInterval);
return;
}
try {
await this.#deleteCompletedRuns();
} catch (error) {
this.logger.error("Error deleting completed runs", error);
}
}, this.intervalInSeconds * 1000);
const failedInterval = setInterval(
async () => {
if (!this.enabled) {
clearInterval(failedInterval);
return;
}
try {
await this.#deleteFailedRuns();
} catch (error) {
this.logger.error("Error deleting completed runs", error);
}
},
// Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
2 * this.intervalInSeconds * 1000
);
const completedPrePullInterval = setInterval(
async () => {
if (!this.enabled) {
clearInterval(completedPrePullInterval);
return;
}
try {
await this.#deleteCompletedPrePulls();
} catch (error) {
this.logger.error("Error deleting completed pre-pulls", error);
}
},
2 * this.intervalInSeconds * 1000
);
// this.#launchTests();
}
async stop() {
if (!this.enabled) {
return;
}
this.enabled = false;
this.logger.log("Shutting down..");
}
}
-385
View File
@@ -1,385 +0,0 @@
import * as k8s from "@kubernetes/client-node";
import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3";
import {
EXIT_CODE_ALREADY_HANDLED,
EXIT_CODE_CHILD_NONZERO,
SimpleLogger,
} from "@trigger.dev/core/v3/apps";
import PQueue from "p-queue";
import { setTimeout } from "timers/promises";
type FailureDetails = Prettify<{
exitCode: number;
reason: string;
logs: string;
overrideCompletion: boolean;
errorCode: TaskRunInternalError["code"];
}>;
type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
type TaskMonitorOptions = {
runtimeEnv: "local" | "kubernetes";
onIndexFailure?: IndexFailureHandler;
onRunFailure?: RunFailureHandler;
namespace?: string;
};
export class TaskMonitor {
#enabled = false;
#logger = new SimpleLogger("[TaskMonitor]");
#taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
#processedPods = new Map<string, number>();
#queue = new PQueue({ concurrency: 10 });
#k8sClient: {
core: k8s.CoreV1Api;
kubeConfig: k8s.KubeConfig;
};
private namespace = "default";
private fieldSelector = "status.phase=Failed";
private labelSelector = "app in (task-index, task-run)";
constructor(private opts: TaskMonitorOptions) {
if (opts.namespace) {
this.namespace = opts.namespace;
}
this.#k8sClient = this.#createK8sClient();
this.#taskInformer = this.#createTaskInformer();
this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
this.#taskInformer.on("error", this.#onInformerError.bind(this));
this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
}
#createTaskInformer() {
const listTasks = () =>
this.#k8sClient.core.listNamespacedPod(
this.namespace,
undefined,
undefined,
undefined,
this.fieldSelector,
this.labelSelector
);
// Uses watch with local caching
// https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
const informer = k8s.makeInformer(
this.#k8sClient.kubeConfig,
`/api/v1/namespaces/${this.namespace}/pods`,
listTasks,
this.labelSelector,
this.fieldSelector
);
return informer;
}
async #onInformerConnected() {
this.#logger.log("Connected");
}
async #onInformerError(error: any) {
this.#logger.error("Error:", error);
// Automatic reconnect
await setTimeout(2_000);
this.#taskInformer.start();
}
#enqueueOnPodUpdated(pod: k8s.V1Pod) {
this.#queue.add(async () => {
try {
// It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
await this.#onPodUpdated(pod);
} catch (error) {
this.#logger.error("Caught onPodUpdated() error:", error);
}
});
}
async #onPodUpdated(pod: k8s.V1Pod) {
this.#logger.debug(`Updated: ${pod.metadata?.name}`);
this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
// We only care about failures
if (pod.status?.phase !== "Failed") {
return;
}
const podName = pod.metadata?.name;
if (!podName) {
this.#logger.error("Pod is nameless", { pod });
return;
}
const containerStatus = pod.status.containerStatuses?.[0];
if (!containerStatus?.state) {
this.#logger.error("Pod failed, but container status doesn't have state", {
status: pod.status,
});
return;
}
if (this.#processedPods.has(podName)) {
this.#logger.debug("Pod update already processed", {
podName,
timestamp: this.#processedPods.get(podName),
});
return;
}
this.#processedPods.set(podName, Date.now());
const podStatus = this.#getPodStatusSummary(pod.status);
const containerState = this.#getContainerStateSummary(containerStatus.state);
const exitCode = containerState.exitCode ?? -1;
if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
this.#logger.debug("Ignoring pod failure, already handled by worker", {
podName,
});
return;
}
const rawLogs = await this.#getLogTail(podName);
this.#logger.log(`${podName} failed with:`, {
podStatus,
containerState,
rawLogs,
});
const rawReason = podStatus.reason ?? containerState.reason ?? "";
const message = podStatus.message ?? containerState.message ?? "";
let reason = rawReason || "Unknown error";
let logs = rawLogs || "";
/** This will only override existing task errors. It will not crash the run. */
let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO;
let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
switch (rawReason) {
case "Error":
reason = "Unknown error.";
errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
break;
case "Evicted":
if (message.startsWith("Pod ephemeral local storage usage")) {
reason = "Storage limit exceeded.";
errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED;
} else if (message) {
reason = `Evicted: ${message}`;
errorCode = TaskRunErrorCodes.POD_EVICTED;
} else {
reason = "Evicted for unknown reason.";
errorCode = TaskRunErrorCodes.POD_EVICTED;
}
if (logs.startsWith("failed to try resolving symlinks")) {
logs = "";
}
break;
case "OOMKilled":
reason =
"[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.";
errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
break;
default:
break;
}
const failureInfo = {
exitCode,
reason,
logs,
overrideCompletion: onlyOverrideExistingError,
errorCode,
} satisfies FailureDetails;
const app = pod.metadata?.labels?.app;
switch (app) {
case "task-index":
const deploymentId = pod.metadata?.labels?.deployment;
if (!deploymentId) {
this.#logger.error("Index is missing ID", { pod });
return;
}
if (this.opts.onIndexFailure) {
await this.opts.onIndexFailure(deploymentId, failureInfo);
}
break;
case "task-run":
const runId = pod.metadata?.labels?.run;
if (!runId) {
this.#logger.error("Run is missing ID", { pod });
return;
}
if (this.opts.onRunFailure) {
await this.opts.onRunFailure(runId, failureInfo);
}
break;
default:
this.#logger.error("Pod has invalid app label", { pod });
return;
}
await this.#deletePod(podName);
}
async #getLogTail(podName: string) {
try {
const logs = await this.#k8sClient.core.readNamespacedPodLog(
podName,
this.namespace,
undefined,
undefined,
undefined,
1024, // limitBytes
undefined,
undefined,
undefined,
20 // tailLines
);
const responseBody = logs.body ?? "";
if (responseBody.startsWith("unable to retrieve container logs")) {
return "";
}
// Type is wrong, body may be undefined
return responseBody;
} catch (error) {
this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
return "";
}
}
#getPodStatusSummary(status: k8s.V1PodStatus) {
return {
reason: status.reason,
message: status.message,
};
}
#getContainerStateSummary(state: k8s.V1ContainerState) {
return {
reason: state.terminated?.reason,
exitCode: state.terminated?.exitCode,
message: state.terminated?.message,
};
}
#createK8sClient() {
const kubeConfig = new k8s.KubeConfig();
if (this.opts.runtimeEnv === "local") {
kubeConfig.loadFromDefault();
} else if (this.opts.runtimeEnv === "kubernetes") {
kubeConfig.loadFromCluster();
} else {
throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
}
return {
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
kubeConfig: kubeConfig,
};
}
#isRecord(candidate: unknown): candidate is Record<string, unknown> {
if (typeof candidate !== "object" || candidate === null) {
return false;
} else {
return true;
}
}
#logK8sError(err: unknown, debugOnly = false) {
if (debugOnly) {
this.#logger.debug("K8s API Error", err);
} else {
this.#logger.error("K8s API Error", err);
}
}
#handleK8sError(err: unknown) {
if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
this.#logK8sError(err);
return;
}
this.#logK8sError(err, true);
if (typeof err.body.message === "string") {
this.#logK8sError({ message: err.body.message });
return;
}
this.#logK8sError({ body: err.body });
}
#printStats(includeMoreDetails = false) {
this.#logger.log("Stats:", {
cacheSize: this.#taskInformer.list().length,
totalProcessed: this.#processedPods.size,
...(includeMoreDetails && {
processedPods: this.#processedPods,
}),
});
}
async #deletePod(name: string) {
this.#logger.debug("Deleting pod:", name);
await this.#k8sClient.core
.deleteNamespacedPod(name, this.namespace)
.catch(this.#handleK8sError.bind(this));
}
async start() {
this.#enabled = true;
const interval = setInterval(() => {
if (!this.#enabled) {
clearInterval(interval);
return;
}
this.#printStats();
}, 300_000);
await this.#taskInformer.start();
// this.#launchTests();
}
async stop() {
if (!this.#enabled) {
return;
}
this.#enabled = false;
this.#logger.log("Shutting down..");
await this.#taskInformer.stop();
this.#printStats(true);
}
}
@@ -1,272 +0,0 @@
import * as k8s from "@kubernetes/client-node";
import { SimpleLogger } from "@trigger.dev/core/v3/apps";
type UptimeHeartbeatOptions = {
runtimeEnv: "local" | "kubernetes";
pingUrl: string;
namespace?: string;
intervalInSeconds?: number;
maxPendingRuns?: number;
maxPendingIndeces?: number;
maxPendingErrors?: number;
leadingEdge?: boolean;
};
export class UptimeHeartbeat {
private enabled = false;
private namespace: string;
private intervalInSeconds: number;
private maxPendingRuns: number;
private maxPendingIndeces: number;
private maxPendingErrors: number;
private leadingEdge = true;
private logger = new SimpleLogger("[UptimeHeartbeat]");
private k8sClient: {
core: k8s.CoreV1Api;
kubeConfig: k8s.KubeConfig;
};
constructor(private opts: UptimeHeartbeatOptions) {
this.namespace = opts.namespace ?? "default";
this.intervalInSeconds = opts.intervalInSeconds ?? 60;
this.maxPendingRuns = opts.maxPendingRuns ?? 25;
this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
this.maxPendingErrors = opts.maxPendingErrors ?? 10;
this.k8sClient = this.#createK8sClient();
}
#createK8sClient() {
const kubeConfig = new k8s.KubeConfig();
if (this.opts.runtimeEnv === "local") {
kubeConfig.loadFromDefault();
} else if (this.opts.runtimeEnv === "kubernetes") {
kubeConfig.loadFromCluster();
} else {
throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
}
return {
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
kubeConfig: kubeConfig,
};
}
#isRecord(candidate: unknown): candidate is Record<string, unknown> {
if (typeof candidate !== "object" || candidate === null) {
return false;
} else {
return true;
}
}
#logK8sError(err: unknown, debugOnly = false) {
if (debugOnly) {
this.logger.debug("K8s API Error", err);
} else {
this.logger.error("K8s API Error", err);
}
}
#handleK8sError(err: unknown) {
if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
this.#logK8sError(err);
return;
}
this.#logK8sError(err, true);
if (typeof err.body.message === "string") {
this.#logK8sError({ message: err.body.message });
return;
}
this.#logK8sError({ body: err.body });
}
async #getPods(opts: {
namespace: string;
fieldSelector?: string;
labelSelector?: string;
}): Promise<Array<k8s.V1Pod> | undefined> {
const listReturn = await this.k8sClient.core
.listNamespacedPod(
opts.namespace,
undefined, // pretty
undefined, // allowWatchBookmarks
undefined, // _continue
opts.fieldSelector,
opts.labelSelector,
this.maxPendingRuns * 2, // limit
undefined, // resourceVersion
undefined, // resourceVersionMatch
undefined, // sendInitialEvents
this.intervalInSeconds, // timeoutSeconds,
undefined // watch
)
.catch(this.#handleK8sError.bind(this));
return listReturn?.body.items;
}
async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
return await this.#getPods({
namespace: this.namespace,
fieldSelector: "status.phase=Pending",
labelSelector: "app=task-index",
});
}
async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
return await this.#getPods({
namespace: this.namespace,
fieldSelector: "status.phase=Pending",
labelSelector: "app=task-run",
});
}
#countPods(pods: Array<k8s.V1Pod>): number {
return pods.length;
}
#filterPendingPods(
pods: Array<k8s.V1Pod>,
waitingReason: "CreateContainerError" | "RunContainerError"
): Array<k8s.V1Pod> {
return pods.filter((pod) => {
const containerStatus = pod.status?.containerStatuses?.[0];
return containerStatus?.state?.waiting?.reason === waitingReason;
});
}
async #sendPing() {
this.logger.log("Sending ping");
const start = Date.now();
const controller = new AbortController();
const timeoutMs = (this.intervalInSeconds * 1000) / 2;
const fetchTimeout = setTimeout(() => {
controller.abort();
}, timeoutMs);
try {
const response = await fetch(this.opts.pingUrl, {
signal: controller.signal,
});
if (!response.ok) {
this.logger.error("Failed to send ping, response not OK", {
status: response.status,
});
return;
}
const elapsedMs = Date.now() - start;
this.logger.log("Ping sent", { elapsedMs });
} catch (error) {
if (error instanceof DOMException && error.name === "AbortError") {
this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
return;
}
this.logger.error("Failed to send ping", error);
} finally {
clearTimeout(fetchTimeout);
}
}
async #heartbeat() {
this.logger.log("Performing heartbeat");
const start = Date.now();
const pendingTasks = await this.#getPendingTasks();
if (!pendingTasks) {
this.logger.error("Failed to get pending tasks");
return;
}
const totalPendingTasks = this.#countPods(pendingTasks);
const pendingIndeces = await this.#getPendingIndeces();
if (!pendingIndeces) {
this.logger.error("Failed to get pending indeces");
return;
}
const totalPendingIndeces = this.#countPods(pendingIndeces);
const elapsedMs = Date.now() - start;
this.logger.log("Finished heartbeat checks", { elapsedMs });
if (totalPendingTasks > this.maxPendingRuns) {
this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
return;
}
if (totalPendingIndeces > this.maxPendingIndeces) {
this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
return;
}
const totalCreateContainerErrors = this.#countPods(
this.#filterPendingPods(pendingTasks, "CreateContainerError")
);
const totalRunContainerErrors = this.#countPods(
this.#filterPendingPods(pendingTasks, "RunContainerError")
);
if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
totalRunContainerErrors,
totalCreateContainerErrors,
});
return;
}
await this.#sendPing();
this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
}
async start() {
this.enabled = true;
this.logger.log("Starting");
if (this.leadingEdge) {
await this.#heartbeat();
}
const heartbeat = setInterval(async () => {
if (!this.enabled) {
clearInterval(heartbeat);
return;
}
try {
await this.#heartbeat();
} catch (error) {
this.logger.error("Error while heartbeating", error);
}
}, this.intervalInSeconds * 1000);
}
async stop() {
if (!this.enabled) {
return;
}
this.enabled = false;
this.logger.log("Shutting down..");
}
}
-17
View File
@@ -1,17 +0,0 @@
{
"compilerOptions": {
"target": "es2020",
"module": "commonjs",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"strict": true,
"skipLibCheck": true,
"paths": {
"@trigger.dev/core": ["../../packages/core/src"],
"@trigger.dev/core/*": ["../../packages/core/src/*"],
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
}
}
}
-14
View File
@@ -1,19 +1,5 @@
import type { IncomingMessage, RequestListener } from "node:http";
export const getTextBody = (req: IncomingMessage) =>
new Promise<string>((resolve) => {
let body = "";
req.on("readable", () => {
const chunk = req.read();
if (chunk) {
body += chunk;
}
});
req.on("end", () => {
resolve(body);
});
});
export async function getJsonBody(req: IncomingMessage): Promise<any> {
return new Promise((resolve, reject) => {
let body = "";
-4
View File
@@ -1,7 +1,3 @@
export * from "./backoff.js";
export * from "./logger.js";
export * from "./process.js";
export * from "./http.js";
export * from "./provider.js";
export * from "./isExecaChildProcess.js";
export * from "./exec.js";
@@ -1,6 +0,0 @@
// @ts-ignore
import type { ExecaChildProcess } from "execa";
export function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
}
-35
View File
@@ -1,35 +0,0 @@
export class SimpleLogger {
#debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? "");
constructor(private prefix?: string) {}
log<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
console.log(...this.#getPrefixedArgs(arg0, ...argN));
return arg0;
}
debug<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
if (!this.#debugEnabled) {
return arg0;
}
console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN));
return arg0;
}
error<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
console.error(...this.#getPrefixedArgs(arg0, ...argN));
return arg0;
}
#getPrefixedArgs(...args: any[]) {
if (!this.prefix) {
return args;
}
return [this.prefix, ...args];
}
}
-4
View File
@@ -1,4 +0,0 @@
/** This was used by the old build system in case of indexing failures */
export const EXIT_CODE_ALREADY_HANDLED = 111;
/** This means what it says and is only set once we have completed the attempt */
export const EXIT_CODE_CHILD_NONZERO = 112;
-382
View File
@@ -1,382 +0,0 @@
import { createServer } from "node:http";
import { getRandomPortNumber, HttpReply, getTextBody } from "./http.js";
import { SimpleLogger } from "./logger.js";
import { isExecaChildProcess } from "./isExecaChildProcess.js";
import { setTimeout } from "node:timers/promises";
import { EXIT_CODE_ALREADY_HANDLED } from "./process.js";
import type { EnvironmentType } from "../schemas/schemas.js";
import type { MachinePreset } from "../schemas/common.js";
import {
ProviderToPlatformMessages,
PlatformToProviderMessages,
ClientToSharedQueueMessages,
SharedQueueToClientMessages,
clientWebsocketMessages,
} from "../schemas/messages.js";
import { ZodMessageSender } from "../zodMessageHandler.js";
import { ZodSocketConnection } from "../zodSocket.js";
const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
export interface TaskOperationsIndexOptions {
shortCode: string;
imageRef: string;
apiKey: string;
apiUrl: string;
// identifiers
envId: string;
envType: EnvironmentType;
orgId: string;
projectId: string;
deploymentId: string;
}
export interface TaskOperationsCreateOptions {
image: string;
machine: MachinePreset;
version: string;
nextAttemptNumber?: number;
// identifiers
envId: string;
envType: EnvironmentType;
orgId: string;
projectId: string;
runId: string;
dequeuedAt?: number;
}
export interface TaskOperationsRestoreOptions {
imageRef: string;
checkpointRef: string;
machine: MachinePreset;
attemptNumber?: number;
// identifiers
envId: string;
envType: EnvironmentType;
orgId: string;
projectId: string;
runId: string;
checkpointId: string;
}
export interface TaskOperationsPrePullDeploymentOptions {
shortCode: string;
imageRef: string;
// identifiers
envId: string;
envType: EnvironmentType;
orgId: string;
projectId: string;
deploymentId: string;
}
export interface TaskOperations {
init: () => Promise<any>;
// CRUD
index: (opts: TaskOperationsIndexOptions) => Promise<any>;
create: (opts: TaskOperationsCreateOptions) => Promise<any>;
restore: (opts: TaskOperationsRestoreOptions) => Promise<any>;
// unimplemented
delete?: (...args: any[]) => Promise<any>;
get?: (...args: any[]) => Promise<any>;
prePullDeployment?: (opts: TaskOperationsPrePullDeploymentOptions) => Promise<any>;
}
type ProviderShellOptions = {
tasks: TaskOperations;
type: "docker" | "kubernetes";
host?: string;
port?: number;
};
interface Provider {
tasks: TaskOperations;
}
export class ProviderShell implements Provider {
tasks: TaskOperations;
#httpPort: number;
#httpServer: ReturnType<typeof createServer>;
platformSocket: ZodSocketConnection<
typeof ProviderToPlatformMessages,
typeof PlatformToProviderMessages
>;
constructor(private options: ProviderShellOptions) {
this.tasks = options.tasks;
this.#httpPort = options.port ?? HTTP_SERVER_PORT;
this.#httpServer = this.#createHttpServer();
this.platformSocket = this.#createPlatformSocket();
this.#createSharedQueueSocket();
}
#createSharedQueueSocket() {
const sharedQueueConnection = new ZodSocketConnection({
namespace: "shared-queue",
host: PLATFORM_HOST,
port: Number(PLATFORM_WS_PORT),
secure: SECURE_CONNECTION,
clientMessages: ClientToSharedQueueMessages,
serverMessages: SharedQueueToClientMessages,
authToken: PLATFORM_SECRET,
handlers: {
SERVER_READY: async (message) => {
// TODO: create new schema without worker requirement
await sender.send("READY_FOR_TASKS", {
backgroundWorkerId: "placeholder",
});
},
BACKGROUND_WORKER_MESSAGE: async (message) => {
if (message.data.type === "SCHEDULE_ATTEMPT") {
try {
await this.tasks.create({
image: message.data.image,
machine: message.data.machine,
version: message.data.version,
nextAttemptNumber: message.data.nextAttemptNumber,
// identifiers
envId: message.data.envId,
envType: message.data.envType,
orgId: message.data.orgId,
projectId: message.data.projectId,
runId: message.data.runId,
dequeuedAt: message.data.dequeuedAt,
});
} catch (error) {
logger.error("create failed", error);
}
}
},
},
});
const sender = new ZodMessageSender({
schema: clientWebsocketMessages,
sender: async (message) => {
return new Promise((resolve, reject) => {
try {
const { type, ...payload } = message;
sharedQueueConnection.socket.emit(type, payload as any);
resolve();
} catch (err) {
reject(err);
}
});
},
});
return sharedQueueConnection;
}
#createPlatformSocket() {
const platformConnection = new ZodSocketConnection({
namespace: "provider",
host: PLATFORM_HOST,
port: Number(PLATFORM_WS_PORT),
secure: SECURE_CONNECTION,
clientMessages: ProviderToPlatformMessages,
serverMessages: PlatformToProviderMessages,
authToken: PLATFORM_SECRET,
extraHeaders: {
"x-trigger-provider-type": this.options.type,
},
handlers: {
INDEX: async (message) => {
try {
await this.tasks.index({
shortCode: message.shortCode,
imageRef: message.imageTag,
apiKey: message.apiKey,
apiUrl: message.apiUrl,
// identifiers
envId: message.envId,
envType: message.envType,
orgId: message.orgId,
projectId: message.projectId,
deploymentId: message.deploymentId,
});
} catch (error) {
if (isExecaChildProcess(error)) {
logger.error("Index failed", {
socketMessage: message,
exitCode: error.exitCode,
escapedCommand: error.escapedCommand,
stdout: error.stdout,
stderr: error.stderr,
});
if (error.exitCode === EXIT_CODE_ALREADY_HANDLED) {
logger.error("Index failure already reported by the worker", {
socketMessage: message,
});
// Add a brief delay to avoid messaging race conditions
await setTimeout(2000);
}
function normalizeStderr(stderr: string) {
return stderr
.split("\n")
.map((line) => line.trim())
.filter((line) => line.length > 0)
.join("\n");
}
return {
success: false,
error: {
name: "Index error",
message: `Crashed with exit code ${error.exitCode}`,
stderr: normalizeStderr(error.stderr),
},
};
} else {
logger.error("Index failed", error);
}
if (error instanceof Error) {
return {
success: false,
error: {
name: "Provider error",
message: error.message,
stack: error.stack,
},
};
} else {
return {
success: false,
error: {
name: "Provider error",
message: "Unknown error",
},
};
}
}
return {
success: true,
};
},
RESTORE: async (message) => {
if (message.type.toLowerCase() !== this.options.type.toLowerCase()) {
logger.error(
`restore failed: ${this.options.type} provider can't restore ${message.type} checkpoints`
);
return;
}
try {
await this.tasks.restore({
checkpointRef: message.location,
machine: message.machine,
imageRef: message.imageRef,
attemptNumber: message.attemptNumber,
// identifiers
envId: message.envId,
envType: message.envType,
orgId: message.orgId,
projectId: message.projectId,
runId: message.runId,
checkpointId: message.checkpointId,
});
} catch (error) {
logger.error("restore failed", error);
}
},
PRE_PULL_DEPLOYMENT: async (message) => {
if (!this.tasks.prePullDeployment) {
logger.debug("prePullDeployment not implemented", message);
return;
}
try {
await this.tasks.prePullDeployment({
shortCode: message.shortCode,
imageRef: message.imageRef,
// identifiers
envId: message.envId,
envType: message.envType,
orgId: message.orgId,
projectId: message.projectId,
deploymentId: message.deploymentId,
});
} catch (error) {
logger.error("prePullDeployment failed", error);
}
},
},
});
return platformConnection;
}
#createHttpServer() {
const httpServer = createServer(async (req, res) => {
logger.log(`[${req.method}]`, req.url);
const reply = new HttpReply(res);
try {
const url = new URL(req.url ?? "", `http://${req.headers.host}`);
switch (url.pathname) {
case "/health": {
return reply.text("ok");
}
case "/whoami": {
return reply.text(`${MACHINE_NAME}`);
}
case "/close": {
this.platformSocket.close();
return reply.text("platform socket closed");
}
case "/delete": {
const body = await getTextBody(req);
if (this.tasks.delete) {
await this.tasks.delete({ runId: body });
return reply.text(`sent delete request: ${body}`);
} else {
return reply.text("delete not implemented", 501);
}
}
default: {
return reply.empty(404);
}
}
} catch (error) {
logger.error("HTTP server error", { error });
reply.empty(500);
return;
}
});
httpServer.on("clientError", (err, socket) => {
socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
});
httpServer.on("listening", () => {
logger.log("server listening on port", this.#httpPort);
});
return httpServer;
}
async listen() {
this.#httpServer.listen(this.#httpPort, this.options.host ?? "0.0.0.0");
await this.tasks.init();
}
}
@@ -1,65 +0,0 @@
import { randomUUID } from "node:crypto";
import { isExecaChildProcess } from "../apps/isExecaChildProcess.js";
export type CheckpointTestResult =
| {
ok: true;
}
| {
ok: false;
message: string;
error?: unknown;
};
export async function testDockerCheckpoint(): Promise<CheckpointTestResult> {
const { $ } = await import("execa");
try {
// Create a dummy container
const container =
await $`docker run -d --rm --name init-dummy-${randomUUID()} docker.io/library/busybox sleep 10`;
// Checkpoint it
await $`docker checkpoint create ${container} init-check`;
} catch (error) {
if (!isExecaChildProcess(error)) {
return {
ok: false,
message: "No checkpoint support: Unknown error.",
error,
};
}
if (error.stderr.includes("criu")) {
if (error.stderr.includes("executable file not found")) {
return {
ok: false,
message: "No checkpoint support: Missing CRIU binary.",
};
}
return {
ok: false,
message: "No checkpoint support: Unknown CRIU error.",
error,
};
}
if (error.stderr.includes("experimental features enabled")) {
return {
ok: false,
message: "No checkpoint support: Please enable docker experimental features.",
};
}
return {
ok: false,
message: "No checkpoint support: Unknown execa error.",
error,
};
}
return {
ok: true,
};
}
-1
View File
@@ -1,5 +1,4 @@
export * from "./checkpointClient.js";
export * from "./checkpointTest.js";
export * from "./httpServer.js";
export * from "./singleton.js";
export * from "./shutdownManager.js";
+1 -736
View File
File diff suppressed because it is too large Load Diff