chore: remove end-of-life v3 execution components (#4194)
v3 (engine V1) is end-of-lifed and the v3 clusters are gone, so this removes the dead v3 execution code from the monorepo. It's the first pass of TRI-11824 - the webapp v3 code paths are deliberately left untouched and gated for a follow-up. ## Apps Deletes the three v3-only execution apps and their build wiring: - `apps/coordinator`, `apps/kubernetes-provider`, `apps/docker-provider` - `.github/workflows/publish-worker.yml` - it built only those three; the v4 worker publish is a separate workflow - Their references in `.changeset/config.json`, `.cursorignore`, `CHANGESETS.md`, `CONTRIBUTING.md`, `.server-changes/README.md` - `pnpm-lock.yaml` regenerated to prune the apps and their app-only dependencies (`socket.io`, `@kubernetes/client-node`, `p-queue`, `execa`, `prom-client`, `tinyexec`) ## Core Removes the helpers in `@trigger.dev/core` that only those apps used - `ProviderShell`, `SimpleLogger`, the `Exec`/process helpers, `isExecaChildProcess`, `getTextBody`, and `testDockerCheckpoint`. Each was verified to have no remaining consumers anywhere in the repo. Kept the helpers still used elsewhere: `ExponentialBackoff` (warm-start client), `HttpReply`/`getJsonBody` (serverOnly http server), `SimpleStructuredLogger` (widely used), and `ZodNamespace`/`ZodSocketConnection` (still referenced by legacy v3 webapp code, hence the follow-up pass). The `./v3/apps` and `./v3/serverOnly` export subpaths remain - only dead members were trimmed from their barrels, so no `package.json` exports changed. ## Verification `@trigger.dev/core` builds, and `typecheck` passes for core, supervisor, cli-v3, run-engine, redis-worker, and webapp. refs TRI-11824
This commit is contained in:
@@ -14,9 +14,6 @@
|
||||
"updateInternalDependencies": "patch",
|
||||
"ignore": [
|
||||
"webapp",
|
||||
"coordinator",
|
||||
"docker-provider",
|
||||
"kubernetes-provider",
|
||||
"supervisor"
|
||||
],
|
||||
"___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@trigger.dev/core": patch
|
||||
---
|
||||
|
||||
Removed internal helpers that were only used by the end-of-life v3 self-hosted compute providers.
|
||||
@@ -5,7 +5,7 @@ paths:
|
||||
|
||||
# Server App Changes
|
||||
|
||||
When modifying server apps (webapp, supervisor, coordinator, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset:
|
||||
When modifying server apps (webapp, supervisor, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset:
|
||||
|
||||
```bash
|
||||
cat > .server-changes/descriptive-name.md << 'EOF'
|
||||
@@ -18,6 +18,6 @@ Brief description of what changed and why.
|
||||
EOF
|
||||
```
|
||||
|
||||
- **area**: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
|
||||
- **area**: `webapp` | `supervisor`
|
||||
- **type**: `feature` | `fix` | `improvement` | `breaking`
|
||||
- If the PR also touches `packages/`, just the changeset is sufficient (no `.server-changes/` needed).
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
apps/docker-provider/
|
||||
apps/kubernetes-provider/
|
||||
apps/proxy/
|
||||
apps/coordinator/
|
||||
packages/rsc/
|
||||
.changeset
|
||||
.zed
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
name: "⚒️ Publish Worker"
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
image_tag:
|
||||
description: The image tag to publish
|
||||
type: string
|
||||
required: false
|
||||
default: ""
|
||||
image_registry:
|
||||
description: The registry namespace to publish under (e.g. ghcr.io/<owner>)
|
||||
type: string
|
||||
required: false
|
||||
default: ""
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME:
|
||||
required: false
|
||||
DOCKERHUB_TOKEN:
|
||||
required: false
|
||||
push:
|
||||
tags:
|
||||
- "infra-dev-*"
|
||||
- "infra-test-*"
|
||||
- "infra-prod-*"
|
||||
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
package: [coordinator, docker-provider, kubernetes-provider]
|
||||
runs-on: warp-ubuntu-latest-x64-8x
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
steps:
|
||||
- name: ⬇️ Checkout git repo
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: 📦 Get image repo
|
||||
id: get_repository
|
||||
env:
|
||||
PACKAGE: ${{ matrix.package }}
|
||||
run: |
|
||||
if [[ "$PACKAGE" == *-provider ]]; then
|
||||
repo="provider/${PACKAGE%-provider}"
|
||||
else
|
||||
repo="$PACKAGE"
|
||||
fi
|
||||
echo "repo=${repo}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- id: get_tag
|
||||
uses: ./.github/actions/get-image-tag
|
||||
with:
|
||||
tag: ${{ inputs.image_tag }}
|
||||
|
||||
- name: 🐋 Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
|
||||
# ..to avoid rate limits when pulling images
|
||||
- name: 🐳 Login to DockerHub
|
||||
if: ${{ env.DOCKERHUB_USERNAME }}
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: 🚢 Build Container Image
|
||||
run: |
|
||||
docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
|
||||
|
||||
# ..to push image
|
||||
- name: 🐙 Login to GitHub Container Registry
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: 🐙 Push to GitHub Container Registry
|
||||
run: |
|
||||
docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
|
||||
docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
|
||||
env:
|
||||
# Resolved by the caller when invoked from publish.yml; falls back to the
|
||||
# IMAGE_REGISTRY repository variable (or ghcr.io/<owner>) for the direct
|
||||
# push triggers above, so a fork publishes to its own namespace.
|
||||
REGISTRY: ${{ inputs.image_registry || vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }}
|
||||
REPOSITORY: ${{ steps.get_repository.outputs.repo }}
|
||||
IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
|
||||
|
||||
# - name: 🐙 Push 'v3' tag to GitHub Container Registry
|
||||
# if: steps.get_tag.outputs.is_semver == 'true'
|
||||
# run: |
|
||||
# docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
|
||||
# docker push "$REGISTRY/$REPOSITORY:v3"
|
||||
# env:
|
||||
# REGISTRY: ghcr.io/triggerdotdev
|
||||
# REPOSITORY: ${{ steps.get_repository.outputs.repo }}
|
||||
@@ -30,7 +30,6 @@ on:
|
||||
- ".github/workflows/unit-tests.yml"
|
||||
- ".github/workflows/e2e.yml"
|
||||
- ".github/workflows/publish-webapp.yml"
|
||||
- ".github/workflows/publish-worker.yml"
|
||||
- "packages/**"
|
||||
- "!packages/**/*.md"
|
||||
- "!packages/**/*.eslintrc"
|
||||
@@ -80,19 +79,6 @@ jobs:
|
||||
# to its own namespace; set the IMAGE_REGISTRY repository variable to override.
|
||||
image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }}
|
||||
|
||||
publish-worker:
|
||||
needs: [typecheck]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
uses: ./.github/workflows/publish-worker.yml
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
with:
|
||||
image_tag: ${{ inputs.image_tag }}
|
||||
image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }}
|
||||
|
||||
publish-worker-v4:
|
||||
needs: [typecheck]
|
||||
permissions:
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Server Changes
|
||||
|
||||
This directory tracks changes to server-only components (webapp, supervisor, coordinator, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented.
|
||||
This directory tracks changes to server-only components (webapp, supervisor, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented.
|
||||
|
||||
## When to add a file
|
||||
|
||||
**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file.
|
||||
**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file.
|
||||
|
||||
**Mixed PRs** (both packages and server): Just add a changeset as usual. No `.server-changes/` file needed — the changeset covers it.
|
||||
|
||||
@@ -31,7 +31,7 @@ Speed up batch queue processing by removing stalls and fixing retry race
|
||||
|
||||
### Fields
|
||||
|
||||
- **area** (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
|
||||
- **area** (required): `webapp` | `supervisor`
|
||||
- **type** (required): `feature` | `fix` | `improvement` | `breaking`
|
||||
|
||||
### Description
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ Speed up batch queue processing by removing stalls and fixing retry race
|
||||
EOF
|
||||
```
|
||||
|
||||
- `area`: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
|
||||
- `area`: `webapp` | `supervisor`
|
||||
- `type`: `feature` | `fix` | `improvement` | `breaking`
|
||||
|
||||
For **mixed PRs** (both packages and server): just add a changeset. No `.server-changes/` file needed.
|
||||
|
||||
+2
-2
@@ -265,7 +265,7 @@ Most of the time the changes you'll make are likely to be categorized as patch r
|
||||
|
||||
## Adding server changes
|
||||
|
||||
Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes.
|
||||
Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes.
|
||||
|
||||
Create a markdown file with a descriptive name:
|
||||
|
||||
@@ -281,7 +281,7 @@ EOF
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `area` (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
|
||||
- `area` (required): `webapp` | `supervisor`
|
||||
- `type` (required): `feature` | `fix` | `improvement` | `breaking`
|
||||
|
||||
The body text (below the frontmatter) is a one-line description of the change. Keep it concise — it will appear in release notes.
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
HTTP_SERVER_PORT=8020
|
||||
PLATFORM_ENABLED=true
|
||||
PLATFORM_WS_PORT=3030
|
||||
SECURE_CONNECTION=false
|
||||
@@ -1,3 +0,0 @@
|
||||
dist/
|
||||
node_modules/
|
||||
.env
|
||||
@@ -1,60 +0,0 @@
|
||||
# syntax=docker/dockerfile:labs
|
||||
|
||||
FROM node:22.23.1-bookworm-slim@sha256:813a7480f28fdadac1f7f5c824bcdad435b5bc1322a5968bbbdef8d058f9dff4 AS node-22
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
FROM node-22 AS pruner
|
||||
|
||||
COPY --chown=node:node . .
|
||||
RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
|
||||
RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
|
||||
|
||||
FROM node-22 AS base
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y buildah ca-certificates dumb-init docker.io busybox \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --chown=node:node .gitignore .gitignore
|
||||
COPY --from=pruner --chown=node:node /app/out/json/ .
|
||||
COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
|
||||
COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
|
||||
|
||||
FROM base AS dev-deps
|
||||
RUN corepack enable
|
||||
ENV NODE_ENV development
|
||||
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
|
||||
|
||||
FROM base AS builder
|
||||
RUN corepack enable
|
||||
|
||||
COPY --from=pruner --chown=node:node /app/out/full/ .
|
||||
COPY --from=dev-deps --chown=node:node /app/ .
|
||||
COPY --chown=node:node turbo.json turbo.json
|
||||
|
||||
RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter coordinator build:bundle
|
||||
|
||||
FROM alpine AS cri-tools
|
||||
|
||||
WORKDIR /cri-tools
|
||||
|
||||
ARG CRICTL_VERSION=v1.29.0
|
||||
ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
|
||||
ADD --checksum=${CRICTL_CHECKSUM} \
|
||||
https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
|
||||
RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
|
||||
|
||||
FROM base AS runner
|
||||
|
||||
RUN corepack enable
|
||||
ENV NODE_ENV production
|
||||
|
||||
COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
|
||||
COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]
|
||||
@@ -1,3 +0,0 @@
|
||||
# Coordinator
|
||||
|
||||
Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"name": "coordinator",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"main": "dist/index.cjs",
|
||||
"scripts": {
|
||||
"build": "npm run build:bundle",
|
||||
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
|
||||
"build:image": "docker build -f Containerfile . -t coordinator",
|
||||
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
|
||||
"start": "tsx src/index.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"nanoid": "^5.0.6",
|
||||
"prom-client": "^15.1.0",
|
||||
"socket.io": "4.7.4",
|
||||
"tinyexec": "^0.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dotenv": "^16.4.2",
|
||||
"esbuild": "^0.19.11",
|
||||
"tsx": "^4.7.0"
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
import { setTimeout as timeout } from "node:timers/promises";
|
||||
|
||||
class ChaosMonkeyError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ChaosMonkeyError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ChaosMonkey {
|
||||
private chaosEventRate = 0.2;
|
||||
private delayInSeconds = 45;
|
||||
|
||||
constructor(
|
||||
private enabled = false,
|
||||
private disableErrors = false,
|
||||
private disableDelays = false
|
||||
) {
|
||||
if (this.enabled) {
|
||||
console.log("🍌 Chaos monkey enabled");
|
||||
}
|
||||
}
|
||||
|
||||
static Error = ChaosMonkeyError;
|
||||
|
||||
enable() {
|
||||
this.enabled = true;
|
||||
console.log("🍌 Chaos monkey enabled");
|
||||
}
|
||||
|
||||
disable() {
|
||||
this.enabled = false;
|
||||
console.log("🍌 Chaos monkey disabled");
|
||||
}
|
||||
|
||||
async call({
|
||||
throwErrors = !this.disableErrors,
|
||||
addDelays = !this.disableDelays,
|
||||
}: {
|
||||
throwErrors?: boolean;
|
||||
addDelays?: boolean;
|
||||
} = {}) {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const random = Math.random();
|
||||
|
||||
if (random > this.chaosEventRate) {
|
||||
// Don't interfere with normal operation
|
||||
return;
|
||||
}
|
||||
|
||||
const chaosEvents: Array<() => Promise<any>> = [];
|
||||
|
||||
if (addDelays) {
|
||||
chaosEvents.push(async () => {
|
||||
console.log("🍌 Chaos monkey: Add delay");
|
||||
|
||||
await timeout(this.delayInSeconds * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
if (throwErrors) {
|
||||
chaosEvents.push(async () => {
|
||||
console.log("🍌 Chaos monkey: Throw error");
|
||||
|
||||
throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
|
||||
});
|
||||
}
|
||||
|
||||
if (chaosEvents.length === 0) {
|
||||
console.error("🍌 Chaos monkey: No events selected");
|
||||
return;
|
||||
}
|
||||
|
||||
const randomIndex = Math.floor(Math.random() * chaosEvents.length);
|
||||
|
||||
const chaosEvent = chaosEvents[randomIndex];
|
||||
|
||||
if (!chaosEvent) {
|
||||
console.error("🍌 Chaos monkey: No event found");
|
||||
return;
|
||||
}
|
||||
|
||||
await chaosEvent();
|
||||
}
|
||||
}
|
||||
@@ -1,709 +0,0 @@
|
||||
import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
|
||||
import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
|
||||
import { nanoid } from "nanoid";
|
||||
import fs from "node:fs/promises";
|
||||
import { ChaosMonkey } from "./chaosMonkey";
|
||||
import { Buildah, Crictl, Exec } from "./exec";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { TempFileCleaner } from "./cleaner";
|
||||
import { numFromEnv, boolFromEnv } from "./util";
|
||||
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
|
||||
|
||||
type CheckpointerInitializeReturn = {
|
||||
canCheckpoint: boolean;
|
||||
willSimulate: boolean;
|
||||
};
|
||||
|
||||
type CheckpointAndPushOptions = {
|
||||
runId: string;
|
||||
leaveRunning?: boolean;
|
||||
projectRef: string;
|
||||
deploymentVersion: string;
|
||||
shouldHeartbeat?: boolean;
|
||||
attemptNumber?: number;
|
||||
};
|
||||
|
||||
type CheckpointAndPushResult =
|
||||
| { success: true; checkpoint: CheckpointData }
|
||||
| {
|
||||
success: false;
|
||||
reason?: "CANCELED" | "ERROR" | "SKIP_RETRYING";
|
||||
};
|
||||
|
||||
type CheckpointData = {
|
||||
location: string;
|
||||
docker: boolean;
|
||||
};
|
||||
|
||||
type CheckpointerOptions = {
|
||||
dockerMode: boolean;
|
||||
forceSimulate: boolean;
|
||||
heartbeat: (runId: string) => void;
|
||||
registryHost?: string;
|
||||
registryNamespace?: string;
|
||||
registryTlsVerify?: boolean;
|
||||
disableCheckpointSupport?: boolean;
|
||||
checkpointPath?: string;
|
||||
simulateCheckpointFailure?: boolean;
|
||||
simulateCheckpointFailureSeconds?: number;
|
||||
simulatePushFailure?: boolean;
|
||||
simulatePushFailureSeconds?: number;
|
||||
chaosMonkey?: ChaosMonkey;
|
||||
};
|
||||
|
||||
async function getFileSize(filePath: string): Promise<number> {
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
return stats.size;
|
||||
} catch (error) {
|
||||
console.error("Error getting file size:", error);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
async function getParsedFileSize(filePath: string) {
|
||||
const sizeInBytes = await getFileSize(filePath);
|
||||
|
||||
let message = `Size in bytes: ${sizeInBytes}`;
|
||||
|
||||
if (sizeInBytes > 1024 * 1024) {
|
||||
const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
|
||||
message = `Size in MB (rounded): ${sizeInMB}`;
|
||||
} else if (sizeInBytes > 1024) {
|
||||
const sizeInKB = (sizeInBytes / 1024).toFixed(2);
|
||||
message = `Size in KB (rounded): ${sizeInKB}`;
|
||||
}
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
sizeInBytes,
|
||||
message,
|
||||
};
|
||||
}
|
||||
|
||||
export class Checkpointer {
|
||||
#initialized = false;
|
||||
#canCheckpoint = false;
|
||||
#dockerMode: boolean;
|
||||
|
||||
#logger = new SimpleStructuredLogger("checkpointer");
|
||||
|
||||
#failedCheckpoints = new Map<string, unknown>();
|
||||
|
||||
// Indexed by run ID
|
||||
#runAbortControllers = new Map<
|
||||
string,
|
||||
{ signal: AbortSignal; abort: AbortController["abort"] }
|
||||
>();
|
||||
|
||||
private registryHost: string;
|
||||
private registryNamespace: string;
|
||||
private registryTlsVerify: boolean;
|
||||
|
||||
private disableCheckpointSupport: boolean;
|
||||
|
||||
private simulateCheckpointFailure: boolean;
|
||||
private simulateCheckpointFailureSeconds: number;
|
||||
private simulatePushFailure: boolean;
|
||||
private simulatePushFailureSeconds: number;
|
||||
|
||||
private chaosMonkey: ChaosMonkey;
|
||||
private tmpCleaner?: TempFileCleaner;
|
||||
|
||||
constructor(private opts: CheckpointerOptions) {
|
||||
this.#dockerMode = opts.dockerMode;
|
||||
|
||||
this.registryHost = opts.registryHost ?? "localhost:5000";
|
||||
this.registryNamespace = opts.registryNamespace ?? "trigger";
|
||||
this.registryTlsVerify = opts.registryTlsVerify ?? true;
|
||||
|
||||
this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
|
||||
|
||||
this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
|
||||
this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
|
||||
this.simulatePushFailure = opts.simulatePushFailure ?? false;
|
||||
this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
|
||||
|
||||
this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
|
||||
this.tmpCleaner = this.#createTmpCleaner();
|
||||
}
|
||||
|
||||
async init(): Promise<CheckpointerInitializeReturn> {
|
||||
if (this.#initialized) {
|
||||
return this.#getInitReturn(this.#canCheckpoint);
|
||||
}
|
||||
|
||||
this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
|
||||
|
||||
if (this.#dockerMode) {
|
||||
const testCheckpoint = await testDockerCheckpoint();
|
||||
|
||||
if (testCheckpoint.ok) {
|
||||
return this.#getInitReturn(true);
|
||||
}
|
||||
|
||||
this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error });
|
||||
return this.#getInitReturn(false);
|
||||
}
|
||||
|
||||
const canLogin = await Buildah.canLogin(this.registryHost);
|
||||
|
||||
if (!canLogin) {
|
||||
this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
|
||||
}
|
||||
|
||||
return this.#getInitReturn(canLogin);
|
||||
}
|
||||
|
||||
#getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
|
||||
this.#canCheckpoint = canCheckpoint;
|
||||
|
||||
if (canCheckpoint) {
|
||||
if (!this.#initialized) {
|
||||
this.#logger.log("Full checkpoint support!");
|
||||
}
|
||||
}
|
||||
|
||||
this.#initialized = true;
|
||||
|
||||
const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
|
||||
|
||||
if (willSimulate) {
|
||||
this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
|
||||
forceSimulate: this.opts.forceSimulate,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
canCheckpoint,
|
||||
willSimulate,
|
||||
};
|
||||
}
|
||||
|
||||
#getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
|
||||
return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
|
||||
}
|
||||
|
||||
#getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
|
||||
const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
|
||||
|
||||
if (this.#dockerMode) {
|
||||
return basename;
|
||||
} else {
|
||||
return Crictl.getExportLocation(basename);
|
||||
}
|
||||
}
|
||||
|
||||
async checkpointAndPush(
|
||||
opts: CheckpointAndPushOptions,
|
||||
delayMs?: number
|
||||
): Promise<CheckpointData | undefined> {
|
||||
const start = performance.now();
|
||||
this.#logger.log(`checkpointAndPush() start`, { start, opts });
|
||||
|
||||
const { runId } = opts;
|
||||
|
||||
let interval: NodeJS.Timer | undefined;
|
||||
if (opts.shouldHeartbeat) {
|
||||
interval = setInterval(() => {
|
||||
this.#logger.log("Sending heartbeat", { runId });
|
||||
this.opts.heartbeat(runId);
|
||||
}, 20_000);
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
const abort = controller.abort.bind(controller);
|
||||
|
||||
const onAbort = () => {
|
||||
this.#logger.error("Checkpoint aborted", { runId, options: opts });
|
||||
};
|
||||
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
|
||||
const removeCurrentAbortController = () => {
|
||||
const controller = this.#runAbortControllers.get(runId);
|
||||
|
||||
// Ensure only the current controller is removed
|
||||
if (controller && controller.signal === signal) {
|
||||
this.#runAbortControllers.delete(runId);
|
||||
}
|
||||
|
||||
// Remove the abort listener in case it hasn't fired
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
};
|
||||
|
||||
if (!this.#dockerMode && !this.#canCheckpoint) {
|
||||
this.#logger.error("No checkpoint support. Simulation requires docker.");
|
||||
this.#failCheckpoint(runId, "NO_SUPPORT");
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.#isRunCheckpointing(runId)) {
|
||||
this.#logger.error("Checkpoint procedure already in progress", { options: opts });
|
||||
this.#failCheckpoint(runId, "IN_PROGRESS");
|
||||
return;
|
||||
}
|
||||
|
||||
// This is a new checkpoint, clear any last failure for this run
|
||||
this.#clearFailedCheckpoint(runId);
|
||||
|
||||
if (this.disableCheckpointSupport) {
|
||||
this.#logger.error("Checkpoint support disabled", { options: opts });
|
||||
this.#failCheckpoint(runId, "DISABLED");
|
||||
return;
|
||||
}
|
||||
|
||||
this.#runAbortControllers.set(runId, { signal, abort });
|
||||
|
||||
try {
|
||||
const result = await this.#checkpointAndPushWithBackoff(opts, { delayMs, signal });
|
||||
|
||||
const end = performance.now();
|
||||
this.#logger.log(`checkpointAndPush() end`, {
|
||||
start,
|
||||
end,
|
||||
diff: end - start,
|
||||
diffWithoutDelay: end - start - (delayMs ?? 0),
|
||||
opts,
|
||||
success: result.success,
|
||||
delayMs,
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
return;
|
||||
}
|
||||
|
||||
return result.checkpoint;
|
||||
} finally {
|
||||
if (opts.shouldHeartbeat) {
|
||||
// @ts-ignore - Some kind of node incompatible type issue
|
||||
clearInterval(interval);
|
||||
}
|
||||
removeCurrentAbortController();
|
||||
}
|
||||
}
|
||||
|
||||
#isRunCheckpointing(runId: string) {
|
||||
return this.#runAbortControllers.has(runId);
|
||||
}
|
||||
|
||||
cancelAllCheckpointsForRun(runId: string): boolean {
|
||||
this.#logger.log("cancelAllCheckpointsForRun: call", { runId });
|
||||
|
||||
// If the last checkpoint failed, pretend we canceled it
|
||||
// This ensures tasks don't wait for external resume messages to continue
|
||||
if (this.#hasFailedCheckpoint(runId)) {
|
||||
this.#logger.log("cancelAllCheckpointsForRun: hasFailedCheckpoint", { runId });
|
||||
this.#clearFailedCheckpoint(runId);
|
||||
return true;
|
||||
}
|
||||
|
||||
const controller = this.#runAbortControllers.get(runId);
|
||||
|
||||
if (!controller) {
|
||||
this.#logger.debug("cancelAllCheckpointsForRun: no abort controller", { runId });
|
||||
return false;
|
||||
}
|
||||
|
||||
const { abort, signal } = controller;
|
||||
|
||||
if (signal.aborted) {
|
||||
this.#logger.debug("cancelAllCheckpointsForRun: signal already aborted", { runId });
|
||||
return false;
|
||||
}
|
||||
|
||||
abort("cancelCheckpoint()");
|
||||
this.#runAbortControllers.delete(runId);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async #checkpointAndPushWithBackoff(
|
||||
{
|
||||
runId,
|
||||
leaveRunning = true, // This mirrors kubernetes behaviour more accurately
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
attemptNumber,
|
||||
}: CheckpointAndPushOptions,
|
||||
{ delayMs, signal }: { delayMs?: number; signal: AbortSignal }
|
||||
): Promise<CheckpointAndPushResult> {
|
||||
if (delayMs && delayMs > 0) {
|
||||
this.#logger.log("Delaying checkpoint", { runId, delayMs });
|
||||
|
||||
try {
|
||||
await setTimeout(delayMs, undefined, { signal });
|
||||
} catch (_error) {
|
||||
this.#logger.log("Checkpoint canceled during initial delay", { runId });
|
||||
return { success: false, reason: "CANCELED" };
|
||||
}
|
||||
}
|
||||
|
||||
this.#logger.log("Checkpointing with backoff", {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
});
|
||||
|
||||
const backoff = new ExponentialBackoff()
|
||||
.type("EqualJitter")
|
||||
.base(3)
|
||||
.max(3 * 3600)
|
||||
.maxElapsed(48 * 3600);
|
||||
|
||||
for await (const { delay, retry } of backoff) {
|
||||
try {
|
||||
if (retry > 0) {
|
||||
this.#logger.error("Retrying checkpoint", {
|
||||
runId,
|
||||
retry,
|
||||
delay,
|
||||
});
|
||||
|
||||
try {
|
||||
await setTimeout(delay.milliseconds, undefined, { signal });
|
||||
} catch (_error) {
|
||||
this.#logger.log("Checkpoint canceled during retry delay", { runId });
|
||||
return { success: false, reason: "CANCELED" };
|
||||
}
|
||||
}
|
||||
|
||||
const result = await this.#checkpointAndPush(
|
||||
{
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
attemptNumber,
|
||||
},
|
||||
{ signal }
|
||||
);
|
||||
|
||||
if (result.success) {
|
||||
return result;
|
||||
}
|
||||
|
||||
if (result.reason === "CANCELED") {
|
||||
this.#logger.log("Checkpoint canceled, won't retry", { runId });
|
||||
// Don't fail the checkpoint, as it was canceled
|
||||
return result;
|
||||
}
|
||||
|
||||
if (result.reason === "SKIP_RETRYING") {
|
||||
this.#logger.log("Skipping retrying", { runId });
|
||||
return result;
|
||||
}
|
||||
|
||||
continue;
|
||||
} catch (error) {
|
||||
this.#logger.error("Checkpoint error", {
|
||||
retry,
|
||||
runId,
|
||||
delay,
|
||||
error: error instanceof Error ? error.message : error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.#logger.error(`Checkpoint failed after exponential backoff`, {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
});
|
||||
this.#failCheckpoint(runId, "ERROR");
|
||||
|
||||
return { success: false, reason: "ERROR" };
|
||||
}
|
||||
|
||||
async #checkpointAndPush(
|
||||
{
|
||||
runId,
|
||||
leaveRunning = true, // This mirrors kubernetes behaviour more accurately
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
attemptNumber,
|
||||
}: CheckpointAndPushOptions,
|
||||
{ signal }: { signal: AbortSignal }
|
||||
): Promise<CheckpointAndPushResult> {
|
||||
await this.init();
|
||||
|
||||
const options = {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
attemptNumber,
|
||||
};
|
||||
|
||||
const shortCode = nanoid(8);
|
||||
const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
|
||||
const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
|
||||
|
||||
const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: signal });
|
||||
const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: signal });
|
||||
|
||||
const cleanup = async () => {
|
||||
const metadata = {
|
||||
runId,
|
||||
exportLocation,
|
||||
imageRef,
|
||||
};
|
||||
|
||||
if (this.#dockerMode) {
|
||||
this.#logger.debug("Skipping cleanup in docker mode", metadata);
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logger.log("Cleaning up", metadata);
|
||||
|
||||
try {
|
||||
await buildah.cleanup();
|
||||
await crictl.cleanup();
|
||||
} catch (error) {
|
||||
this.#logger.error("Error during cleanup", { ...metadata, error });
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
await this.chaosMonkey.call();
|
||||
|
||||
this.#logger.log("checkpointAndPush: checkpointing", { options });
|
||||
|
||||
const containterName = this.#getRunContainerName(runId);
|
||||
|
||||
// Create checkpoint (docker)
|
||||
if (this.#dockerMode) {
|
||||
await this.#createDockerCheckpoint(
|
||||
signal,
|
||||
runId,
|
||||
exportLocation,
|
||||
leaveRunning,
|
||||
attemptNumber
|
||||
);
|
||||
|
||||
this.#logger.log("checkpointAndPush: checkpoint created", {
|
||||
runId,
|
||||
location: exportLocation,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint: {
|
||||
location: exportLocation,
|
||||
docker: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Create checkpoint (CRI)
|
||||
if (!this.#canCheckpoint) {
|
||||
this.#logger.error("No checkpoint support in kubernetes mode.");
|
||||
return { success: false, reason: "SKIP_RETRYING" };
|
||||
}
|
||||
|
||||
const containerId = await crictl.ps(containterName, true);
|
||||
|
||||
if (!containerId.stdout) {
|
||||
this.#logger.error("could not find container id", { options, containterName });
|
||||
return { success: false, reason: "SKIP_RETRYING" };
|
||||
}
|
||||
|
||||
const start = performance.now();
|
||||
|
||||
if (this.simulateCheckpointFailure) {
|
||||
if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
|
||||
this.#logger.error("Simulating checkpoint failure", { options });
|
||||
throw new Error("SIMULATE_CHECKPOINT_FAILURE");
|
||||
}
|
||||
}
|
||||
|
||||
// Create checkpoint
|
||||
await crictl.checkpoint(containerId.stdout, exportLocation);
|
||||
const postCheckpoint = performance.now();
|
||||
|
||||
// Print checkpoint size
|
||||
const size = await getParsedFileSize(exportLocation);
|
||||
this.#logger.log("checkpoint archive created", { size, options });
|
||||
|
||||
// Create image from checkpoint
|
||||
const workingContainer = await buildah.from("scratch");
|
||||
const postFrom = performance.now();
|
||||
|
||||
await buildah.add(workingContainer.stdout, exportLocation, "/");
|
||||
const postAdd = performance.now();
|
||||
|
||||
await buildah.config(workingContainer.stdout, [
|
||||
`io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`,
|
||||
]);
|
||||
const postConfig = performance.now();
|
||||
|
||||
await buildah.commit(workingContainer.stdout, imageRef);
|
||||
const postCommit = performance.now();
|
||||
|
||||
if (this.simulatePushFailure) {
|
||||
if (performance.now() < this.simulatePushFailureSeconds * 1000) {
|
||||
this.#logger.error("Simulating push failure", { options });
|
||||
throw new Error("SIMULATE_PUSH_FAILURE");
|
||||
}
|
||||
}
|
||||
|
||||
// Push checkpoint image
|
||||
await buildah.push(imageRef, this.registryTlsVerify);
|
||||
const postPush = performance.now();
|
||||
|
||||
const perf = {
|
||||
"crictl checkpoint": postCheckpoint - start,
|
||||
"buildah from": postFrom - postCheckpoint,
|
||||
"buildah add": postAdd - postFrom,
|
||||
"buildah config": postConfig - postAdd,
|
||||
"buildah commit": postCommit - postConfig,
|
||||
"buildah push": postPush - postCommit,
|
||||
};
|
||||
|
||||
this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint: {
|
||||
location: imageRef,
|
||||
docker: false,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof Exec.Result) {
|
||||
if (error.aborted) {
|
||||
this.#logger.error("Checkpoint canceled: Exec", { options });
|
||||
|
||||
return { success: false, reason: "CANCELED" };
|
||||
} else {
|
||||
this.#logger.error("Checkpoint command error", { options, error });
|
||||
|
||||
return { success: false, reason: "ERROR" };
|
||||
}
|
||||
}
|
||||
|
||||
this.#logger.error("Unhandled checkpoint error", {
|
||||
options,
|
||||
error: error instanceof Error ? error.message : error,
|
||||
});
|
||||
|
||||
return { success: false, reason: "ERROR" };
|
||||
} finally {
|
||||
await cleanup();
|
||||
|
||||
if (signal.aborted) {
|
||||
this.#logger.error("Checkpoint canceled: Cleanup", { options });
|
||||
|
||||
// Overrides any prior return value (intentional use of return-in-finally)
|
||||
// eslint-disable-next-line no-unsafe-finally
|
||||
return { success: false, reason: "CANCELED" };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async unpause(runId: string, attemptNumber?: number): Promise<void> {
|
||||
try {
|
||||
const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
|
||||
const exec = new Exec({ logger: this.#logger });
|
||||
await exec.x("docker", ["unpause", containterNameWithAttempt]);
|
||||
} catch (error) {
|
||||
this.#logger.error("[Docker] Error during unpause", { runId, attemptNumber, error });
|
||||
}
|
||||
}
|
||||
|
||||
async #createDockerCheckpoint(
|
||||
abortSignal: AbortSignal,
|
||||
runId: string,
|
||||
exportLocation: string,
|
||||
leaveRunning: boolean,
|
||||
attemptNumber?: number
|
||||
) {
|
||||
const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
|
||||
const exec = new Exec({ logger: this.#logger, abortSignal });
|
||||
|
||||
try {
|
||||
if (this.opts.forceSimulate || !this.#canCheckpoint) {
|
||||
this.#logger.log("Simulating checkpoint");
|
||||
|
||||
await exec.x("docker", ["pause", containterNameWithAttempt]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.simulateCheckpointFailure) {
|
||||
if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
|
||||
this.#logger.error("Simulating checkpoint failure", {
|
||||
runId,
|
||||
exportLocation,
|
||||
leaveRunning,
|
||||
attemptNumber,
|
||||
});
|
||||
|
||||
throw new Error("SIMULATE_CHECKPOINT_FAILURE");
|
||||
}
|
||||
}
|
||||
|
||||
const args = ["checkpoint", "create"];
|
||||
|
||||
if (leaveRunning) {
|
||||
args.push("--leave-running");
|
||||
}
|
||||
|
||||
args.push(containterNameWithAttempt, exportLocation);
|
||||
|
||||
await exec.x("docker", args);
|
||||
} catch (error) {
|
||||
this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
#failCheckpoint(runId: string, error: unknown) {
|
||||
this.#failedCheckpoints.set(runId, error);
|
||||
}
|
||||
|
||||
#clearFailedCheckpoint(runId: string) {
|
||||
this.#failedCheckpoints.delete(runId);
|
||||
}
|
||||
|
||||
#hasFailedCheckpoint(runId: string) {
|
||||
return this.#failedCheckpoints.has(runId);
|
||||
}
|
||||
|
||||
#getRunContainerName(suffix: string, attemptNumber?: number) {
|
||||
return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
|
||||
}
|
||||
|
||||
#createTmpCleaner() {
|
||||
if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean);
|
||||
const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? [];
|
||||
const paths = pathsOverride.length ? pathsOverride : defaultPaths;
|
||||
|
||||
if (paths.length === 0) {
|
||||
this.#logger.error("TempFileCleaner enabled but no paths to clean", {
|
||||
defaultPaths,
|
||||
pathsOverride,
|
||||
TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
const cleaner = new TempFileCleaner({
|
||||
paths,
|
||||
maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60),
|
||||
intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300),
|
||||
leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false),
|
||||
});
|
||||
|
||||
cleaner.start();
|
||||
|
||||
return cleaner;
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
|
||||
import { Exec } from "./exec";
|
||||
import { setTimeout } from "timers/promises";
|
||||
|
||||
interface TempFileCleanerOptions {
|
||||
paths: string[];
|
||||
maxAgeMinutes: number;
|
||||
intervalSeconds: number;
|
||||
leadingEdge?: boolean;
|
||||
}
|
||||
|
||||
export class TempFileCleaner {
|
||||
private enabled = false;
|
||||
|
||||
private logger: SimpleStructuredLogger;
|
||||
private exec: Exec;
|
||||
|
||||
constructor(private opts: TempFileCleanerOptions) {
|
||||
this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts });
|
||||
this.exec = new Exec({ logger: this.logger });
|
||||
}
|
||||
|
||||
async start() {
|
||||
this.logger.log("TempFileCleaner.start");
|
||||
this.enabled = true;
|
||||
|
||||
if (!this.opts.leadingEdge) {
|
||||
await this.wait();
|
||||
}
|
||||
|
||||
while (this.enabled) {
|
||||
try {
|
||||
await this.clean();
|
||||
} catch (error) {
|
||||
this.logger.error("error during tick", { error });
|
||||
}
|
||||
|
||||
await this.wait();
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.logger.log("TempFileCleaner.stop");
|
||||
this.enabled = false;
|
||||
}
|
||||
|
||||
private wait() {
|
||||
return setTimeout(this.opts.intervalSeconds * 1000);
|
||||
}
|
||||
|
||||
private async clean() {
|
||||
for (const path of this.opts.paths) {
|
||||
try {
|
||||
await this.cleanSingle(path);
|
||||
} catch (error) {
|
||||
this.logger.error("error while cleaning", { path, error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanSingle(startingPoint: string) {
|
||||
const maxAgeMinutes = this.opts.maxAgeMinutes;
|
||||
|
||||
const ignoreStartingPoint = ["!", "-path", startingPoint];
|
||||
const onlyDirectDescendants = ["-maxdepth", "1"];
|
||||
const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`];
|
||||
|
||||
const baseArgs = [
|
||||
startingPoint,
|
||||
...ignoreStartingPoint,
|
||||
...onlyDirectDescendants,
|
||||
...onlyOldFiles,
|
||||
];
|
||||
|
||||
const duArgs = ["-exec", "du", "-ch", "{}", "+"];
|
||||
const rmArgs = ["-exec", "rm", "-rf", "{}", "+"];
|
||||
|
||||
const du = this.x("find", [...baseArgs, ...duArgs]);
|
||||
const duOutput = await du;
|
||||
|
||||
const duLines = duOutput.stdout.trim().split("\n");
|
||||
const fileCount = duLines.length - 1; // last line is the total
|
||||
const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0];
|
||||
|
||||
if (fileCount === 0) {
|
||||
this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes });
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
|
||||
|
||||
const rm = this.x("find", [...baseArgs, ...rmArgs]);
|
||||
const rmOutput = await rm;
|
||||
|
||||
if (rmOutput.stderr.length > 0) {
|
||||
this.logger.error("delete unsuccessful", { rmOutput });
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
|
||||
}
|
||||
|
||||
private get x() {
|
||||
return this.exec.x.bind(this.exec);
|
||||
}
|
||||
}
|
||||
@@ -1,293 +0,0 @@
|
||||
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
|
||||
import { randomUUID } from "crypto";
|
||||
import { homedir } from "os";
|
||||
import { type Result, x } from "tinyexec";
|
||||
|
||||
class TinyResult {
|
||||
pid?: number;
|
||||
exitCode?: number;
|
||||
aborted: boolean;
|
||||
killed: boolean;
|
||||
|
||||
constructor(result: Result) {
|
||||
this.pid = result.pid;
|
||||
this.exitCode = result.exitCode;
|
||||
this.aborted = result.aborted;
|
||||
this.killed = result.killed;
|
||||
}
|
||||
}
|
||||
|
||||
interface ExecOptions {
|
||||
logger?: SimpleStructuredLogger;
|
||||
abortSignal?: AbortSignal;
|
||||
logOutput?: boolean;
|
||||
trimArgs?: boolean;
|
||||
neverThrow?: boolean;
|
||||
}
|
||||
|
||||
export class Exec {
|
||||
private logger: SimpleStructuredLogger;
|
||||
private abortSignal: AbortSignal | undefined;
|
||||
|
||||
private logOutput: boolean;
|
||||
private trimArgs: boolean;
|
||||
private neverThrow: boolean;
|
||||
|
||||
constructor(opts: ExecOptions) {
|
||||
this.logger = opts.logger ?? new SimpleStructuredLogger("exec");
|
||||
this.abortSignal = opts.abortSignal;
|
||||
|
||||
this.logOutput = opts.logOutput ?? true;
|
||||
this.trimArgs = opts.trimArgs ?? true;
|
||||
this.neverThrow = opts.neverThrow ?? false;
|
||||
}
|
||||
|
||||
async x(
|
||||
command: string,
|
||||
args?: string[],
|
||||
opts?: { neverThrow?: boolean; ignoreAbort?: boolean }
|
||||
) {
|
||||
const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args;
|
||||
|
||||
const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`;
|
||||
this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed });
|
||||
|
||||
const result = x(command, argsTrimmed, {
|
||||
signal: opts?.ignoreAbort ? undefined : this.abortSignal,
|
||||
// We don't use this as it doesn't cover killed and aborted processes
|
||||
// throwOnError: true,
|
||||
});
|
||||
|
||||
const output = await result;
|
||||
|
||||
const metadata = {
|
||||
command,
|
||||
argsRaw: args,
|
||||
argsTrimmed,
|
||||
globalOpts: {
|
||||
trimArgs: this.trimArgs,
|
||||
neverThrow: this.neverThrow,
|
||||
hasAbortSignal: !!this.abortSignal,
|
||||
},
|
||||
localOpts: opts,
|
||||
stdout: output.stdout,
|
||||
stderr: output.stderr,
|
||||
pid: result.pid,
|
||||
exitCode: result.exitCode,
|
||||
aborted: result.aborted,
|
||||
killed: result.killed,
|
||||
};
|
||||
|
||||
if (this.logOutput) {
|
||||
this.logger.debug(`output: ${commandWithFirstArg}`, metadata);
|
||||
}
|
||||
|
||||
if (this.neverThrow || opts?.neverThrow) {
|
||||
return output;
|
||||
}
|
||||
|
||||
if (result.aborted) {
|
||||
this.logger.error(`aborted: ${commandWithFirstArg}`, metadata);
|
||||
throw new TinyResult(result);
|
||||
}
|
||||
|
||||
if (result.killed) {
|
||||
this.logger.error(`killed: ${commandWithFirstArg}`, metadata);
|
||||
throw new TinyResult(result);
|
||||
}
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata);
|
||||
throw new TinyResult(result);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static Result = TinyResult;
|
||||
}
|
||||
|
||||
interface BuildahOptions {
|
||||
id?: string;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export class Buildah {
|
||||
private id: string;
|
||||
private logger: SimpleStructuredLogger;
|
||||
private exec: Exec;
|
||||
|
||||
private containers = new Set<string>();
|
||||
private images = new Set<string>();
|
||||
|
||||
constructor(opts: BuildahOptions) {
|
||||
this.id = opts.id ?? randomUUID();
|
||||
this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id });
|
||||
|
||||
this.exec = new Exec({
|
||||
logger: this.logger,
|
||||
abortSignal: opts.abortSignal,
|
||||
});
|
||||
|
||||
this.logger.log("initiaized", { opts });
|
||||
}
|
||||
|
||||
private get x() {
|
||||
return this.exec.x.bind(this.exec);
|
||||
}
|
||||
|
||||
async from(baseImage: string) {
|
||||
const output = await this.x("buildah", ["from", baseImage]);
|
||||
this.containers.add(output.stdout);
|
||||
return output;
|
||||
}
|
||||
|
||||
async add(container: string, src: string, dest: string) {
|
||||
return await this.x("buildah", ["add", container, src, dest]);
|
||||
}
|
||||
|
||||
async config(container: string, annotations: string[]) {
|
||||
const args = ["config"];
|
||||
|
||||
for (const annotation of annotations) {
|
||||
args.push(`--annotation=${annotation}`);
|
||||
}
|
||||
|
||||
args.push(container);
|
||||
|
||||
return await this.x("buildah", args);
|
||||
}
|
||||
|
||||
async commit(container: string, imageRef: string) {
|
||||
const output = await this.x("buildah", ["commit", container, imageRef]);
|
||||
this.images.add(output.stdout);
|
||||
return output;
|
||||
}
|
||||
|
||||
async push(imageRef: string, registryTlsVerify?: boolean) {
|
||||
return await this.x("buildah", [
|
||||
"push",
|
||||
`--tls-verify=${String(!!registryTlsVerify)}`,
|
||||
imageRef,
|
||||
]);
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
if (this.containers.size > 0) {
|
||||
try {
|
||||
const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true });
|
||||
this.containers.clear();
|
||||
|
||||
if (output.stderr.length > 0) {
|
||||
this.logger.error("failed to remove some containers", { output });
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error("failed to clean up containers", { error, containers: this.containers });
|
||||
}
|
||||
} else {
|
||||
this.logger.debug("no containers to clean up");
|
||||
}
|
||||
|
||||
if (this.images.size > 0) {
|
||||
try {
|
||||
const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true });
|
||||
this.images.clear();
|
||||
|
||||
if (output.stderr.length > 0) {
|
||||
this.logger.error("failed to remove some images", { output });
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error("failed to clean up images", { error, images: this.images });
|
||||
}
|
||||
} else {
|
||||
this.logger.debug("no images to clean up");
|
||||
}
|
||||
}
|
||||
|
||||
static async canLogin(registryHost: string) {
|
||||
try {
|
||||
await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true });
|
||||
return true;
|
||||
} catch (_error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static get tmpDir() {
|
||||
return process.env.TMPDIR ?? "/var/tmp";
|
||||
}
|
||||
|
||||
static get storageRootDir() {
|
||||
return process.getuid?.() === 0
|
||||
? "/var/lib/containers/storage"
|
||||
: `${homedir()}/.local/share/containers/storage`;
|
||||
}
|
||||
}
|
||||
|
||||
interface CrictlOptions {
|
||||
id?: string;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export class Crictl {
|
||||
private id: string;
|
||||
private logger: SimpleStructuredLogger;
|
||||
private exec: Exec;
|
||||
|
||||
private archives = new Set<string>();
|
||||
|
||||
constructor(opts: CrictlOptions) {
|
||||
this.id = opts.id ?? randomUUID();
|
||||
this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id });
|
||||
|
||||
this.exec = new Exec({
|
||||
logger: this.logger,
|
||||
abortSignal: opts.abortSignal,
|
||||
});
|
||||
|
||||
this.logger.log("initiaized", { opts });
|
||||
}
|
||||
|
||||
private get x() {
|
||||
return this.exec.x.bind(this.exec);
|
||||
}
|
||||
|
||||
async ps(containerName: string, quiet?: boolean) {
|
||||
return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]);
|
||||
}
|
||||
|
||||
async checkpoint(containerId: string, exportLocation: string) {
|
||||
const output = await this.x("crictl", [
|
||||
"checkpoint",
|
||||
`--export=${exportLocation}`,
|
||||
containerId,
|
||||
]);
|
||||
this.archives.add(exportLocation);
|
||||
return output;
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
if (this.archives.size > 0) {
|
||||
try {
|
||||
const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true });
|
||||
this.archives.clear();
|
||||
|
||||
if (output.stderr.length > 0) {
|
||||
this.logger.error("failed to remove some archives", { output });
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error("failed to clean up archives", { error, archives: this.archives });
|
||||
}
|
||||
} else {
|
||||
this.logger.debug("no archives to clean up");
|
||||
}
|
||||
}
|
||||
|
||||
static getExportLocation(identifier: string) {
|
||||
return `${this.checkpointDir}/${identifier}.tar`;
|
||||
}
|
||||
|
||||
static get checkpointDir() {
|
||||
return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints";
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,31 +0,0 @@
|
||||
export const boolFromEnv = (env: string, defaultValue: boolean): boolean => {
|
||||
const value = process.env[env];
|
||||
|
||||
if (!value) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
return ["1", "true"].includes(value);
|
||||
};
|
||||
|
||||
export const numFromEnv = (env: string, defaultValue: number): number => {
|
||||
const value = process.env[env];
|
||||
|
||||
if (!value) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
return parseInt(value, 10);
|
||||
};
|
||||
|
||||
export function safeJsonParse(json?: string): unknown {
|
||||
if (!json) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(json);
|
||||
} catch (_e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2020",
|
||||
"module": "commonjs",
|
||||
"esModuleInterop": true,
|
||||
"resolveJsonModule": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"paths": {
|
||||
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
|
||||
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
HTTP_SERVER_PORT=8050
|
||||
|
||||
PLATFORM_WS_PORT=3030
|
||||
PLATFORM_SECRET=provider-secret
|
||||
SECURE_CONNECTION=false
|
||||
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
|
||||
|
||||
# Use this if you are on macOS
|
||||
# COORDINATOR_HOST="host.docker.internal"
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
|
||||
@@ -1,3 +0,0 @@
|
||||
dist/
|
||||
node_modules/
|
||||
.env
|
||||
@@ -1,47 +0,0 @@
|
||||
FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
FROM node-22-alpine AS pruner
|
||||
|
||||
COPY --chown=node:node . .
|
||||
RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
|
||||
RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
|
||||
|
||||
FROM node-22-alpine AS base
|
||||
|
||||
RUN apk add --no-cache dumb-init docker
|
||||
|
||||
COPY --chown=node:node .gitignore .gitignore
|
||||
COPY --from=pruner --chown=node:node /app/out/json/ .
|
||||
COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
|
||||
COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
|
||||
|
||||
FROM base AS dev-deps
|
||||
RUN corepack enable
|
||||
ENV NODE_ENV development
|
||||
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
|
||||
|
||||
FROM base AS builder
|
||||
RUN corepack enable
|
||||
|
||||
COPY --from=pruner --chown=node:node /app/out/full/ .
|
||||
COPY --from=dev-deps --chown=node:node /app/ .
|
||||
COPY --chown=node:node turbo.json turbo.json
|
||||
|
||||
RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter docker-provider build:bundle
|
||||
|
||||
FROM base AS runner
|
||||
|
||||
RUN corepack enable
|
||||
ENV NODE_ENV production
|
||||
|
||||
COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
USER node
|
||||
|
||||
CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]
|
||||
@@ -1,3 +0,0 @@
|
||||
# Docker provider
|
||||
|
||||
The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"name": "docker-provider",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"main": "dist/index.cjs",
|
||||
"scripts": {
|
||||
"build": "npm run build:bundle",
|
||||
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
|
||||
"build:image": "docker build -f Containerfile . -t docker-provider",
|
||||
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
|
||||
"start": "tsx src/index.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"execa": "^8.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dotenv": "^16.4.2",
|
||||
"esbuild": "^0.19.11",
|
||||
"tsx": "^4.7.0"
|
||||
}
|
||||
}
|
||||
@@ -1,295 +0,0 @@
|
||||
import type { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
|
||||
import type {
|
||||
TaskOperations,
|
||||
TaskOperationsCreateOptions,
|
||||
TaskOperationsIndexOptions,
|
||||
TaskOperationsRestoreOptions,
|
||||
} from "@trigger.dev/core/v3/apps";
|
||||
import { ProviderShell, SimpleLogger, isExecaChildProcess } from "@trigger.dev/core/v3/apps";
|
||||
import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
|
||||
import { $, type ExecaChildProcess, execa } from "execa";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
|
||||
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
||||
const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
|
||||
const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
||||
const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host";
|
||||
|
||||
const OTEL_EXPORTER_OTLP_ENDPOINT =
|
||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
|
||||
|
||||
const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
|
||||
process.env.FORCE_CHECKPOINT_SIMULATION ?? "false"
|
||||
);
|
||||
|
||||
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
|
||||
|
||||
type TaskOperationsInitReturn = {
|
||||
canCheckpoint: boolean;
|
||||
willSimulate: boolean;
|
||||
};
|
||||
|
||||
class DockerTaskOperations implements TaskOperations {
|
||||
#initialized = false;
|
||||
#canCheckpoint = false;
|
||||
|
||||
constructor(private opts = { forceSimulate: false }) {}
|
||||
|
||||
async init(): Promise<TaskOperationsInitReturn> {
|
||||
if (this.#initialized) {
|
||||
return this.#getInitReturn(this.#canCheckpoint);
|
||||
}
|
||||
|
||||
logger.log("Initializing task operations");
|
||||
|
||||
const testCheckpoint = await testDockerCheckpoint();
|
||||
|
||||
if (testCheckpoint.ok) {
|
||||
return this.#getInitReturn(true);
|
||||
}
|
||||
|
||||
logger.error(testCheckpoint.message, testCheckpoint.error);
|
||||
return this.#getInitReturn(false);
|
||||
}
|
||||
|
||||
#getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
|
||||
this.#canCheckpoint = canCheckpoint;
|
||||
|
||||
if (canCheckpoint) {
|
||||
if (!this.#initialized) {
|
||||
logger.log("Full checkpoint support!");
|
||||
}
|
||||
}
|
||||
|
||||
this.#initialized = true;
|
||||
|
||||
const willSimulate = !canCheckpoint || this.opts.forceSimulate;
|
||||
|
||||
if (willSimulate) {
|
||||
logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
|
||||
forceSimulate: this.opts.forceSimulate,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
canCheckpoint,
|
||||
willSimulate,
|
||||
};
|
||||
}
|
||||
|
||||
async index(opts: TaskOperationsIndexOptions) {
|
||||
await this.init();
|
||||
|
||||
const containerName = this.#getIndexContainerName(opts.shortCode);
|
||||
|
||||
logger.log(`Indexing task ${opts.imageRef}`, {
|
||||
host: COORDINATOR_HOST,
|
||||
port: COORDINATOR_PORT,
|
||||
});
|
||||
|
||||
logger.debug(
|
||||
await execa("docker", [
|
||||
"run",
|
||||
`--network=${DOCKER_NETWORK}`,
|
||||
"--rm",
|
||||
`--env=INDEX_TASKS=true`,
|
||||
`--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
|
||||
`--env=TRIGGER_API_URL=${opts.apiUrl}`,
|
||||
`--env=TRIGGER_ENV_ID=${opts.envId}`,
|
||||
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
|
||||
`--env=POD_NAME=${containerName}`,
|
||||
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
|
||||
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
|
||||
`--name=${containerName}`,
|
||||
`${opts.imageRef}`,
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
async create(opts: TaskOperationsCreateOptions) {
|
||||
await this.init();
|
||||
|
||||
const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
|
||||
|
||||
const runArgs = [
|
||||
"run",
|
||||
`--network=${DOCKER_NETWORK}`,
|
||||
"--detach",
|
||||
`--env=TRIGGER_ENV_ID=${opts.envId}`,
|
||||
`--env=TRIGGER_RUN_ID=${opts.runId}`,
|
||||
`--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
|
||||
`--env=POD_NAME=${containerName}`,
|
||||
`--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
|
||||
`--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
|
||||
`--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
|
||||
`--name=${containerName}`,
|
||||
];
|
||||
|
||||
if (process.env.ENFORCE_MACHINE_PRESETS) {
|
||||
runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
|
||||
}
|
||||
|
||||
if (opts.dequeuedAt) {
|
||||
runArgs.push(`--env=TRIGGER_RUN_DEQUEUED_AT_MS=${opts.dequeuedAt}`);
|
||||
}
|
||||
|
||||
runArgs.push(`${opts.image}`);
|
||||
|
||||
try {
|
||||
logger.debug(await execa("docker", runArgs));
|
||||
} catch (error) {
|
||||
if (!isExecaChildProcess(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
logger.error("Create failed:", {
|
||||
opts,
|
||||
exitCode: error.exitCode,
|
||||
escapedCommand: error.escapedCommand,
|
||||
stdout: error.stdout,
|
||||
stderr: error.stderr,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async restore(opts: TaskOperationsRestoreOptions) {
|
||||
await this.init();
|
||||
|
||||
const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
|
||||
|
||||
if (!this.#canCheckpoint || this.opts.forceSimulate) {
|
||||
logger.log("Simulating restore");
|
||||
|
||||
const unpause = logger.debug(await $`docker unpause ${containerName}`);
|
||||
|
||||
if (unpause.exitCode !== 0) {
|
||||
throw new Error("docker unpause command failed");
|
||||
}
|
||||
|
||||
await this.#sendPostStart(containerName);
|
||||
return;
|
||||
}
|
||||
|
||||
const { exitCode } = logger.debug(
|
||||
await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}`
|
||||
);
|
||||
|
||||
if (exitCode !== 0) {
|
||||
throw new Error("docker start command failed");
|
||||
}
|
||||
|
||||
await this.#sendPostStart(containerName);
|
||||
}
|
||||
|
||||
async delete(opts: { runId: string }) {
|
||||
await this.init();
|
||||
|
||||
const containerName = this.#getRunContainerName(opts.runId);
|
||||
await this.#sendPreStop(containerName);
|
||||
|
||||
logger.log("noop: delete");
|
||||
}
|
||||
|
||||
async get(opts: { runId: string }) {
|
||||
await this.init();
|
||||
|
||||
logger.log("noop: get");
|
||||
}
|
||||
|
||||
#getIndexContainerName(suffix: string) {
|
||||
return `task-index-${suffix}`;
|
||||
}
|
||||
|
||||
#getRunContainerName(suffix: string, attemptNumber?: number) {
|
||||
return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
|
||||
}
|
||||
|
||||
async #sendPostStart(containerName: string): Promise<void> {
|
||||
try {
|
||||
const port = await this.#getHttpServerPort(containerName);
|
||||
logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
|
||||
} catch (error) {
|
||||
logger.error("postStart error", { error });
|
||||
throw new Error("postStart command failed");
|
||||
}
|
||||
}
|
||||
|
||||
async #sendPreStop(containerName: string): Promise<void> {
|
||||
try {
|
||||
const port = await this.#getHttpServerPort(containerName);
|
||||
logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
|
||||
} catch (error) {
|
||||
logger.error("preStop error", { error });
|
||||
throw new Error("preStop command failed");
|
||||
}
|
||||
}
|
||||
|
||||
async #getHttpServerPort(containerName: string): Promise<number> {
|
||||
// We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
|
||||
// FIXME: Skip this in prod
|
||||
const logs = logger.debug(await $`docker logs ${containerName}`);
|
||||
const matches = logs.stdout.match(/http server listening on port (?<port>[0-9]+)/);
|
||||
|
||||
const port = Number(matches?.groups?.port);
|
||||
|
||||
if (!port) {
|
||||
throw new Error("failed to extract port from logs");
|
||||
}
|
||||
|
||||
return port;
|
||||
}
|
||||
|
||||
async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
|
||||
containerName: string,
|
||||
port: number,
|
||||
type: THookType,
|
||||
cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
|
||||
retryCount = 0
|
||||
): Promise<ExecaChildProcess> {
|
||||
try {
|
||||
return await execa("docker", [
|
||||
"exec",
|
||||
containerName,
|
||||
"busybox",
|
||||
"wget",
|
||||
"-q",
|
||||
"-O-",
|
||||
`127.0.0.1:${port}/${type}?cause=${cause}`,
|
||||
]);
|
||||
} catch (error: any) {
|
||||
if (type === "postStart" && retryCount < 6) {
|
||||
logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
|
||||
await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
|
||||
|
||||
return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
|
||||
}
|
||||
|
||||
logger.error(`final ${type} error`, { message: error?.message });
|
||||
throw new Error(`${type} command failed after ${retryCount - 1} retries`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const provider = new ProviderShell({
|
||||
tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
|
||||
type: "docker",
|
||||
});
|
||||
|
||||
provider.listen();
|
||||
|
||||
function exponentialBackoff(
|
||||
retryCount: number,
|
||||
exponential: number,
|
||||
minDelay: number,
|
||||
maxDelay: number,
|
||||
jitter: number
|
||||
): number {
|
||||
// Calculate the delay using the exponential backoff formula
|
||||
const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
|
||||
|
||||
// Calculate the jitter
|
||||
const jitterValue = Math.random() * jitter;
|
||||
|
||||
// Return the calculated delay with jitter
|
||||
return delay + jitterValue;
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2020",
|
||||
"module": "commonjs",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"paths": {
|
||||
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
|
||||
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
HTTP_SERVER_PORT=8060
|
||||
|
||||
PLATFORM_WS_PORT=3030
|
||||
PLATFORM_SECRET=provider-secret
|
||||
SECURE_CONNECTION=false
|
||||
|
||||
# Use this if you are on macOS
|
||||
# COORDINATOR_HOST="host.docker.internal"
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
|
||||
@@ -1,3 +0,0 @@
|
||||
dist/
|
||||
node_modules/
|
||||
.env
|
||||
@@ -1,47 +0,0 @@
|
||||
FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
FROM node-22-alpine AS pruner
|
||||
|
||||
COPY --chown=node:node . .
|
||||
RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
|
||||
RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
|
||||
|
||||
FROM node-22-alpine AS base
|
||||
|
||||
RUN apk add --no-cache dumb-init
|
||||
|
||||
COPY --chown=node:node .gitignore .gitignore
|
||||
COPY --from=pruner --chown=node:node /app/out/json/ .
|
||||
COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
|
||||
COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
|
||||
|
||||
FROM base AS dev-deps
|
||||
RUN corepack enable
|
||||
ENV NODE_ENV development
|
||||
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
|
||||
|
||||
FROM base AS builder
|
||||
RUN corepack enable
|
||||
|
||||
COPY --from=pruner --chown=node:node /app/out/full/ .
|
||||
COPY --from=dev-deps --chown=node:node /app/ .
|
||||
COPY --chown=node:node turbo.json turbo.json
|
||||
|
||||
RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter kubernetes-provider build:bundle
|
||||
|
||||
FROM base AS runner
|
||||
|
||||
RUN corepack enable
|
||||
ENV NODE_ENV production
|
||||
|
||||
COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
USER node
|
||||
|
||||
CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]
|
||||
@@ -1,3 +0,0 @@
|
||||
# Kubernetes provider
|
||||
|
||||
The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions.
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "kubernetes-provider",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"main": "dist/index.cjs",
|
||||
"scripts": {
|
||||
"build": "npm run build:bundle",
|
||||
"build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
|
||||
"build:image": "docker build -f Containerfile . -t kubernetes-provider",
|
||||
"dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
|
||||
"start": "tsx src/index.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@kubernetes/client-node": "^0.20.0",
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"p-queue": "^8.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dotenv": "^16.4.2",
|
||||
"esbuild": "^0.19.11",
|
||||
"tsx": "^4.7.0"
|
||||
}
|
||||
}
|
||||
@@ -1,782 +0,0 @@
|
||||
import * as k8s from "@kubernetes/client-node";
|
||||
import type {
|
||||
EnvironmentType,
|
||||
MachinePreset,
|
||||
PostStartCauses,
|
||||
PreStopCauses,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import type {
|
||||
TaskOperations,
|
||||
TaskOperationsCreateOptions,
|
||||
TaskOperationsIndexOptions,
|
||||
TaskOperationsPrePullDeploymentOptions,
|
||||
TaskOperationsRestoreOptions,
|
||||
} from "@trigger.dev/core/v3/apps";
|
||||
import { ProviderShell, SimpleLogger } from "@trigger.dev/core/v3/apps";
|
||||
import { PodCleaner } from "./podCleaner";
|
||||
import { TaskMonitor } from "./taskMonitor";
|
||||
import { UptimeHeartbeat } from "./uptimeHeartbeat";
|
||||
import { assertExhaustive } from "@trigger.dev/core";
|
||||
import { CustomLabelHelper } from "./labelHelper";
|
||||
|
||||
const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
|
||||
const NODE_NAME = process.env.NODE_NAME || "local";
|
||||
const OTEL_EXPORTER_OTLP_ENDPOINT =
|
||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
|
||||
const COORDINATOR_HOST = process.env.COORDINATOR_HOST ?? undefined;
|
||||
const COORDINATOR_PORT = process.env.COORDINATOR_PORT ?? undefined;
|
||||
const KUBERNETES_NAMESPACE = process.env.KUBERNETES_NAMESPACE ?? "default";
|
||||
|
||||
const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
|
||||
|
||||
const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
|
||||
const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
|
||||
const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
|
||||
const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
|
||||
const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
|
||||
|
||||
const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi";
|
||||
const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi";
|
||||
|
||||
// Image config
|
||||
const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true";
|
||||
const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS;
|
||||
const PAUSE_IMAGE = process.env.PAUSE_IMAGE || "registry.k8s.io/pause:3.9";
|
||||
const BUSYBOX_IMAGE = process.env.BUSYBOX_IMAGE || "registry.digitalocean.com/trigger/busybox";
|
||||
const DEPLOYMENT_IMAGE_PREFIX = process.env.DEPLOYMENT_IMAGE_PREFIX;
|
||||
const RESTORE_IMAGE_PREFIX = process.env.RESTORE_IMAGE_PREFIX;
|
||||
const UTILITY_IMAGE_PREFIX = process.env.UTILITY_IMAGE_PREFIX;
|
||||
|
||||
const logger = new SimpleLogger(`[${NODE_NAME}]`);
|
||||
logger.log(`running in ${RUNTIME_ENV} mode`);
|
||||
|
||||
type Namespace = {
|
||||
metadata: {
|
||||
name: string;
|
||||
};
|
||||
};
|
||||
|
||||
type ResourceQuantities = {
|
||||
[K in "cpu" | "memory" | "ephemeral-storage"]?: string;
|
||||
};
|
||||
|
||||
class KubernetesTaskOperations implements TaskOperations {
|
||||
#namespace: Namespace = {
|
||||
metadata: {
|
||||
name: "default",
|
||||
},
|
||||
};
|
||||
|
||||
#k8sApi: {
|
||||
core: k8s.CoreV1Api;
|
||||
batch: k8s.BatchV1Api;
|
||||
apps: k8s.AppsV1Api;
|
||||
};
|
||||
|
||||
#labelHelper = new CustomLabelHelper();
|
||||
|
||||
constructor(opts: { namespace?: string } = {}) {
|
||||
if (opts.namespace) {
|
||||
this.#namespace.metadata.name = opts.namespace;
|
||||
}
|
||||
|
||||
this.#k8sApi = this.#createK8sApi();
|
||||
}
|
||||
|
||||
async init() {
|
||||
// noop
|
||||
}
|
||||
|
||||
async index(opts: TaskOperationsIndexOptions) {
|
||||
await this.#createJob(
|
||||
{
|
||||
metadata: {
|
||||
name: this.#getIndexContainerName(opts.shortCode),
|
||||
namespace: this.#namespace.metadata.name,
|
||||
},
|
||||
spec: {
|
||||
completions: 1,
|
||||
backoffLimit: 0,
|
||||
ttlSecondsAfterFinished: 300,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
...this.#getSharedLabels(opts),
|
||||
app: "task-index",
|
||||
"app.kubernetes.io/part-of": "trigger-worker",
|
||||
"app.kubernetes.io/component": "index",
|
||||
deployment: opts.deploymentId,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
...this.#defaultPodSpec,
|
||||
containers: [
|
||||
{
|
||||
name: this.#getIndexContainerName(opts.shortCode),
|
||||
image: getImageRef("deployment", opts.imageRef),
|
||||
ports: [
|
||||
{
|
||||
containerPort: 8000,
|
||||
},
|
||||
],
|
||||
resources: {
|
||||
limits: {
|
||||
cpu: "1",
|
||||
memory: "2G",
|
||||
"ephemeral-storage": "2Gi",
|
||||
},
|
||||
},
|
||||
lifecycle: {
|
||||
preStop: {
|
||||
exec: {
|
||||
command: this.#getLifecycleCommand("preStop", "terminate"),
|
||||
},
|
||||
},
|
||||
},
|
||||
env: [
|
||||
...this.#getSharedEnv(opts.envId),
|
||||
{
|
||||
name: "INDEX_TASKS",
|
||||
value: "true",
|
||||
},
|
||||
{
|
||||
name: "TRIGGER_SECRET_KEY",
|
||||
value: opts.apiKey,
|
||||
},
|
||||
{
|
||||
name: "TRIGGER_API_URL",
|
||||
value: opts.apiUrl,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
this.#namespace
|
||||
);
|
||||
}
|
||||
|
||||
async create(opts: TaskOperationsCreateOptions) {
|
||||
const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
|
||||
|
||||
await this.#createPod(
|
||||
{
|
||||
metadata: {
|
||||
name: containerName,
|
||||
namespace: this.#namespace.metadata.name,
|
||||
labels: {
|
||||
...this.#labelHelper.getAdditionalLabels("create"),
|
||||
...this.#getSharedLabels(opts),
|
||||
app: "task-run",
|
||||
"app.kubernetes.io/part-of": "trigger-worker",
|
||||
"app.kubernetes.io/component": "create",
|
||||
run: opts.runId,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
...this.#defaultPodSpec,
|
||||
terminationGracePeriodSeconds: 60 * 60,
|
||||
containers: [
|
||||
{
|
||||
name: containerName,
|
||||
image: getImageRef("deployment", opts.image),
|
||||
ports: [
|
||||
{
|
||||
containerPort: 8000,
|
||||
},
|
||||
],
|
||||
resources: this.#getResourcesForMachine(opts.machine),
|
||||
lifecycle: {
|
||||
preStop: {
|
||||
exec: {
|
||||
command: this.#getLifecycleCommand("preStop", "terminate"),
|
||||
},
|
||||
},
|
||||
},
|
||||
env: [
|
||||
...this.#getSharedEnv(opts.envId),
|
||||
{
|
||||
name: "TRIGGER_RUN_ID",
|
||||
value: opts.runId,
|
||||
},
|
||||
...(opts.dequeuedAt
|
||||
? [{ name: "TRIGGER_RUN_DEQUEUED_AT_MS", value: String(opts.dequeuedAt) }]
|
||||
: []),
|
||||
],
|
||||
volumeMounts: [
|
||||
{
|
||||
name: "taskinfo",
|
||||
mountPath: "/etc/taskinfo",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{
|
||||
name: "taskinfo",
|
||||
emptyDir: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
this.#namespace
|
||||
);
|
||||
}
|
||||
|
||||
async restore(opts: TaskOperationsRestoreOptions) {
|
||||
await this.#createPod(
|
||||
{
|
||||
metadata: {
|
||||
name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
|
||||
namespace: this.#namespace.metadata.name,
|
||||
labels: {
|
||||
...this.#labelHelper.getAdditionalLabels("restore"),
|
||||
...this.#getSharedLabels(opts),
|
||||
app: "task-run",
|
||||
"app.kubernetes.io/part-of": "trigger-worker",
|
||||
"app.kubernetes.io/component": "restore",
|
||||
run: opts.runId,
|
||||
checkpoint: opts.checkpointId,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
...this.#defaultPodSpec,
|
||||
initContainers: [
|
||||
{
|
||||
name: "pull-base-image",
|
||||
image: getImageRef("deployment", opts.imageRef),
|
||||
command: ["sleep", "0"],
|
||||
},
|
||||
{
|
||||
name: "populate-taskinfo",
|
||||
image: getImageRef("utility", BUSYBOX_IMAGE),
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/bin/sh", "-c"],
|
||||
args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
|
||||
env: this.#coordinatorEnvVars,
|
||||
volumeMounts: [
|
||||
{
|
||||
name: "taskinfo",
|
||||
mountPath: "/etc/taskinfo",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
containers: [
|
||||
{
|
||||
name: this.#getRunContainerName(opts.runId),
|
||||
image: getImageRef("restore", opts.checkpointRef),
|
||||
ports: [
|
||||
{
|
||||
containerPort: 8000,
|
||||
},
|
||||
],
|
||||
resources: this.#getResourcesForMachine(opts.machine),
|
||||
lifecycle: {
|
||||
postStart: {
|
||||
exec: {
|
||||
command: this.#getLifecycleCommand("postStart", "restore"),
|
||||
},
|
||||
},
|
||||
preStop: {
|
||||
exec: {
|
||||
command: this.#getLifecycleCommand("preStop", "terminate"),
|
||||
},
|
||||
},
|
||||
},
|
||||
volumeMounts: [
|
||||
{
|
||||
name: "taskinfo",
|
||||
mountPath: "/etc/taskinfo",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{
|
||||
name: "taskinfo",
|
||||
emptyDir: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
this.#namespace
|
||||
);
|
||||
}
|
||||
|
||||
async delete(opts: { runId: string }) {
|
||||
await this.#deletePod({
|
||||
runId: opts.runId,
|
||||
namespace: this.#namespace,
|
||||
});
|
||||
}
|
||||
|
||||
async get(opts: { runId: string }) {
|
||||
await this.#getPod(opts.runId, this.#namespace);
|
||||
}
|
||||
|
||||
async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
|
||||
if (PRE_PULL_DISABLED) {
|
||||
logger.debug("Pre-pull is disabled, skipping.", { opts });
|
||||
return;
|
||||
}
|
||||
|
||||
const metaName = this.#getPrePullContainerName(opts.shortCode);
|
||||
|
||||
const metaLabels = {
|
||||
...this.#getSharedLabels(opts),
|
||||
app: "task-prepull",
|
||||
"app.kubernetes.io/part-of": "trigger-worker",
|
||||
"app.kubernetes.io/component": "prepull",
|
||||
deployment: opts.deploymentId,
|
||||
name: metaName,
|
||||
} satisfies k8s.V1ObjectMeta["labels"];
|
||||
|
||||
await this.#createDaemonSet(
|
||||
{
|
||||
metadata: {
|
||||
name: metaName,
|
||||
namespace: this.#namespace.metadata.name,
|
||||
labels: metaLabels,
|
||||
},
|
||||
spec: {
|
||||
selector: {
|
||||
matchLabels: {
|
||||
name: metaName,
|
||||
},
|
||||
},
|
||||
template: {
|
||||
metadata: {
|
||||
labels: metaLabels,
|
||||
},
|
||||
spec: {
|
||||
...this.#defaultPodSpec,
|
||||
restartPolicy: "Always",
|
||||
affinity: {
|
||||
nodeAffinity: {
|
||||
requiredDuringSchedulingIgnoredDuringExecution: {
|
||||
nodeSelectorTerms: [
|
||||
{
|
||||
matchExpressions: [
|
||||
{
|
||||
key: "trigger.dev/pre-pull-disabled",
|
||||
operator: "DoesNotExist",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
initContainers: [
|
||||
{
|
||||
name: "prepull",
|
||||
image: getImageRef("deployment", opts.imageRef),
|
||||
command: ["/usr/bin/true"],
|
||||
resources: {
|
||||
limits: {
|
||||
cpu: "0.25",
|
||||
memory: "100Mi",
|
||||
"ephemeral-storage": "1Gi",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
containers: [
|
||||
{
|
||||
name: "pause",
|
||||
image: getImageRef("utility", PAUSE_IMAGE),
|
||||
resources: {
|
||||
limits: {
|
||||
cpu: "1m",
|
||||
memory: "12Mi",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
this.#namespace
|
||||
);
|
||||
}
|
||||
|
||||
#envTypeToLabelValue(type: EnvironmentType) {
|
||||
switch (type) {
|
||||
case "PRODUCTION":
|
||||
return "prod";
|
||||
case "STAGING":
|
||||
return "stg";
|
||||
case "DEVELOPMENT":
|
||||
return "dev";
|
||||
case "PREVIEW":
|
||||
return "preview";
|
||||
}
|
||||
}
|
||||
|
||||
get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
|
||||
const pullSecrets = ["registry-trigger", "registry-trigger-failover"];
|
||||
|
||||
if (ADDITIONAL_PULL_SECRETS) {
|
||||
pullSecrets.push(...ADDITIONAL_PULL_SECRETS.split(","));
|
||||
}
|
||||
|
||||
const imagePullSecrets = pullSecrets.map(
|
||||
(name) => ({ name }) satisfies k8s.V1LocalObjectReference
|
||||
);
|
||||
|
||||
return {
|
||||
restartPolicy: "Never",
|
||||
automountServiceAccountToken: false,
|
||||
imagePullSecrets,
|
||||
nodeSelector: {
|
||||
nodetype: "worker",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
get #defaultResourceRequests(): ResourceQuantities {
|
||||
return {
|
||||
"ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST,
|
||||
};
|
||||
}
|
||||
|
||||
get #defaultResourceLimits(): ResourceQuantities {
|
||||
return {
|
||||
"ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT,
|
||||
};
|
||||
}
|
||||
|
||||
get #coordinatorHostEnvVar(): k8s.V1EnvVar {
|
||||
return COORDINATOR_HOST
|
||||
? {
|
||||
name: "COORDINATOR_HOST",
|
||||
value: COORDINATOR_HOST,
|
||||
}
|
||||
: {
|
||||
name: "COORDINATOR_HOST",
|
||||
valueFrom: {
|
||||
fieldRef: {
|
||||
fieldPath: "status.hostIP",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
get #coordinatorPortEnvVar(): k8s.V1EnvVar | undefined {
|
||||
if (COORDINATOR_PORT) {
|
||||
return {
|
||||
name: "COORDINATOR_PORT",
|
||||
value: COORDINATOR_PORT,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
get #coordinatorEnvVars(): k8s.V1EnvVar[] {
|
||||
const envVars = [this.#coordinatorHostEnvVar];
|
||||
|
||||
if (this.#coordinatorPortEnvVar) {
|
||||
envVars.push(this.#coordinatorPortEnvVar);
|
||||
}
|
||||
|
||||
return envVars;
|
||||
}
|
||||
|
||||
#getSharedEnv(envId: string): k8s.V1EnvVar[] {
|
||||
return [
|
||||
{
|
||||
name: "TRIGGER_ENV_ID",
|
||||
value: envId,
|
||||
},
|
||||
{
|
||||
name: "DEBUG",
|
||||
value: process.env.DEBUG ? "1" : "0",
|
||||
},
|
||||
{
|
||||
name: "HTTP_SERVER_PORT",
|
||||
value: "8000",
|
||||
},
|
||||
{
|
||||
name: "OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||
value: OTEL_EXPORTER_OTLP_ENDPOINT,
|
||||
},
|
||||
{
|
||||
name: "POD_NAME",
|
||||
valueFrom: {
|
||||
fieldRef: {
|
||||
fieldPath: "metadata.name",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "MACHINE_NAME",
|
||||
valueFrom: {
|
||||
fieldRef: {
|
||||
fieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "TRIGGER_POD_SCHEDULED_AT_MS",
|
||||
value: Date.now().toString(),
|
||||
},
|
||||
...this.#coordinatorEnvVars,
|
||||
];
|
||||
}
|
||||
|
||||
#getSharedLabels(
|
||||
opts:
|
||||
| TaskOperationsIndexOptions
|
||||
| TaskOperationsCreateOptions
|
||||
| TaskOperationsRestoreOptions
|
||||
| TaskOperationsPrePullDeploymentOptions
|
||||
): Record<string, string> {
|
||||
return {
|
||||
env: opts.envId,
|
||||
envtype: this.#envTypeToLabelValue(opts.envType),
|
||||
org: opts.orgId,
|
||||
project: opts.projectId,
|
||||
};
|
||||
}
|
||||
|
||||
#getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
|
||||
return {
|
||||
cpu: `${preset.cpu * 0.75}`,
|
||||
memory: `${preset.memory}G`,
|
||||
};
|
||||
}
|
||||
|
||||
#getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
|
||||
return {
|
||||
cpu: `${preset.cpu}`,
|
||||
memory: `${preset.memory}G`,
|
||||
};
|
||||
}
|
||||
|
||||
#getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements {
|
||||
return {
|
||||
requests: {
|
||||
...this.#defaultResourceRequests,
|
||||
...this.#getResourceRequestsForMachine(preset),
|
||||
},
|
||||
limits: {
|
||||
...this.#defaultResourceLimits,
|
||||
...this.#getResourceLimitsForMachine(preset),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
#getLifecycleCommand<THookType extends "postStart" | "preStop">(
|
||||
type: THookType,
|
||||
cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
|
||||
) {
|
||||
const retries = 5;
|
||||
|
||||
// This will retry sending the lifecycle hook up to `retries` times
|
||||
// The sleep is required as this may start running before the HTTP server is up
|
||||
const exec = [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
`for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
|
||||
];
|
||||
|
||||
logger.debug("getLifecycleCommand()", { exec });
|
||||
|
||||
return exec;
|
||||
}
|
||||
|
||||
#getIndexContainerName(suffix: string) {
|
||||
return `task-index-${suffix}`;
|
||||
}
|
||||
|
||||
#getRunContainerName(suffix: string, attemptNumber?: number) {
|
||||
return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
|
||||
}
|
||||
|
||||
#getPrePullContainerName(suffix: string) {
|
||||
return `task-prepull-${suffix}`;
|
||||
}
|
||||
|
||||
#createK8sApi() {
|
||||
const kubeConfig = new k8s.KubeConfig();
|
||||
|
||||
if (RUNTIME_ENV === "local") {
|
||||
kubeConfig.loadFromDefault();
|
||||
} else if (RUNTIME_ENV === "kubernetes") {
|
||||
kubeConfig.loadFromCluster();
|
||||
} else {
|
||||
throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`);
|
||||
}
|
||||
|
||||
return {
|
||||
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
|
||||
batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
|
||||
apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
|
||||
};
|
||||
}
|
||||
|
||||
async #createPod(pod: k8s.V1Pod, namespace: Namespace) {
|
||||
try {
|
||||
const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
|
||||
logger.debug(res.body);
|
||||
} catch (err: unknown) {
|
||||
this.#handleK8sError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async #deletePod(opts: { runId: string; namespace: Namespace }) {
|
||||
try {
|
||||
const res = await this.#k8sApi.core.deleteNamespacedPod(
|
||||
opts.runId,
|
||||
opts.namespace.metadata.name
|
||||
);
|
||||
logger.debug(res.body);
|
||||
} catch (err: unknown) {
|
||||
this.#handleK8sError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async #getPod(runId: string, namespace: Namespace) {
|
||||
try {
|
||||
const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name);
|
||||
logger.debug(res.body);
|
||||
return res.body;
|
||||
} catch (err: unknown) {
|
||||
this.#handleK8sError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async #createJob(job: k8s.V1Job, namespace: Namespace) {
|
||||
try {
|
||||
const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
|
||||
logger.debug(res.body);
|
||||
} catch (err: unknown) {
|
||||
this.#handleK8sError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
|
||||
try {
|
||||
const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
|
||||
namespace.metadata.name,
|
||||
daemonSet
|
||||
);
|
||||
logger.debug(res.body);
|
||||
} catch (err: unknown) {
|
||||
this.#handleK8sError(err);
|
||||
}
|
||||
}
|
||||
|
||||
#throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
|
||||
if (typeof candidate !== "object" || candidate === null) {
|
||||
throw candidate;
|
||||
}
|
||||
}
|
||||
|
||||
#handleK8sError(err: unknown) {
|
||||
this.#throwUnlessRecord(err);
|
||||
|
||||
if ("body" in err && err.body) {
|
||||
logger.error(err.body);
|
||||
this.#throwUnlessRecord(err.body);
|
||||
|
||||
if (typeof err.body.message === "string") {
|
||||
throw new Error(err.body?.message);
|
||||
} else {
|
||||
throw err.body;
|
||||
}
|
||||
} else {
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type ImageType = "deployment" | "restore" | "utility";
|
||||
|
||||
function getImagePrefix(type: ImageType) {
|
||||
switch (type) {
|
||||
case "deployment":
|
||||
return DEPLOYMENT_IMAGE_PREFIX;
|
||||
case "restore":
|
||||
return RESTORE_IMAGE_PREFIX;
|
||||
case "utility":
|
||||
return UTILITY_IMAGE_PREFIX;
|
||||
default:
|
||||
assertExhaustive(type);
|
||||
}
|
||||
}
|
||||
|
||||
function getImageRef(type: ImageType, ref: string) {
|
||||
const prefix = getImagePrefix(type);
|
||||
return prefix ? `${prefix}/${ref}` : ref;
|
||||
}
|
||||
|
||||
const provider = new ProviderShell({
|
||||
tasks: new KubernetesTaskOperations({
|
||||
namespace: KUBERNETES_NAMESPACE,
|
||||
}),
|
||||
type: "kubernetes",
|
||||
});
|
||||
|
||||
provider.listen();
|
||||
|
||||
const taskMonitor = new TaskMonitor({
|
||||
runtimeEnv: RUNTIME_ENV,
|
||||
namespace: KUBERNETES_NAMESPACE,
|
||||
onIndexFailure: async (deploymentId, details) => {
|
||||
logger.log("Indexing failed", { deploymentId, details });
|
||||
|
||||
try {
|
||||
provider.platformSocket.send("INDEXING_FAILED", {
|
||||
deploymentId,
|
||||
error: {
|
||||
name: `Crashed with exit code ${details.exitCode}`,
|
||||
message: details.reason,
|
||||
stack: details.logs,
|
||||
},
|
||||
overrideCompletion: details.overrideCompletion,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
}
|
||||
},
|
||||
onRunFailure: async (runId, details) => {
|
||||
logger.log("Run failed:", { runId, details });
|
||||
|
||||
try {
|
||||
provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
taskMonitor.start();
|
||||
|
||||
const podCleaner = new PodCleaner({
|
||||
runtimeEnv: RUNTIME_ENV,
|
||||
namespace: KUBERNETES_NAMESPACE,
|
||||
intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
|
||||
});
|
||||
|
||||
podCleaner.start();
|
||||
|
||||
if (UPTIME_HEARTBEAT_URL) {
|
||||
const uptimeHeartbeat = new UptimeHeartbeat({
|
||||
runtimeEnv: RUNTIME_ENV,
|
||||
namespace: KUBERNETES_NAMESPACE,
|
||||
intervalInSeconds: UPTIME_INTERVAL_SECONDS,
|
||||
pingUrl: UPTIME_HEARTBEAT_URL,
|
||||
maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
|
||||
maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
|
||||
maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
|
||||
});
|
||||
|
||||
uptimeHeartbeat.start();
|
||||
} else {
|
||||
logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
import { assertExhaustive } from "@trigger.dev/core";
|
||||
|
||||
const CREATE_LABEL_ENV_VAR_PREFIX = "DEPLOYMENT_LABEL_";
|
||||
const RESTORE_LABEL_ENV_VAR_PREFIX = "RESTORE_LABEL_";
|
||||
const LABEL_SAMPLE_RATE_POSTFIX = "_SAMPLE_RATE";
|
||||
|
||||
type OperationType = "create" | "restore";
|
||||
|
||||
type CustomLabel = {
|
||||
key: string;
|
||||
value: string;
|
||||
sampleRate: number;
|
||||
};
|
||||
|
||||
export class CustomLabelHelper {
|
||||
// Labels and sample rates are defined in environment variables so only need to be computed once
|
||||
private createLabels?: CustomLabel[];
|
||||
private restoreLabels?: CustomLabel[];
|
||||
|
||||
private getLabelPrefix(type: OperationType) {
|
||||
const prefix = type === "create" ? CREATE_LABEL_ENV_VAR_PREFIX : RESTORE_LABEL_ENV_VAR_PREFIX;
|
||||
return prefix.toLowerCase();
|
||||
}
|
||||
|
||||
private getLabelSampleRatePostfix() {
|
||||
return LABEL_SAMPLE_RATE_POSTFIX.toLowerCase();
|
||||
}
|
||||
|
||||
// Can only range from 0 to 1
|
||||
private fractionFromPercent(percent: number) {
|
||||
return Math.min(1, Math.max(0, percent / 100));
|
||||
}
|
||||
|
||||
private isLabelSampleRateEnvVar(key: string) {
|
||||
return key.toLowerCase().endsWith(this.getLabelSampleRatePostfix());
|
||||
}
|
||||
|
||||
private isLabelEnvVar(type: OperationType, key: string) {
|
||||
const prefix = this.getLabelPrefix(type);
|
||||
return key.toLowerCase().startsWith(prefix) && !this.isLabelSampleRateEnvVar(key);
|
||||
}
|
||||
|
||||
private getSampleRateEnvVarKey(type: OperationType, envKey: string) {
|
||||
return `${envKey.toLowerCase()}${this.getLabelSampleRatePostfix()}`;
|
||||
}
|
||||
|
||||
private getLabelNameFromEnvVarKey(type: OperationType, key: string) {
|
||||
return key
|
||||
.slice(this.getLabelPrefix(type).length)
|
||||
.toLowerCase()
|
||||
.replace(/___/g, ".")
|
||||
.replace(/__/g, "/")
|
||||
.replace(/_/g, "-");
|
||||
}
|
||||
|
||||
private getCaseInsensitiveEnvValue(key: string) {
|
||||
for (const [envKey, value] of Object.entries(process.env)) {
|
||||
if (envKey.toLowerCase() === key.toLowerCase()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the sample rate for a given label as fraction of 100 */
|
||||
private getSampleRateFromEnvVarKey(type: OperationType, envKey: string) {
|
||||
// Apply default: always sample
|
||||
const DEFAULT_SAMPLE_RATE_PERCENT = 100;
|
||||
const defaultSampleRateFraction = this.fractionFromPercent(DEFAULT_SAMPLE_RATE_PERCENT);
|
||||
|
||||
const value = this.getCaseInsensitiveEnvValue(this.getSampleRateEnvVarKey(type, envKey));
|
||||
|
||||
if (!value) {
|
||||
return defaultSampleRateFraction;
|
||||
}
|
||||
|
||||
const sampleRatePercent = parseFloat(value || String(DEFAULT_SAMPLE_RATE_PERCENT));
|
||||
|
||||
if (isNaN(sampleRatePercent)) {
|
||||
return defaultSampleRateFraction;
|
||||
}
|
||||
|
||||
const fractionalSampleRate = this.fractionFromPercent(sampleRatePercent);
|
||||
|
||||
return fractionalSampleRate;
|
||||
}
|
||||
|
||||
private getCustomLabels(type: OperationType): CustomLabel[] {
|
||||
switch (type) {
|
||||
case "create":
|
||||
if (this.createLabels) {
|
||||
return this.createLabels;
|
||||
}
|
||||
break;
|
||||
case "restore":
|
||||
if (this.restoreLabels) {
|
||||
return this.restoreLabels;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
assertExhaustive(type);
|
||||
}
|
||||
|
||||
const customLabels: CustomLabel[] = [];
|
||||
|
||||
for (const [envKey, value] of Object.entries(process.env)) {
|
||||
const key = envKey.toLowerCase();
|
||||
|
||||
// Only process env vars that start with the expected prefix
|
||||
if (!this.isLabelEnvVar(type, key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip sample rates - deal with them separately
|
||||
if (this.isLabelSampleRateEnvVar(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const labelName = this.getLabelNameFromEnvVarKey(type, key);
|
||||
const sampleRate = this.getSampleRateFromEnvVarKey(type, key);
|
||||
|
||||
const label = {
|
||||
key: labelName,
|
||||
value: value || "",
|
||||
sampleRate,
|
||||
} satisfies CustomLabel;
|
||||
|
||||
customLabels.push(label);
|
||||
}
|
||||
|
||||
return customLabels;
|
||||
}
|
||||
|
||||
getAdditionalLabels(type: OperationType): Record<string, string> {
|
||||
const labels = this.getCustomLabels(type);
|
||||
|
||||
const additionalLabels: Record<string, string> = {};
|
||||
|
||||
for (const { key, value, sampleRate } of labels) {
|
||||
// Always apply label if sample rate is 1
|
||||
if (sampleRate === 1) {
|
||||
additionalLabels[key] = value;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Math.random() <= sampleRate) {
|
||||
additionalLabels[key] = value;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return additionalLabels;
|
||||
}
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
import * as k8s from "@kubernetes/client-node";
|
||||
import { SimpleLogger } from "@trigger.dev/core/v3/apps";
|
||||
|
||||
type PodCleanerOptions = {
|
||||
runtimeEnv: "local" | "kubernetes";
|
||||
namespace?: string;
|
||||
intervalInSeconds?: number;
|
||||
};
|
||||
|
||||
export class PodCleaner {
|
||||
private enabled = false;
|
||||
private namespace = "default";
|
||||
private intervalInSeconds = 300;
|
||||
|
||||
private logger = new SimpleLogger("[PodCleaner]");
|
||||
private k8sClient: {
|
||||
core: k8s.CoreV1Api;
|
||||
apps: k8s.AppsV1Api;
|
||||
kubeConfig: k8s.KubeConfig;
|
||||
};
|
||||
|
||||
constructor(private opts: PodCleanerOptions) {
|
||||
if (opts.namespace) {
|
||||
this.namespace = opts.namespace;
|
||||
}
|
||||
|
||||
if (opts.intervalInSeconds) {
|
||||
this.intervalInSeconds = opts.intervalInSeconds;
|
||||
}
|
||||
|
||||
this.k8sClient = this.#createK8sClient();
|
||||
}
|
||||
|
||||
#createK8sClient() {
|
||||
const kubeConfig = new k8s.KubeConfig();
|
||||
|
||||
if (this.opts.runtimeEnv === "local") {
|
||||
kubeConfig.loadFromDefault();
|
||||
} else if (this.opts.runtimeEnv === "kubernetes") {
|
||||
kubeConfig.loadFromCluster();
|
||||
} else {
|
||||
throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
|
||||
}
|
||||
|
||||
return {
|
||||
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
|
||||
apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
|
||||
kubeConfig: kubeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
#isRecord(candidate: unknown): candidate is Record<string, unknown> {
|
||||
if (typeof candidate !== "object" || candidate === null) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
#logK8sError(err: unknown, debugOnly = false) {
|
||||
if (debugOnly) {
|
||||
this.logger.debug("K8s API Error", err);
|
||||
} else {
|
||||
this.logger.error("K8s API Error", err);
|
||||
}
|
||||
}
|
||||
|
||||
#handleK8sError(err: unknown) {
|
||||
if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
|
||||
this.#logK8sError(err);
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logK8sError(err, true);
|
||||
|
||||
if (typeof err.body.message === "string") {
|
||||
this.#logK8sError({ message: err.body.message });
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logK8sError({ body: err.body });
|
||||
}
|
||||
|
||||
async #deletePods(opts: {
|
||||
namespace: string;
|
||||
dryRun?: boolean;
|
||||
fieldSelector?: string;
|
||||
labelSelector?: string;
|
||||
}) {
|
||||
return await this.k8sClient.core
|
||||
.deleteCollectionNamespacedPod(
|
||||
opts.namespace,
|
||||
undefined, // pretty
|
||||
undefined, // continue
|
||||
opts.dryRun ? "All" : undefined,
|
||||
opts.fieldSelector,
|
||||
undefined, // gracePeriodSeconds
|
||||
opts.labelSelector
|
||||
)
|
||||
.catch(this.#handleK8sError.bind(this));
|
||||
}
|
||||
|
||||
async #deleteDaemonSets(opts: {
|
||||
namespace: string;
|
||||
dryRun?: boolean;
|
||||
fieldSelector?: string;
|
||||
labelSelector?: string;
|
||||
}) {
|
||||
return await this.k8sClient.apps
|
||||
.deleteCollectionNamespacedDaemonSet(
|
||||
opts.namespace,
|
||||
undefined, // pretty
|
||||
undefined, // continue
|
||||
opts.dryRun ? "All" : undefined,
|
||||
opts.fieldSelector,
|
||||
undefined, // gracePeriodSeconds
|
||||
opts.labelSelector
|
||||
)
|
||||
.catch(this.#handleK8sError.bind(this));
|
||||
}
|
||||
|
||||
async #deleteCompletedRuns() {
|
||||
this.logger.log("Deleting completed runs");
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
const result = await this.#deletePods({
|
||||
namespace: this.namespace,
|
||||
fieldSelector: "status.phase=Succeeded",
|
||||
labelSelector: "app=task-run",
|
||||
});
|
||||
|
||||
const elapsedMs = Date.now() - start;
|
||||
|
||||
if (!result) {
|
||||
this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
|
||||
return;
|
||||
}
|
||||
|
||||
const total = (result.response as any)?.body?.items?.length ?? 0;
|
||||
|
||||
this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
|
||||
}
|
||||
|
||||
async #deleteFailedRuns() {
|
||||
this.logger.log("Deleting failed runs");
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
const result = await this.#deletePods({
|
||||
namespace: this.namespace,
|
||||
fieldSelector: "status.phase=Failed",
|
||||
labelSelector: "app=task-run",
|
||||
});
|
||||
|
||||
const elapsedMs = Date.now() - start;
|
||||
|
||||
if (!result) {
|
||||
this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
|
||||
return;
|
||||
}
|
||||
|
||||
const total = (result.response as any)?.body?.items?.length ?? 0;
|
||||
|
||||
this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
|
||||
}
|
||||
|
||||
async #deleteCompletedPrePulls() {
|
||||
this.logger.log("Deleting completed pre-pulls");
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
const result = await this.#deleteDaemonSets({
|
||||
namespace: this.namespace,
|
||||
labelSelector: "app=task-prepull",
|
||||
});
|
||||
|
||||
const elapsedMs = Date.now() - start;
|
||||
|
||||
if (!result) {
|
||||
this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
|
||||
return;
|
||||
}
|
||||
|
||||
const total = (result.response as any)?.body?.items?.length ?? 0;
|
||||
|
||||
this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
|
||||
}
|
||||
|
||||
async start() {
|
||||
this.enabled = true;
|
||||
this.logger.log("Starting");
|
||||
|
||||
const completedInterval = setInterval(async () => {
|
||||
if (!this.enabled) {
|
||||
clearInterval(completedInterval);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#deleteCompletedRuns();
|
||||
} catch (error) {
|
||||
this.logger.error("Error deleting completed runs", error);
|
||||
}
|
||||
}, this.intervalInSeconds * 1000);
|
||||
|
||||
const failedInterval = setInterval(
|
||||
async () => {
|
||||
if (!this.enabled) {
|
||||
clearInterval(failedInterval);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#deleteFailedRuns();
|
||||
} catch (error) {
|
||||
this.logger.error("Error deleting completed runs", error);
|
||||
}
|
||||
},
|
||||
// Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
|
||||
2 * this.intervalInSeconds * 1000
|
||||
);
|
||||
|
||||
const completedPrePullInterval = setInterval(
|
||||
async () => {
|
||||
if (!this.enabled) {
|
||||
clearInterval(completedPrePullInterval);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#deleteCompletedPrePulls();
|
||||
} catch (error) {
|
||||
this.logger.error("Error deleting completed pre-pulls", error);
|
||||
}
|
||||
},
|
||||
2 * this.intervalInSeconds * 1000
|
||||
);
|
||||
|
||||
// this.#launchTests();
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.enabled = false;
|
||||
this.logger.log("Shutting down..");
|
||||
}
|
||||
}
|
||||
@@ -1,385 +0,0 @@
|
||||
import * as k8s from "@kubernetes/client-node";
|
||||
import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3";
|
||||
import {
|
||||
EXIT_CODE_ALREADY_HANDLED,
|
||||
EXIT_CODE_CHILD_NONZERO,
|
||||
SimpleLogger,
|
||||
} from "@trigger.dev/core/v3/apps";
|
||||
import PQueue from "p-queue";
|
||||
import { setTimeout } from "timers/promises";
|
||||
|
||||
type FailureDetails = Prettify<{
|
||||
exitCode: number;
|
||||
reason: string;
|
||||
logs: string;
|
||||
overrideCompletion: boolean;
|
||||
errorCode: TaskRunInternalError["code"];
|
||||
}>;
|
||||
|
||||
type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
|
||||
|
||||
type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
|
||||
|
||||
type TaskMonitorOptions = {
|
||||
runtimeEnv: "local" | "kubernetes";
|
||||
onIndexFailure?: IndexFailureHandler;
|
||||
onRunFailure?: RunFailureHandler;
|
||||
namespace?: string;
|
||||
};
|
||||
|
||||
export class TaskMonitor {
|
||||
#enabled = false;
|
||||
|
||||
#logger = new SimpleLogger("[TaskMonitor]");
|
||||
#taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
|
||||
#processedPods = new Map<string, number>();
|
||||
#queue = new PQueue({ concurrency: 10 });
|
||||
|
||||
#k8sClient: {
|
||||
core: k8s.CoreV1Api;
|
||||
kubeConfig: k8s.KubeConfig;
|
||||
};
|
||||
|
||||
private namespace = "default";
|
||||
private fieldSelector = "status.phase=Failed";
|
||||
private labelSelector = "app in (task-index, task-run)";
|
||||
|
||||
constructor(private opts: TaskMonitorOptions) {
|
||||
if (opts.namespace) {
|
||||
this.namespace = opts.namespace;
|
||||
}
|
||||
|
||||
this.#k8sClient = this.#createK8sClient();
|
||||
|
||||
this.#taskInformer = this.#createTaskInformer();
|
||||
this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
|
||||
this.#taskInformer.on("error", this.#onInformerError.bind(this));
|
||||
this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
|
||||
}
|
||||
|
||||
#createTaskInformer() {
|
||||
const listTasks = () =>
|
||||
this.#k8sClient.core.listNamespacedPod(
|
||||
this.namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
this.fieldSelector,
|
||||
this.labelSelector
|
||||
);
|
||||
|
||||
// Uses watch with local caching
|
||||
// https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
|
||||
const informer = k8s.makeInformer(
|
||||
this.#k8sClient.kubeConfig,
|
||||
`/api/v1/namespaces/${this.namespace}/pods`,
|
||||
listTasks,
|
||||
this.labelSelector,
|
||||
this.fieldSelector
|
||||
);
|
||||
|
||||
return informer;
|
||||
}
|
||||
|
||||
async #onInformerConnected() {
|
||||
this.#logger.log("Connected");
|
||||
}
|
||||
|
||||
async #onInformerError(error: any) {
|
||||
this.#logger.error("Error:", error);
|
||||
|
||||
// Automatic reconnect
|
||||
await setTimeout(2_000);
|
||||
this.#taskInformer.start();
|
||||
}
|
||||
|
||||
#enqueueOnPodUpdated(pod: k8s.V1Pod) {
|
||||
this.#queue.add(async () => {
|
||||
try {
|
||||
// It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
|
||||
await this.#onPodUpdated(pod);
|
||||
} catch (error) {
|
||||
this.#logger.error("Caught onPodUpdated() error:", error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async #onPodUpdated(pod: k8s.V1Pod) {
|
||||
this.#logger.debug(`Updated: ${pod.metadata?.name}`);
|
||||
this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
|
||||
|
||||
// We only care about failures
|
||||
if (pod.status?.phase !== "Failed") {
|
||||
return;
|
||||
}
|
||||
|
||||
const podName = pod.metadata?.name;
|
||||
|
||||
if (!podName) {
|
||||
this.#logger.error("Pod is nameless", { pod });
|
||||
return;
|
||||
}
|
||||
|
||||
const containerStatus = pod.status.containerStatuses?.[0];
|
||||
|
||||
if (!containerStatus?.state) {
|
||||
this.#logger.error("Pod failed, but container status doesn't have state", {
|
||||
status: pod.status,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.#processedPods.has(podName)) {
|
||||
this.#logger.debug("Pod update already processed", {
|
||||
podName,
|
||||
timestamp: this.#processedPods.get(podName),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
this.#processedPods.set(podName, Date.now());
|
||||
|
||||
const podStatus = this.#getPodStatusSummary(pod.status);
|
||||
const containerState = this.#getContainerStateSummary(containerStatus.state);
|
||||
const exitCode = containerState.exitCode ?? -1;
|
||||
|
||||
if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
|
||||
this.#logger.debug("Ignoring pod failure, already handled by worker", {
|
||||
podName,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const rawLogs = await this.#getLogTail(podName);
|
||||
|
||||
this.#logger.log(`${podName} failed with:`, {
|
||||
podStatus,
|
||||
containerState,
|
||||
rawLogs,
|
||||
});
|
||||
|
||||
const rawReason = podStatus.reason ?? containerState.reason ?? "";
|
||||
const message = podStatus.message ?? containerState.message ?? "";
|
||||
|
||||
let reason = rawReason || "Unknown error";
|
||||
let logs = rawLogs || "";
|
||||
|
||||
/** This will only override existing task errors. It will not crash the run. */
|
||||
let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO;
|
||||
|
||||
let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
|
||||
|
||||
switch (rawReason) {
|
||||
case "Error":
|
||||
reason = "Unknown error.";
|
||||
errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
|
||||
break;
|
||||
case "Evicted":
|
||||
if (message.startsWith("Pod ephemeral local storage usage")) {
|
||||
reason = "Storage limit exceeded.";
|
||||
errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED;
|
||||
} else if (message) {
|
||||
reason = `Evicted: ${message}`;
|
||||
errorCode = TaskRunErrorCodes.POD_EVICTED;
|
||||
} else {
|
||||
reason = "Evicted for unknown reason.";
|
||||
errorCode = TaskRunErrorCodes.POD_EVICTED;
|
||||
}
|
||||
|
||||
if (logs.startsWith("failed to try resolving symlinks")) {
|
||||
logs = "";
|
||||
}
|
||||
break;
|
||||
case "OOMKilled":
|
||||
reason =
|
||||
"[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.";
|
||||
errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
const failureInfo = {
|
||||
exitCode,
|
||||
reason,
|
||||
logs,
|
||||
overrideCompletion: onlyOverrideExistingError,
|
||||
errorCode,
|
||||
} satisfies FailureDetails;
|
||||
|
||||
const app = pod.metadata?.labels?.app;
|
||||
|
||||
switch (app) {
|
||||
case "task-index":
|
||||
const deploymentId = pod.metadata?.labels?.deployment;
|
||||
|
||||
if (!deploymentId) {
|
||||
this.#logger.error("Index is missing ID", { pod });
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.opts.onIndexFailure) {
|
||||
await this.opts.onIndexFailure(deploymentId, failureInfo);
|
||||
}
|
||||
break;
|
||||
case "task-run":
|
||||
const runId = pod.metadata?.labels?.run;
|
||||
|
||||
if (!runId) {
|
||||
this.#logger.error("Run is missing ID", { pod });
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.opts.onRunFailure) {
|
||||
await this.opts.onRunFailure(runId, failureInfo);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.#logger.error("Pod has invalid app label", { pod });
|
||||
return;
|
||||
}
|
||||
|
||||
await this.#deletePod(podName);
|
||||
}
|
||||
|
||||
async #getLogTail(podName: string) {
|
||||
try {
|
||||
const logs = await this.#k8sClient.core.readNamespacedPodLog(
|
||||
podName,
|
||||
this.namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
1024, // limitBytes
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
20 // tailLines
|
||||
);
|
||||
|
||||
const responseBody = logs.body ?? "";
|
||||
|
||||
if (responseBody.startsWith("unable to retrieve container logs")) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Type is wrong, body may be undefined
|
||||
return responseBody;
|
||||
} catch (error) {
|
||||
this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
#getPodStatusSummary(status: k8s.V1PodStatus) {
|
||||
return {
|
||||
reason: status.reason,
|
||||
message: status.message,
|
||||
};
|
||||
}
|
||||
|
||||
#getContainerStateSummary(state: k8s.V1ContainerState) {
|
||||
return {
|
||||
reason: state.terminated?.reason,
|
||||
exitCode: state.terminated?.exitCode,
|
||||
message: state.terminated?.message,
|
||||
};
|
||||
}
|
||||
|
||||
#createK8sClient() {
|
||||
const kubeConfig = new k8s.KubeConfig();
|
||||
|
||||
if (this.opts.runtimeEnv === "local") {
|
||||
kubeConfig.loadFromDefault();
|
||||
} else if (this.opts.runtimeEnv === "kubernetes") {
|
||||
kubeConfig.loadFromCluster();
|
||||
} else {
|
||||
throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
|
||||
}
|
||||
|
||||
return {
|
||||
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
|
||||
kubeConfig: kubeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
#isRecord(candidate: unknown): candidate is Record<string, unknown> {
|
||||
if (typeof candidate !== "object" || candidate === null) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
#logK8sError(err: unknown, debugOnly = false) {
|
||||
if (debugOnly) {
|
||||
this.#logger.debug("K8s API Error", err);
|
||||
} else {
|
||||
this.#logger.error("K8s API Error", err);
|
||||
}
|
||||
}
|
||||
|
||||
#handleK8sError(err: unknown) {
|
||||
if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
|
||||
this.#logK8sError(err);
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logK8sError(err, true);
|
||||
|
||||
if (typeof err.body.message === "string") {
|
||||
this.#logK8sError({ message: err.body.message });
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logK8sError({ body: err.body });
|
||||
}
|
||||
|
||||
#printStats(includeMoreDetails = false) {
|
||||
this.#logger.log("Stats:", {
|
||||
cacheSize: this.#taskInformer.list().length,
|
||||
totalProcessed: this.#processedPods.size,
|
||||
...(includeMoreDetails && {
|
||||
processedPods: this.#processedPods,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
async #deletePod(name: string) {
|
||||
this.#logger.debug("Deleting pod:", name);
|
||||
|
||||
await this.#k8sClient.core
|
||||
.deleteNamespacedPod(name, this.namespace)
|
||||
.catch(this.#handleK8sError.bind(this));
|
||||
}
|
||||
|
||||
async start() {
|
||||
this.#enabled = true;
|
||||
|
||||
const interval = setInterval(() => {
|
||||
if (!this.#enabled) {
|
||||
clearInterval(interval);
|
||||
return;
|
||||
}
|
||||
|
||||
this.#printStats();
|
||||
}, 300_000);
|
||||
|
||||
await this.#taskInformer.start();
|
||||
|
||||
// this.#launchTests();
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (!this.#enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.#enabled = false;
|
||||
this.#logger.log("Shutting down..");
|
||||
|
||||
await this.#taskInformer.stop();
|
||||
|
||||
this.#printStats(true);
|
||||
}
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
import * as k8s from "@kubernetes/client-node";
|
||||
import { SimpleLogger } from "@trigger.dev/core/v3/apps";
|
||||
|
||||
type UptimeHeartbeatOptions = {
|
||||
runtimeEnv: "local" | "kubernetes";
|
||||
pingUrl: string;
|
||||
namespace?: string;
|
||||
intervalInSeconds?: number;
|
||||
maxPendingRuns?: number;
|
||||
maxPendingIndeces?: number;
|
||||
maxPendingErrors?: number;
|
||||
leadingEdge?: boolean;
|
||||
};
|
||||
|
||||
export class UptimeHeartbeat {
|
||||
private enabled = false;
|
||||
private namespace: string;
|
||||
|
||||
private intervalInSeconds: number;
|
||||
private maxPendingRuns: number;
|
||||
private maxPendingIndeces: number;
|
||||
private maxPendingErrors: number;
|
||||
|
||||
private leadingEdge = true;
|
||||
|
||||
private logger = new SimpleLogger("[UptimeHeartbeat]");
|
||||
private k8sClient: {
|
||||
core: k8s.CoreV1Api;
|
||||
kubeConfig: k8s.KubeConfig;
|
||||
};
|
||||
|
||||
constructor(private opts: UptimeHeartbeatOptions) {
|
||||
this.namespace = opts.namespace ?? "default";
|
||||
|
||||
this.intervalInSeconds = opts.intervalInSeconds ?? 60;
|
||||
this.maxPendingRuns = opts.maxPendingRuns ?? 25;
|
||||
this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
|
||||
this.maxPendingErrors = opts.maxPendingErrors ?? 10;
|
||||
|
||||
this.k8sClient = this.#createK8sClient();
|
||||
}
|
||||
|
||||
#createK8sClient() {
|
||||
const kubeConfig = new k8s.KubeConfig();
|
||||
|
||||
if (this.opts.runtimeEnv === "local") {
|
||||
kubeConfig.loadFromDefault();
|
||||
} else if (this.opts.runtimeEnv === "kubernetes") {
|
||||
kubeConfig.loadFromCluster();
|
||||
} else {
|
||||
throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
|
||||
}
|
||||
|
||||
return {
|
||||
core: kubeConfig.makeApiClient(k8s.CoreV1Api),
|
||||
kubeConfig: kubeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
#isRecord(candidate: unknown): candidate is Record<string, unknown> {
|
||||
if (typeof candidate !== "object" || candidate === null) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
#logK8sError(err: unknown, debugOnly = false) {
|
||||
if (debugOnly) {
|
||||
this.logger.debug("K8s API Error", err);
|
||||
} else {
|
||||
this.logger.error("K8s API Error", err);
|
||||
}
|
||||
}
|
||||
|
||||
#handleK8sError(err: unknown) {
|
||||
if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
|
||||
this.#logK8sError(err);
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logK8sError(err, true);
|
||||
|
||||
if (typeof err.body.message === "string") {
|
||||
this.#logK8sError({ message: err.body.message });
|
||||
return;
|
||||
}
|
||||
|
||||
this.#logK8sError({ body: err.body });
|
||||
}
|
||||
|
||||
async #getPods(opts: {
|
||||
namespace: string;
|
||||
fieldSelector?: string;
|
||||
labelSelector?: string;
|
||||
}): Promise<Array<k8s.V1Pod> | undefined> {
|
||||
const listReturn = await this.k8sClient.core
|
||||
.listNamespacedPod(
|
||||
opts.namespace,
|
||||
undefined, // pretty
|
||||
undefined, // allowWatchBookmarks
|
||||
undefined, // _continue
|
||||
opts.fieldSelector,
|
||||
opts.labelSelector,
|
||||
this.maxPendingRuns * 2, // limit
|
||||
undefined, // resourceVersion
|
||||
undefined, // resourceVersionMatch
|
||||
undefined, // sendInitialEvents
|
||||
this.intervalInSeconds, // timeoutSeconds,
|
||||
undefined // watch
|
||||
)
|
||||
.catch(this.#handleK8sError.bind(this));
|
||||
|
||||
return listReturn?.body.items;
|
||||
}
|
||||
|
||||
async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
|
||||
return await this.#getPods({
|
||||
namespace: this.namespace,
|
||||
fieldSelector: "status.phase=Pending",
|
||||
labelSelector: "app=task-index",
|
||||
});
|
||||
}
|
||||
|
||||
async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
|
||||
return await this.#getPods({
|
||||
namespace: this.namespace,
|
||||
fieldSelector: "status.phase=Pending",
|
||||
labelSelector: "app=task-run",
|
||||
});
|
||||
}
|
||||
|
||||
#countPods(pods: Array<k8s.V1Pod>): number {
|
||||
return pods.length;
|
||||
}
|
||||
|
||||
#filterPendingPods(
|
||||
pods: Array<k8s.V1Pod>,
|
||||
waitingReason: "CreateContainerError" | "RunContainerError"
|
||||
): Array<k8s.V1Pod> {
|
||||
return pods.filter((pod) => {
|
||||
const containerStatus = pod.status?.containerStatuses?.[0];
|
||||
return containerStatus?.state?.waiting?.reason === waitingReason;
|
||||
});
|
||||
}
|
||||
|
||||
async #sendPing() {
|
||||
this.logger.log("Sending ping");
|
||||
|
||||
const start = Date.now();
|
||||
const controller = new AbortController();
|
||||
|
||||
const timeoutMs = (this.intervalInSeconds * 1000) / 2;
|
||||
|
||||
const fetchTimeout = setTimeout(() => {
|
||||
controller.abort();
|
||||
}, timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(this.opts.pingUrl, {
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
this.logger.error("Failed to send ping, response not OK", {
|
||||
status: response.status,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const elapsedMs = Date.now() - start;
|
||||
this.logger.log("Ping sent", { elapsedMs });
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") {
|
||||
this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.error("Failed to send ping", error);
|
||||
} finally {
|
||||
clearTimeout(fetchTimeout);
|
||||
}
|
||||
}
|
||||
|
||||
async #heartbeat() {
|
||||
this.logger.log("Performing heartbeat");
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
const pendingTasks = await this.#getPendingTasks();
|
||||
|
||||
if (!pendingTasks) {
|
||||
this.logger.error("Failed to get pending tasks");
|
||||
return;
|
||||
}
|
||||
|
||||
const totalPendingTasks = this.#countPods(pendingTasks);
|
||||
|
||||
const pendingIndeces = await this.#getPendingIndeces();
|
||||
|
||||
if (!pendingIndeces) {
|
||||
this.logger.error("Failed to get pending indeces");
|
||||
return;
|
||||
}
|
||||
|
||||
const totalPendingIndeces = this.#countPods(pendingIndeces);
|
||||
|
||||
const elapsedMs = Date.now() - start;
|
||||
|
||||
this.logger.log("Finished heartbeat checks", { elapsedMs });
|
||||
|
||||
if (totalPendingTasks > this.maxPendingRuns) {
|
||||
this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
|
||||
return;
|
||||
}
|
||||
|
||||
if (totalPendingIndeces > this.maxPendingIndeces) {
|
||||
this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
|
||||
return;
|
||||
}
|
||||
|
||||
const totalCreateContainerErrors = this.#countPods(
|
||||
this.#filterPendingPods(pendingTasks, "CreateContainerError")
|
||||
);
|
||||
const totalRunContainerErrors = this.#countPods(
|
||||
this.#filterPendingPods(pendingTasks, "RunContainerError")
|
||||
);
|
||||
|
||||
if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
|
||||
this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
|
||||
totalRunContainerErrors,
|
||||
totalCreateContainerErrors,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await this.#sendPing();
|
||||
|
||||
this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
|
||||
}
|
||||
|
||||
async start() {
|
||||
this.enabled = true;
|
||||
this.logger.log("Starting");
|
||||
|
||||
if (this.leadingEdge) {
|
||||
await this.#heartbeat();
|
||||
}
|
||||
|
||||
const heartbeat = setInterval(async () => {
|
||||
if (!this.enabled) {
|
||||
clearInterval(heartbeat);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#heartbeat();
|
||||
} catch (error) {
|
||||
this.logger.error("Error while heartbeating", error);
|
||||
}
|
||||
}, this.intervalInSeconds * 1000);
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (!this.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.enabled = false;
|
||||
this.logger.log("Shutting down..");
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2020",
|
||||
"module": "commonjs",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"paths": {
|
||||
"@trigger.dev/core": ["../../packages/core/src"],
|
||||
"@trigger.dev/core/*": ["../../packages/core/src/*"],
|
||||
"@trigger.dev/core/v3": ["../../packages/core/src/v3"],
|
||||
"@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,5 @@
|
||||
import type { IncomingMessage, RequestListener } from "node:http";
|
||||
|
||||
export const getTextBody = (req: IncomingMessage) =>
|
||||
new Promise<string>((resolve) => {
|
||||
let body = "";
|
||||
req.on("readable", () => {
|
||||
const chunk = req.read();
|
||||
if (chunk) {
|
||||
body += chunk;
|
||||
}
|
||||
});
|
||||
req.on("end", () => {
|
||||
resolve(body);
|
||||
});
|
||||
});
|
||||
|
||||
export async function getJsonBody(req: IncomingMessage): Promise<any> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let body = "";
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
export * from "./backoff.js";
|
||||
export * from "./logger.js";
|
||||
export * from "./process.js";
|
||||
export * from "./http.js";
|
||||
export * from "./provider.js";
|
||||
export * from "./isExecaChildProcess.js";
|
||||
export * from "./exec.js";
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
// @ts-ignore
|
||||
import type { ExecaChildProcess } from "execa";
|
||||
|
||||
export function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
|
||||
return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
export class SimpleLogger {
|
||||
#debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? "");
|
||||
|
||||
constructor(private prefix?: string) {}
|
||||
|
||||
log<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
|
||||
console.log(...this.#getPrefixedArgs(arg0, ...argN));
|
||||
|
||||
return arg0;
|
||||
}
|
||||
|
||||
debug<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
|
||||
if (!this.#debugEnabled) {
|
||||
return arg0;
|
||||
}
|
||||
|
||||
console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN));
|
||||
|
||||
return arg0;
|
||||
}
|
||||
|
||||
error<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
|
||||
console.error(...this.#getPrefixedArgs(arg0, ...argN));
|
||||
|
||||
return arg0;
|
||||
}
|
||||
|
||||
#getPrefixedArgs(...args: any[]) {
|
||||
if (!this.prefix) {
|
||||
return args;
|
||||
}
|
||||
|
||||
return [this.prefix, ...args];
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
/** This was used by the old build system in case of indexing failures */
|
||||
export const EXIT_CODE_ALREADY_HANDLED = 111;
|
||||
/** This means what it says and is only set once we have completed the attempt */
|
||||
export const EXIT_CODE_CHILD_NONZERO = 112;
|
||||
@@ -1,382 +0,0 @@
|
||||
import { createServer } from "node:http";
|
||||
import { getRandomPortNumber, HttpReply, getTextBody } from "./http.js";
|
||||
import { SimpleLogger } from "./logger.js";
|
||||
import { isExecaChildProcess } from "./isExecaChildProcess.js";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { EXIT_CODE_ALREADY_HANDLED } from "./process.js";
|
||||
import type { EnvironmentType } from "../schemas/schemas.js";
|
||||
import type { MachinePreset } from "../schemas/common.js";
|
||||
import {
|
||||
ProviderToPlatformMessages,
|
||||
PlatformToProviderMessages,
|
||||
ClientToSharedQueueMessages,
|
||||
SharedQueueToClientMessages,
|
||||
clientWebsocketMessages,
|
||||
} from "../schemas/messages.js";
|
||||
import { ZodMessageSender } from "../zodMessageHandler.js";
|
||||
import { ZodSocketConnection } from "../zodSocket.js";
|
||||
|
||||
const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
||||
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
||||
|
||||
const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
|
||||
const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
|
||||
const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
|
||||
const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
|
||||
|
||||
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
|
||||
|
||||
export interface TaskOperationsIndexOptions {
|
||||
shortCode: string;
|
||||
imageRef: string;
|
||||
apiKey: string;
|
||||
apiUrl: string;
|
||||
// identifiers
|
||||
envId: string;
|
||||
envType: EnvironmentType;
|
||||
orgId: string;
|
||||
projectId: string;
|
||||
deploymentId: string;
|
||||
}
|
||||
|
||||
export interface TaskOperationsCreateOptions {
|
||||
image: string;
|
||||
machine: MachinePreset;
|
||||
version: string;
|
||||
nextAttemptNumber?: number;
|
||||
// identifiers
|
||||
envId: string;
|
||||
envType: EnvironmentType;
|
||||
orgId: string;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
dequeuedAt?: number;
|
||||
}
|
||||
|
||||
export interface TaskOperationsRestoreOptions {
|
||||
imageRef: string;
|
||||
checkpointRef: string;
|
||||
machine: MachinePreset;
|
||||
attemptNumber?: number;
|
||||
// identifiers
|
||||
envId: string;
|
||||
envType: EnvironmentType;
|
||||
orgId: string;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
checkpointId: string;
|
||||
}
|
||||
|
||||
export interface TaskOperationsPrePullDeploymentOptions {
|
||||
shortCode: string;
|
||||
imageRef: string;
|
||||
// identifiers
|
||||
envId: string;
|
||||
envType: EnvironmentType;
|
||||
orgId: string;
|
||||
projectId: string;
|
||||
deploymentId: string;
|
||||
}
|
||||
|
||||
export interface TaskOperations {
|
||||
init: () => Promise<any>;
|
||||
|
||||
// CRUD
|
||||
index: (opts: TaskOperationsIndexOptions) => Promise<any>;
|
||||
create: (opts: TaskOperationsCreateOptions) => Promise<any>;
|
||||
restore: (opts: TaskOperationsRestoreOptions) => Promise<any>;
|
||||
|
||||
// unimplemented
|
||||
delete?: (...args: any[]) => Promise<any>;
|
||||
get?: (...args: any[]) => Promise<any>;
|
||||
|
||||
prePullDeployment?: (opts: TaskOperationsPrePullDeploymentOptions) => Promise<any>;
|
||||
}
|
||||
|
||||
type ProviderShellOptions = {
|
||||
tasks: TaskOperations;
|
||||
type: "docker" | "kubernetes";
|
||||
host?: string;
|
||||
port?: number;
|
||||
};
|
||||
|
||||
interface Provider {
|
||||
tasks: TaskOperations;
|
||||
}
|
||||
|
||||
export class ProviderShell implements Provider {
|
||||
tasks: TaskOperations;
|
||||
|
||||
#httpPort: number;
|
||||
#httpServer: ReturnType<typeof createServer>;
|
||||
platformSocket: ZodSocketConnection<
|
||||
typeof ProviderToPlatformMessages,
|
||||
typeof PlatformToProviderMessages
|
||||
>;
|
||||
|
||||
constructor(private options: ProviderShellOptions) {
|
||||
this.tasks = options.tasks;
|
||||
this.#httpPort = options.port ?? HTTP_SERVER_PORT;
|
||||
this.#httpServer = this.#createHttpServer();
|
||||
this.platformSocket = this.#createPlatformSocket();
|
||||
this.#createSharedQueueSocket();
|
||||
}
|
||||
|
||||
#createSharedQueueSocket() {
|
||||
const sharedQueueConnection = new ZodSocketConnection({
|
||||
namespace: "shared-queue",
|
||||
host: PLATFORM_HOST,
|
||||
port: Number(PLATFORM_WS_PORT),
|
||||
secure: SECURE_CONNECTION,
|
||||
clientMessages: ClientToSharedQueueMessages,
|
||||
serverMessages: SharedQueueToClientMessages,
|
||||
authToken: PLATFORM_SECRET,
|
||||
handlers: {
|
||||
SERVER_READY: async (message) => {
|
||||
// TODO: create new schema without worker requirement
|
||||
await sender.send("READY_FOR_TASKS", {
|
||||
backgroundWorkerId: "placeholder",
|
||||
});
|
||||
},
|
||||
BACKGROUND_WORKER_MESSAGE: async (message) => {
|
||||
if (message.data.type === "SCHEDULE_ATTEMPT") {
|
||||
try {
|
||||
await this.tasks.create({
|
||||
image: message.data.image,
|
||||
machine: message.data.machine,
|
||||
version: message.data.version,
|
||||
nextAttemptNumber: message.data.nextAttemptNumber,
|
||||
// identifiers
|
||||
envId: message.data.envId,
|
||||
envType: message.data.envType,
|
||||
orgId: message.data.orgId,
|
||||
projectId: message.data.projectId,
|
||||
runId: message.data.runId,
|
||||
dequeuedAt: message.data.dequeuedAt,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("create failed", error);
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const sender = new ZodMessageSender({
|
||||
schema: clientWebsocketMessages,
|
||||
sender: async (message) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const { type, ...payload } = message;
|
||||
sharedQueueConnection.socket.emit(type, payload as any);
|
||||
resolve();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
return sharedQueueConnection;
|
||||
}
|
||||
|
||||
#createPlatformSocket() {
|
||||
const platformConnection = new ZodSocketConnection({
|
||||
namespace: "provider",
|
||||
host: PLATFORM_HOST,
|
||||
port: Number(PLATFORM_WS_PORT),
|
||||
secure: SECURE_CONNECTION,
|
||||
clientMessages: ProviderToPlatformMessages,
|
||||
serverMessages: PlatformToProviderMessages,
|
||||
authToken: PLATFORM_SECRET,
|
||||
extraHeaders: {
|
||||
"x-trigger-provider-type": this.options.type,
|
||||
},
|
||||
handlers: {
|
||||
INDEX: async (message) => {
|
||||
try {
|
||||
await this.tasks.index({
|
||||
shortCode: message.shortCode,
|
||||
imageRef: message.imageTag,
|
||||
apiKey: message.apiKey,
|
||||
apiUrl: message.apiUrl,
|
||||
// identifiers
|
||||
envId: message.envId,
|
||||
envType: message.envType,
|
||||
orgId: message.orgId,
|
||||
projectId: message.projectId,
|
||||
deploymentId: message.deploymentId,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isExecaChildProcess(error)) {
|
||||
logger.error("Index failed", {
|
||||
socketMessage: message,
|
||||
exitCode: error.exitCode,
|
||||
escapedCommand: error.escapedCommand,
|
||||
stdout: error.stdout,
|
||||
stderr: error.stderr,
|
||||
});
|
||||
|
||||
if (error.exitCode === EXIT_CODE_ALREADY_HANDLED) {
|
||||
logger.error("Index failure already reported by the worker", {
|
||||
socketMessage: message,
|
||||
});
|
||||
|
||||
// Add a brief delay to avoid messaging race conditions
|
||||
await setTimeout(2000);
|
||||
}
|
||||
|
||||
function normalizeStderr(stderr: string) {
|
||||
return stderr
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: {
|
||||
name: "Index error",
|
||||
message: `Crashed with exit code ${error.exitCode}`,
|
||||
stderr: normalizeStderr(error.stderr),
|
||||
},
|
||||
};
|
||||
} else {
|
||||
logger.error("Index failed", error);
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
success: false,
|
||||
error: {
|
||||
name: "Provider error",
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
},
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: {
|
||||
name: "Provider error",
|
||||
message: "Unknown error",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
},
|
||||
RESTORE: async (message) => {
|
||||
if (message.type.toLowerCase() !== this.options.type.toLowerCase()) {
|
||||
logger.error(
|
||||
`restore failed: ${this.options.type} provider can't restore ${message.type} checkpoints`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.tasks.restore({
|
||||
checkpointRef: message.location,
|
||||
machine: message.machine,
|
||||
imageRef: message.imageRef,
|
||||
attemptNumber: message.attemptNumber,
|
||||
// identifiers
|
||||
envId: message.envId,
|
||||
envType: message.envType,
|
||||
orgId: message.orgId,
|
||||
projectId: message.projectId,
|
||||
runId: message.runId,
|
||||
checkpointId: message.checkpointId,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("restore failed", error);
|
||||
}
|
||||
},
|
||||
PRE_PULL_DEPLOYMENT: async (message) => {
|
||||
if (!this.tasks.prePullDeployment) {
|
||||
logger.debug("prePullDeployment not implemented", message);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.tasks.prePullDeployment({
|
||||
shortCode: message.shortCode,
|
||||
imageRef: message.imageRef,
|
||||
// identifiers
|
||||
envId: message.envId,
|
||||
envType: message.envType,
|
||||
orgId: message.orgId,
|
||||
projectId: message.projectId,
|
||||
deploymentId: message.deploymentId,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("prePullDeployment failed", error);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return platformConnection;
|
||||
}
|
||||
|
||||
#createHttpServer() {
|
||||
const httpServer = createServer(async (req, res) => {
|
||||
logger.log(`[${req.method}]`, req.url);
|
||||
|
||||
const reply = new HttpReply(res);
|
||||
|
||||
try {
|
||||
const url = new URL(req.url ?? "", `http://${req.headers.host}`);
|
||||
|
||||
switch (url.pathname) {
|
||||
case "/health": {
|
||||
return reply.text("ok");
|
||||
}
|
||||
case "/whoami": {
|
||||
return reply.text(`${MACHINE_NAME}`);
|
||||
}
|
||||
case "/close": {
|
||||
this.platformSocket.close();
|
||||
return reply.text("platform socket closed");
|
||||
}
|
||||
case "/delete": {
|
||||
const body = await getTextBody(req);
|
||||
|
||||
if (this.tasks.delete) {
|
||||
await this.tasks.delete({ runId: body });
|
||||
return reply.text(`sent delete request: ${body}`);
|
||||
} else {
|
||||
return reply.text("delete not implemented", 501);
|
||||
}
|
||||
}
|
||||
default: {
|
||||
return reply.empty(404);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("HTTP server error", { error });
|
||||
reply.empty(500);
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
httpServer.on("clientError", (err, socket) => {
|
||||
socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
|
||||
});
|
||||
|
||||
httpServer.on("listening", () => {
|
||||
logger.log("server listening on port", this.#httpPort);
|
||||
});
|
||||
|
||||
return httpServer;
|
||||
}
|
||||
|
||||
async listen() {
|
||||
this.#httpServer.listen(this.#httpPort, this.options.host ?? "0.0.0.0");
|
||||
await this.tasks.init();
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { isExecaChildProcess } from "../apps/isExecaChildProcess.js";
|
||||
|
||||
export type CheckpointTestResult =
|
||||
| {
|
||||
ok: true;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
message: string;
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
export async function testDockerCheckpoint(): Promise<CheckpointTestResult> {
|
||||
const { $ } = await import("execa");
|
||||
|
||||
try {
|
||||
// Create a dummy container
|
||||
const container =
|
||||
await $`docker run -d --rm --name init-dummy-${randomUUID()} docker.io/library/busybox sleep 10`;
|
||||
|
||||
// Checkpoint it
|
||||
await $`docker checkpoint create ${container} init-check`;
|
||||
} catch (error) {
|
||||
if (!isExecaChildProcess(error)) {
|
||||
return {
|
||||
ok: false,
|
||||
message: "No checkpoint support: Unknown error.",
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
if (error.stderr.includes("criu")) {
|
||||
if (error.stderr.includes("executable file not found")) {
|
||||
return {
|
||||
ok: false,
|
||||
message: "No checkpoint support: Missing CRIU binary.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
message: "No checkpoint support: Unknown CRIU error.",
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
if (error.stderr.includes("experimental features enabled")) {
|
||||
return {
|
||||
ok: false,
|
||||
message: "No checkpoint support: Please enable docker experimental features.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
message: "No checkpoint support: Unknown execa error.",
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
export * from "./checkpointClient.js";
|
||||
export * from "./checkpointTest.js";
|
||||
export * from "./httpServer.js";
|
||||
export * from "./singleton.js";
|
||||
export * from "./shutdownManager.js";
|
||||
|
||||
Generated
+1
-736
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user