diff --git a/.changeset/spicy-terms-bow.md b/.changeset/spicy-terms-bow.md new file mode 100644 index 000000000..bdb478fa1 --- /dev/null +++ b/.changeset/spicy-terms-bow.md @@ -0,0 +1,9 @@ +--- +"@trigger.dev/core-apps": patch +"trigger.dev": patch +--- + +- Fix init command SDK pinning +- Show --api-url / -a flag where needed +- CLI now also respects `TRIGGER_TELEMETRY_DISABLED` +- Dedicated docker checkpoint test function \ No newline at end of file diff --git a/.env.example b/.env.example index bf5be8f95..26152641b 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,8 @@ DEV_OTEL_BATCH_PROCESSING_ENABLED="0" # OPTIONAL VARIABLES # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected. # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com" +# Accounts with these emails will get global admin rights. This grants access to the admin UI. +# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com" # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication. # AUTH_GITHUB_CLIENT_ID= # AUTH_GITHUB_CLIENT_SECRET= diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index e8d943724..ef98fc339 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -39,11 +39,25 @@ jobs: exit 1 fi echo "::set-output name=version::${IMAGE_TAG}" + - name: 🔢 Get the commit hash id: get_commit run: | echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7) + - name: 📛 Set the tags + id: set_tags + run: | + ref_without_tag=ghcr.io/triggerdotdev/trigger.dev + image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }} + + # if it's a versioned tag, also tag it as latest + if [[ "${{ github.ref_name }}" == v.docker.* ]]; then + image_tags=$image_tags,$ref_without_tag:latest + fi + + echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT" + - name: 🐙 Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -56,6 +70,5 @@ jobs: with: file: ./docker/Dockerfile platforms: linux/amd64,linux/arm64 - tags: | - ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }} + tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }} push: true diff --git a/.github/workflows/publish-infra.yml b/.github/workflows/publish-infra.yml index ec67436a6..c6d858b7d 100644 --- a/.github/workflows/publish-infra.yml +++ b/.github/workflows/publish-infra.yml @@ -1,6 +1,7 @@ name: "🚢 Publish Infra Images" on: + workflow_call: push: tags: - "infra-dev-*" @@ -29,9 +30,6 @@ permissions: packages: write contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - env: AWS_REGION: us-east-1 @@ -39,7 +37,7 @@ jobs: build: strategy: matrix: - package: [coordinator, kubernetes-provider] + package: [coordinator, docker-provider, kubernetes-provider] runs-on: buildjet-16vcpu-ubuntu-2204 env: DOCKER_BUILDKIT: "1" @@ -48,20 +46,40 @@ jobs: - name: Generate image reference id: prep - # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-* run: | - env=$(echo ${{ github.ref_name }} | cut -d- -f2) - sha=${GITHUB_SHA::7} - ts=$(date +%s) + # set image repo if [[ "${{ matrix.package }}" == *-provider ]]; then - provider_type=$(echo ${{ matrix.package }} | cut -d- -f1) + provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1) repository=provider/${provider_type} else - repository=${{ matrix.package }} + repository="${{ matrix.package }}" fi - echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT" echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT" + # set image tag + if [[ "${{ github.ref_type }}" == "tag" ]]; then + if [[ "${{ github.ref_name }}" == infra-*-* ]]; then + env=$(echo ${{ github.ref_name }} | cut -d- -f2) + sha=$(echo ${{ github.sha }} | head -c7) + ts=$(date +%s) + image_tag=${env}-${sha}-${ts} + elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then + version="${GITHUB_REF_NAME#v.docker.}" + image_tag="v${version}" + elif [[ "${{ github.ref_name }}" == build-* ]]; then + image_tag="${GITHUB_REF_NAME#build-}" + else + echo "Invalid tag: ${{ github.ref_name }}" + exit 1 + fi + elif [[ "${{ github.ref_name }}" == "main" ]]; then + image_tag="main" + else + echo "Invalid reference: ${{ github.ref }}" + exit 1 + fi + echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -92,3 +110,12 @@ jobs: REGISTRY: ghcr.io/triggerdotdev REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }} IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }} + + - name: 🐙 Push 'latest' to GitHub Container Registry + if: startsWith(github.ref_name, 'v.docker.') + run: | + docker tag infra_image $REGISTRY/$REPOSITORY:latest + docker push $REGISTRY/$REPOSITORY:latest + env: + REGISTRY: ghcr.io/triggerdotdev + REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 73d15343b..fdef9ab4a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -57,3 +57,8 @@ jobs: needs: [typecheck, units] uses: ./.github/workflows/publish-docker.yml secrets: inherit + + publish-infra: + needs: [typecheck, units] + uses: ./.github/workflows/publish-infra.yml + secrets: inherit diff --git a/apps/coordinator/Containerfile b/apps/coordinator/Containerfile index cd301811b..c42b663d2 100644 --- a/apps/coordinator/Containerfile +++ b/apps/coordinator/Containerfile @@ -1,19 +1,19 @@ # syntax=docker/dockerfile:labs -FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18 +FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20 WORKDIR /app -FROM node-18 AS pruner +FROM node-20 AS pruner COPY --chown=node:node . . RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + -FROM node-18 AS base +FROM node-20 AS base RUN apt-get update \ - && apt-get install -y buildah ca-certificates dumb-init \ + && apt-get install -y buildah ca-certificates dumb-init docker.io \ && rm -rf /var/lib/apt/lists/* COPY --chown=node:node .gitignore .gitignore diff --git a/apps/coordinator/src/index.ts b/apps/coordinator/src/index.ts index 19ad2b909..30b179e7f 100644 --- a/apps/coordinator/src/index.ts +++ b/apps/coordinator/src/index.ts @@ -12,7 +12,7 @@ import { } from "@trigger.dev/core/v3"; import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace"; import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket"; -import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps"; +import { HttpReply, getTextBody, SimpleLogger, testDockerCheckpoint } from "@trigger.dev/core-apps"; import { ExponentialBackoff } from "./backoff"; import { collectDefaultMetrics, register, Gauge } from "prom-client"; @@ -72,7 +72,10 @@ type CheckpointAndPushOptions = { type CheckpointAndPushResult = | { success: true; checkpoint: CheckpointData } - | { success: false; reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" }; + | { + success: false; + reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING"; + }; type CheckpointData = { location: string; @@ -125,65 +128,53 @@ class Checkpointer { constructor(private opts = { forceSimulate: false }) {} - async initialize(): Promise { + async init(): Promise { if (this.#initialized) { - return this.#getInitializeReturn(); + return this.#getInitReturn(this.#canCheckpoint); } this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`); if (this.#dockerMode) { - try { - await $`criu --version`; - } catch (error) { - this.#logger.error("No checkpoint support: Missing CRIU binary"); - this.#logger.error("Will simulate instead"); - this.#canCheckpoint = false; - this.#initialized = true; + const testCheckpoint = await testDockerCheckpoint(); - return this.#getInitializeReturn(); + if (testCheckpoint.ok) { + return this.#getInitReturn(true); } - try { - await $`docker checkpoint`; - } catch (error) { - this.#logger.error( - "No checkpoint support: Docker needs to have experimental features enabled" - ); - this.#logger.error("Will simulate instead"); - this.#canCheckpoint = false; - this.#initialized = true; - - return this.#getInitializeReturn(); - } + this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? ""); + return this.#getInitReturn(false); } else { try { await $`buildah login --get-login ${REGISTRY_HOST}`; } catch (error) { this.#logger.error(`No checkpoint support: Not logged in to registry ${REGISTRY_HOST}`); - this.#canCheckpoint = false; - this.#initialized = true; - - return this.#getInitializeReturn(); + return this.#getInitReturn(false); } } - this.#logger.log( - `Full checkpoint support${ - this.#dockerMode && this.opts.forceSimulate ? " with forced simulation enabled." : "!" - }` - ); - - this.#initialized = true; - this.#canCheckpoint = true; - - return this.#getInitializeReturn(); + return this.#getInitReturn(true); } - #getInitializeReturn(): CheckpointerInitializeReturn { + #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn { + this.#initialized = true; + this.#canCheckpoint = canCheckpoint; + + if (canCheckpoint) { + this.#logger.log("Full checkpoint support!"); + } + + const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate); + + if (willSimulate) { + this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", { + forceSimulate: this.opts.forceSimulate, + }); + } + return { - canCheckpoint: this.#canCheckpoint, - willSimulate: this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate), + canCheckpoint, + willSimulate, }; } @@ -327,6 +318,11 @@ class Checkpointer { return result; } + if (result.reason === "SKIP_RETRYING") { + this.#logger.log("Skipping retrying", { runId }); + return result; + } + continue; } catch (error) { this.#logger.error("Checkpoint error", { @@ -355,7 +351,7 @@ class Checkpointer { projectRef, deploymentVersion, }: CheckpointAndPushOptions): Promise { - await this.initialize(); + await this.init(); const options = { runId, @@ -473,7 +469,8 @@ class Checkpointer { // Create checkpoint (CRI) if (!this.#canCheckpoint) { - throw new Error("No checkpoint support in kubernetes mode."); + this.#logger.error("No checkpoint support in kubernetes mode."); + return { success: false, reason: "SKIP_RETRYING" }; } const containerId = this.#logger.debug( @@ -484,7 +481,8 @@ class Checkpointer { ); if (!containerId.stdout) { - throw new Error("could not find container id"); + this.#logger.error("could not find container id", { options, containterName }); + return { success: false, reason: "SKIP_RETRYING" }; } const start = performance.now(); @@ -617,7 +615,7 @@ class TaskCoordinator { private host = "0.0.0.0" ) { this.#httpServer = this.#createHttpServer(); - this.#checkpointer.initialize(); + this.#checkpointer.init(); this.#delayThresholdInMs = this.#getDelayThreshold(); if (process.env.DELAY_THRESHOLD_IN_MS) { @@ -1034,7 +1032,7 @@ class TaskCoordinator { return; } - const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize(); + const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); const willCheckpointAndRestore = canCheckpoint || willSimulate; @@ -1131,7 +1129,7 @@ class TaskCoordinator { return; } - const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize(); + const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); const willCheckpointAndRestore = canCheckpoint || willSimulate; @@ -1185,7 +1183,7 @@ class TaskCoordinator { socket.on("WAIT_FOR_TASK", async (message, callback) => { logger.log("[WAIT_FOR_TASK]", message); - const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize(); + const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); const willCheckpointAndRestore = canCheckpoint || willSimulate; @@ -1227,7 +1225,7 @@ class TaskCoordinator { socket.on("WAIT_FOR_BATCH", async (message, callback) => { logger.log("[WAIT_FOR_BATCH]", message); - const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize(); + const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); const willCheckpointAndRestore = canCheckpoint || willSimulate; diff --git a/apps/docker-provider/Containerfile b/apps/docker-provider/Containerfile index f10d20655..bea730bda 100644 --- a/apps/docker-provider/Containerfile +++ b/apps/docker-provider/Containerfile @@ -1,16 +1,47 @@ -# syntax=docker/dockerfile:labs - -FROM node:18-slim AS base - -RUN apt-get update \ - && apt-get install -y dumb-init - -FROM base +FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine WORKDIR /app -COPY --chown=node dist/index.mjs /app/ +FROM node-20-alpine AS pruner + +COPY --chown=node:node . . +RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker +RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + + +FROM node-20-alpine AS base + +RUN apk add --no-cache dumb-init docker + +COPY --chown=node:node .gitignore .gitignore +COPY --from=pruner --chown=node:node /app/out/json/ . +COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml +COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml + +FROM base AS dev-deps +RUN corepack enable +ENV NODE_ENV development + +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile + +FROM base AS builder +RUN corepack enable + +COPY --from=pruner --chown=node:node /app/out/full/ . +COPY --from=dev-deps --chown=node:node /app/ . +COPY --chown=node:node turbo.json turbo.json + +RUN pnpm run -r --filter docker-provider build:bundle + +FROM base AS runner + +RUN corepack enable +ENV NODE_ENV production + +COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs EXPOSE 8000 -ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.mjs" ] +USER node + +CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts index a5e588956..d339b5a93 100644 --- a/apps/docker-provider/src/index.ts +++ b/apps/docker-provider/src/index.ts @@ -6,6 +6,8 @@ import { TaskOperationsRestoreOptions, TaskOperationsCreateOptions, TaskOperationsIndexOptions, + isExecaChildProcess, + testDockerCheckpoint, } from "@trigger.dev/core-apps"; import { setTimeout } from "node:timers/promises"; import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3"; @@ -23,70 +25,58 @@ const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes( const logger = new SimpleLogger(`[${MACHINE_NAME}]`); -type InitializeReturn = { +type TaskOperationsInitReturn = { canCheckpoint: boolean; willSimulate: boolean; }; -function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited { - return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca; -} - class DockerTaskOperations implements TaskOperations { #initialized = false; #canCheckpoint = false; constructor(private opts = { forceSimulate: false }) {} - async #initialize(): Promise { + async init(): Promise { if (this.#initialized) { - return this.#getInitializeReturn(); + return this.#getInitReturn(this.#canCheckpoint); } logger.log("Initializing task operations"); - if (this.opts.forceSimulate) { - logger.log("Forced simulation enabled. Will simulate regardless of checkpoint support."); + const testCheckpoint = await testDockerCheckpoint(); + + if (testCheckpoint.ok) { + return this.#getInitReturn(true); } - try { - await $`criu --version`; - } catch (error) { - logger.error("No checkpoint support: Missing CRIU binary. Will simulate instead."); - this.#canCheckpoint = false; - this.#initialized = true; - - return this.#getInitializeReturn(); - } - - try { - await $`docker checkpoint`; - } catch (error) { - logger.error("No checkpoint support: Docker needs to have experimental features enabled"); - logger.error("Will simulate instead"); - this.#canCheckpoint = false; - this.#initialized = true; - - return this.#getInitializeReturn(); - } - - logger.log("Full checkpoint support!"); - - this.#initialized = true; - this.#canCheckpoint = true; - - return this.#getInitializeReturn(); + logger.error(testCheckpoint.message, testCheckpoint.error); + return this.#getInitReturn(false); } - #getInitializeReturn(): InitializeReturn { + #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn { + this.#initialized = true; + this.#canCheckpoint = canCheckpoint; + + if (canCheckpoint) { + logger.log("Full checkpoint support!"); + } + + const willSimulate = !canCheckpoint || this.opts.forceSimulate; + + if (willSimulate) { + logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", { + forceSimulate: this.opts.forceSimulate, + }); + } + return { - canCheckpoint: this.#canCheckpoint, - willSimulate: !this.#canCheckpoint || this.opts.forceSimulate, + canCheckpoint, + willSimulate, }; } async index(opts: TaskOperationsIndexOptions) { - await this.#initialize(); + await this.init(); const containerName = this.#getIndexContainerName(opts.shortCode); @@ -129,7 +119,7 @@ class DockerTaskOperations implements TaskOperations { } async create(opts: TaskOperationsCreateOptions) { - await this.#initialize(); + await this.init(); const containerName = this.#getRunContainerName(opts.runId); @@ -165,7 +155,7 @@ class DockerTaskOperations implements TaskOperations { } async restore(opts: TaskOperationsRestoreOptions) { - await this.#initialize(); + await this.init(); const containerName = this.#getRunContainerName(opts.runId); @@ -194,7 +184,7 @@ class DockerTaskOperations implements TaskOperations { } async delete(opts: { runId: string }) { - await this.#initialize(); + await this.init(); const containerName = this.#getRunContainerName(opts.runId); await this.#sendPreStop(containerName); @@ -203,7 +193,7 @@ class DockerTaskOperations implements TaskOperations { } async get(opts: { runId: string }) { - await this.#initialize(); + await this.init(); logger.log("noop: get"); } diff --git a/apps/kubernetes-provider/Containerfile b/apps/kubernetes-provider/Containerfile index f9fad85f4..fb96304c2 100644 --- a/apps/kubernetes-provider/Containerfile +++ b/apps/kubernetes-provider/Containerfile @@ -1,14 +1,14 @@ -FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine +FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine WORKDIR /app -FROM node-18-alpine AS pruner +FROM node-20-alpine AS pruner COPY --chown=node:node . . RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + -FROM node-18-alpine AS base +FROM node-20-alpine AS base RUN apk add --no-cache dumb-init diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index 0f177b59c..67aa0cf8b 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -47,6 +47,10 @@ class KubernetesTaskOperations implements TaskOperations { this.#k8sApi = this.#createK8sApi(); } + async init() { + // noop + } + async index(opts: TaskOperationsIndexOptions) { await this.#createJob( { diff --git a/apps/webapp/app/components/SetupCommands.tsx b/apps/webapp/app/components/SetupCommands.tsx index 8b3bb274c..510e58d77 100644 --- a/apps/webapp/app/components/SetupCommands.tsx +++ b/apps/webapp/app/components/SetupCommands.tsx @@ -8,6 +8,7 @@ import { } from "./primitives/ClientTabs"; import { ClipboardField } from "./primitives/ClipboardField"; import { Paragraph } from "./primitives/Paragraph"; +import { useAppOrigin } from "~/hooks/useAppOrigin"; export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey: string }) { return ( @@ -133,9 +134,38 @@ export function TriggerDevStep({ extra }: { extra?: string }) { // Trigger.dev version 3 setup commands const v3PackageTag = "beta"; +function getApiUrlArg() { + const appOrigin = useAppOrigin(); + + let apiUrl: string | undefined = undefined; + + switch (appOrigin) { + case "https://cloud.trigger.dev": + // don't display the arg, use the CLI default + break; + case "https://test-cloud.trigger.dev": + apiUrl = "https://test-api.trigger.dev"; + break; + case "https://internal.trigger.dev": + apiUrl = "https://internal-api.trigger.dev"; + break; + default: + apiUrl = appOrigin; + break; + } + + return apiUrl ? `-a ${apiUrl}` : undefined; +} + export function InitCommandV3() { const project = useProject(); const projectRef = project.ref; + + const apiUrlArg = getApiUrlArg(); + + const initCommandParts = [`trigger.dev@${v3PackageTag}`, "init", `-p ${projectRef}`, apiUrlArg]; + const initCommand = initCommandParts.filter(Boolean).join(" "); + return ( @@ -148,7 +178,7 @@ export function InitCommandV3() { variant="primary/medium" iconButton className="mb-4" - value={`npx trigger.dev@${v3PackageTag} init -p ${projectRef}`} + value={`npx ${initCommand}`} /> @@ -156,7 +186,7 @@ export function InitCommandV3() { variant="primary/medium" iconButton className="mb-4" - value={`pnpm dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`} + value={`pnpm dlx ${initCommand}`} /> @@ -164,7 +194,7 @@ export function InitCommandV3() { variant="primary/medium" iconButton className="mb-4" - value={`yarn dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`} + value={`yarn dlx ${initCommand}`} /> diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 315f89067..7b72710fc 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -27,15 +27,17 @@ const EnvironmentSchema = z.object({ .string() .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") .optional(), + ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(), REMIX_APP_PORT: z.string().optional(), LOGIN_ORIGIN: z.string().default("http://localhost:3030"), APP_ORIGIN: z.string().default("http://localhost:3030"), APP_ENV: z.string().default(process.env.NODE_ENV), SERVICE_NAME: z.string().default("trigger.dev webapp"), SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"), - POSTHOG_PROJECT_KEY: z.string().optional(), + POSTHOG_PROJECT_KEY: z.string().default("phc_LFH7kJiGhdIlnO22hTAKgHpaKhpM8gkzWAFvHmf5vfS"), TELEMETRY_TRIGGER_API_KEY: z.string().optional(), TELEMETRY_TRIGGER_API_URL: z.string().optional(), + TRIGGER_TELEMETRY_DISABLED: z.string().optional(), HIGHLIGHT_PROJECT_ID: z.string().optional(), AUTH_GITHUB_CLIENT_ID: z.string().optional(), AUTH_GITHUB_CLIENT_SECRET: z.string().optional(), diff --git a/apps/webapp/app/features.server.ts b/apps/webapp/app/features.server.ts index a5f11e79b..8acc49f98 100644 --- a/apps/webapp/app/features.server.ts +++ b/apps/webapp/app/features.server.ts @@ -7,20 +7,28 @@ export type TriggerFeatures = { alertsEnabled: boolean; }; -// If the request host is cloud.trigger.dev then we are on the managed cloud -// or if env.NODE_ENV is development -export function featuresForRequest(request: Request): TriggerFeatures { - const url = requestUrl(request); - - const isManagedCloud = - url.host === "cloud.trigger.dev" || - url.host === "test-cloud.trigger.dev" || - url.host === "internal.trigger.dev" || - process.env.CLOUD_ENV === "development"; +function isManagedCloud(host: string): boolean { + return ( + host === "cloud.trigger.dev" || + host === "test-cloud.trigger.dev" || + host === "internal.trigger.dev" || + process.env.CLOUD_ENV === "development" + ); +} +function featuresForHost(host: string): TriggerFeatures { return { - isManagedCloud, + isManagedCloud: isManagedCloud(host), v3Enabled: env.V3_ENABLED === "true", alertsEnabled: env.ALERT_FROM_EMAIL !== undefined && env.ALERT_RESEND_API_KEY !== undefined, }; } + +export function featuresForRequest(request: Request): TriggerFeatures { + const url = requestUrl(request); + return featuresForUrl(url); +} + +export function featuresForUrl(url: URL): TriggerFeatures { + return featuresForHost(url.host); +} diff --git a/apps/webapp/app/models/organization.server.ts b/apps/webapp/app/models/organization.server.ts index 814f946f3..049ba3412 100644 --- a/apps/webapp/app/models/organization.server.ts +++ b/apps/webapp/app/models/organization.server.ts @@ -8,10 +8,10 @@ import type { import { customAlphabet } from "nanoid"; import slug from "slug"; import { prisma, PrismaClientOrTransaction } from "~/db.server"; -import { createProject } from "./project.server"; import { generate } from "random-words"; import { createApiKeyForEnv, createPkApiKeyForEnv, envSlug } from "./api-key.server"; import { env } from "~/env.server"; +import { featuresForUrl } from "~/features.server"; export type { Organization }; @@ -52,6 +52,8 @@ export async function createOrganization( ); } + const features = featuresForUrl(new URL(env.APP_ORIGIN)); + const organization = await prisma.organization.create({ data: { title, @@ -64,6 +66,7 @@ export async function createOrganization( role: "ADMIN", }, }, + v3Enabled: features.v3Enabled && !features.isManagedCloud, }, include: { members: true, diff --git a/apps/webapp/app/models/user.server.ts b/apps/webapp/app/models/user.server.ts index 19d5a2ee7..df4d43ca4 100644 --- a/apps/webapp/app/models/user.server.ts +++ b/apps/webapp/app/models/user.server.ts @@ -47,12 +47,21 @@ export async function findOrCreateMagicLinkUser( }, }); + const adminEmailRegex = env.ADMIN_EMAILS ? new RegExp(env.ADMIN_EMAILS) : undefined; + const makeAdmin = adminEmailRegex ? adminEmailRegex.test(input.email) : false; + const user = await prisma.user.upsert({ where: { email: input.email, }, - update: { email: input.email }, - create: { email: input.email, authenticationMethod: "MAGIC_LINK" }, + update: { + email: input.email, + }, + create: { + email: input.email, + authenticationMethod: "MAGIC_LINK", + admin: makeAdmin, // only on create, to prevent automatically removing existing admins + }, }); return { diff --git a/apps/webapp/app/services/telemetry.server.ts b/apps/webapp/app/services/telemetry.server.ts index 341a8094a..afd4d1f75 100644 --- a/apps/webapp/app/services/telemetry.server.ts +++ b/apps/webapp/app/services/telemetry.server.ts @@ -22,6 +22,11 @@ class Telemetry { #triggerClient: TriggerClient | undefined = undefined; constructor({ postHogApiKey, trigger }: Options) { + if (env.TRIGGER_TELEMETRY_DISABLED !== undefined) { + console.log("📉 Telemetry disabled"); + return; + } + if (postHogApiKey) { this.#posthogClient = new PostHog(postHogApiKey, { host: "https://eu.posthog.com" }); } else { diff --git a/docs/images/v3/self-hosting.png b/docs/images/v3/self-hosting.png new file mode 100644 index 000000000..b41724da4 Binary files /dev/null and b/docs/images/v3/self-hosting.png differ diff --git a/docs/v3/cli-deploy.mdx b/docs/v3/cli-deploy.mdx index d9933da59..addf9c996 100644 --- a/docs/v3/cli-deploy.mdx +++ b/docs/v3/cli-deploy.mdx @@ -68,7 +68,7 @@ Load the built image into your local docker. #### Self-hosted (builds locally) `--self-hosted` -Builds and loads the image using your local docker. Use the `--registry` option to specify the registry to push the image to when using `--self-hosted`, or just use `--push-image` to push to the default registry. +Builds and loads the image using your local docker. Use the `--registry` option to specify the registry to push the image to when using `--self-hosted`, or just use `--push` to push to the default registry. #### Registry `--registry` @@ -76,9 +76,7 @@ Builds and loads the image using your local docker. Use the `--registry` option The registry to push the image to when using --self-hosted. -#### Push image `--push-image` - - +#### Push image `--push` When using the --self-hosted flag, push the image to the default registry. (defaults to false when not using --registry) diff --git a/docs/v3/open-source-self-hosting.mdx b/docs/v3/open-source-self-hosting.mdx index 87b33a0ce..51e1142fc 100644 --- a/docs/v3/open-source-self-hosting.mdx +++ b/docs/v3/open-source-self-hosting.mdx @@ -3,4 +3,275 @@ title: "Self-hosting" description: "You can self-host Trigger.dev on your own infrastructure." --- - +## Overview + + + Self-hosting architecture + + +The self-hosting guide comes in two parts. The first part is a simple setup where you run everything on one server. In the second part, the webapp and worker components are split on two separate machines. + +You're going to need at least one Debian (or derivative) machine with Docker and Docker Compose installed. We'll also use Ngrok to expose the webapp to the internet. + +## Caveats + +The v3 worker components don't have ARM support yet. + +This guide outlines a quick way to start self-hosting Trigger.dev. Scaling, security, and reliability concerns are not fully addressed here. It's unlikely to result in a production-ready deployment on its own, but it's a good starting point. + +As self-hosted deployments tend to have unique requirements and configurations, we don't provide specific advice for scaling up or improving security and reliability. + +Should the burden ever get too much, we'd be happy to see you on [Trigger.dev cloud](https://trigger.dev/pricing) where we deal with these concerns for you. + + +- The Docker [checkpoint command](https://docs.docker.com/reference/cli/docker/checkpoint/) is an experimental feature which may not work as expected. It won't be enabled by default. Instead, the containers will stay up and their processes frozen. They won't consume CPU but they _will_ consume RAM. +- The Docker provider does not currently enforce any resource limits. This means your tasks can consume up to the total machine CPU and RAM. Having no limits may be preferable when self-hosting, but can impact the performance of other services. +- The worker components (not the tasks!) have direct access to the Docker socket. This means they can run any Docker command. To restrict access, you may want to consider using [Docker Socket Proxy](https://github.com/Tecnativa/docker-socket-proxy). +- The task containers are running with host networking. This means there is no network isolation between them and the host machine. They will be able to access any networked service on the host. +- There is currently no support for adding multiple worker machines. This would require a more elaborate provider, or possibly a switch to Docker Swarm. This is not currently planned, but you are welcome to [contribute](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md). + + +## Requirements + +- 4 CPU +- 8 GB RAM +- Debian or derivative +- Optional: A separate machine for the worker components + +You will also need a way to expose the webapp to the internet. This can be done with a reverse proxy, or with a service like Ngrok. We will be using the latter in this guide. + +## Part 1: Single server + +This is the simplest setup. You run everything on one server. It's a good option if you have spare capacity on an existing machine, and have no need to independently scale worker capacity. + +### Server setup + +Some very basic steps to get started: + +1. [Install Docker](https://docs.docker.com/get-docker/) +2. [Install Docker Compose](https://docs.docker.com/compose/install/) +3. [Install Ngrok](https://ngrok.com/download) + +On a Debian server, you can install everything you need with the following commands: + +```bash +curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | \ + sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \ + echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | \ + sudo tee /etc/apt/sources.list.d/ngrok.list + +sudo apt-get update +sudo apt-get install -y \ + docker.io \ + docker-compose \ + ngrok +``` + +### Trigger.dev setup + +1. Clone the [Trigger.dev docker repository](https://github.com/triggerdotdev/docker) and checkout the v3 branch +```bash +git clone https://github.com/triggerdotdev/docker +cd docker +git checkout v3 +``` + +2. Run the start script and follow the prompts +```bash +./start.sh # hint: you can append -d to run in detached mode +``` + +### Manual setup + +Alternatively, you can follow these manual steps after cloning the docker repo: + +1. Create the `.env` file + +```bash +cp .env.example .env +``` + +2. Generate the required secrets + +```bash +echo MAGIC_LINK_SECRET=$(openssl rand -hex 16) +echo SESSION_SECRET=$(openssl rand -hex 16) +echo ENCRYPTION_KEY=$(openssl rand -hex 16) +echo PROVIDER_SECRET=$(openssl rand -hex 32) +echo COORDINATOR_SECRET=$(openssl rand -hex 32) +``` + +3. Replace the default secrets in the `.env` file with the generated ones + +4. Run docker compose to start the services + +```bash +. lib.sh # source the helper function +docker_compose -p=trigger up +``` + +### Tunnelling + +You will need to expose the webapp to the internet. You can use Ngrok for this. If you already have a working reverse proxy setup and a domain, you can skip to the last step. + +1. Start Ngrok. You may get prompted to sign up - it's free. + +```bash +./tunnel.sh +``` + +2. Copy the domain from the output, for example: `1234-42-42-42-42.ngrok-free.app` + +3. Uncomment the `TRIGGER_PROTOCOL` and `TRIGGER_DOMAIN` lines in the `.env` file. Set it to the domain you copied. + +```bash +TRIGGER_PROTOCOL=https +TRIGGER_DOMAIN=1234-42-42-42-42.ngrok-free.app +``` + +4. Quit the start script and launch it again, or run this: + +```bash +./stop.sh && ./start.sh +``` + +### Registry setup + +If you want to deploy v3 projects, you will need access to a Docker registry. The [CLI deploy](/v3/cli-deploy) command will push the images, and then the worker machine can pull them when needed. We will use Docker Hub as an example. + +1. Sign up for a free account at [Docker Hub](https://hub.docker.com/) + +2. Edit the `.env` file and add the registry details + +```bash +DEPLOY_REGISTRY_HOST=docker.io +DEPLOY_REGISTRY_NAMESPACE= +``` + +3. Log in to Docker Hub both locally and your server. For the split setup, this will be the worker machine. You may want to create an [access token](https://hub.docker.com/settings/security) for this. + +```bash +docker login -u +``` + +4. Restart the services + +```bash +./stop.sh && ./start.sh +``` + +5. You can now deploy v3 projects using the CLI with these flags: + +```bash +trigger deploy --self-hosted --push +``` + +## Part 2: Split services + +With this setup, the webapp will run on a different machine than the worker components. This allows independent scaling of your workload capacity. + +### Webapp setup + +All steps are the same as in Part 1, except for the following: + +1. Run the start script with the `webapp` argument + +```bash +./start.sh webapp +``` + +2. Tunnelling is now _required_. Please follow the tunnelling section from above. + +### Worker setup + +1. Copy your `.env` file from the webapp to the worker machine + +```bash +# an example using scp +scp -3 root@:docker/.env root@:docker/.env +``` + +2. Run the start script with the `worker` argument + +```bash +./start.sh worker +``` + +2. Tunnelling is _not_ required for the worker components. + +## Checkpoint support + +This requires an _experimental Docker feature_. Successfully checkpointing a task today, does not mean you will be able to restore it tomorrow. Your data may be lost. You've been warned! + +Checkpointing allows you to save the state of a running container to disk and restore it later. This can be useful for +long-running tasks that need to be paused and resumed without losing state. Think fan-out and fan-in, or long waits in email campaigns. + +### Requirements + +- Debian, **NOT** a derivative like Ubuntu +- Additional storage space for the checkpointed containers + +### Setup + +Underneath the hood this uses Checkpoint and Restore in Userspace, or [CRIU](https://github.com/checkpoint-restore/criu) in short. We'll have to do a few things to get this working: + +1. Install CRIU + +```bash +sudo apt-get update +sudo apt-get install criu +``` + +2. Tweak the config so we can successfully checkpoint our workloads + +```bash +mkdir /etc/criu + +cat << EOF >/etc/criu/runc.conf +tcp-close +EOF +``` + +3. Make sure everything works + +```bash +sudo criu check +``` + +3. Enable Docker experimental features, by adding the following to `/etc/docker/daemon.json` + +```json +{ + "experimental": true +} +``` + +4. Restart the Docker daemon + +```bash +sudo systemctl restart docker +``` + +5. Uncomment `FORCE_CHECKPOINT_SIMULATION=0` in your `.env` file. Alternatively, run this: + +```bash +echo "FORCE_CHECKPOINT_SIMULATION=0" >> .env +``` + +6. Restart the services + +```bash +# if you're running everything on the same machine +./stop.sh && ./start.sh + +# if you're running the worker on a different machine +./stop.sh worker && ./start.sh worker +``` + +## Telemetry + +By default, the Trigger.dev webapp sends telemetry data to our servers. This data is used to improve the product and is not shared with third parties. If you would like to opt-out of this, you can set the `TRIGGER_TELEMETRY_DISABLED` environment variable in your `.env` file. The value doesn't matter, it just can't be empty. For example: + +```bash +TRIGGER_TELEMETRY_DISABLED=1 +``` diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index fced3d579..9a2102cee 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -107,7 +107,7 @@ export function configureDeployCommand(program: Command) { .addOption( new CommandOption( "--self-hosted", - "Build and load the image using your local Docker. Use the --registry option to specify the registry to push the image to when using --self-hosted, or just use --push-image to push to the default registry." + "Build and load the image using your local Docker. Use the --registry option to specify the registry to push the image to when using --self-hosted, or just use --push to push to the default registry." ).hideHelp() ) .addOption( diff --git a/packages/cli-v3/src/commands/init.ts b/packages/cli-v3/src/commands/init.ts index 3611b0486..16f794c32 100644 --- a/packages/cli-v3/src/commands/init.ts +++ b/packages/cli-v3/src/commands/init.ts @@ -30,6 +30,8 @@ import { logger } from "../utilities/logger"; import { cliRootPath } from "../utilities/resolveInternalFilePath"; import { login } from "./login"; import { spinner } from "../utilities/windows"; +import { CLOUD_API_URL } from "../consts"; +import * as packageJson from "../../package.json"; const InitCommandOptions = CommonCommandOptions.extend({ projectRef: z.string().optional(), @@ -53,7 +55,7 @@ export function configureInitCommand(program: Command) { .option( "-t, --tag ", "The version of the @trigger.dev/sdk package to install", - "beta" + packageJson.version ) .option("--skip-package-install", "Skip installing the @trigger.dev/sdk package") .option("--override-config", "Override the existing config file if it exists") @@ -160,7 +162,9 @@ async function _initCommand(dir: string, options: InitCommandOptions) { log.info("Next steps:"); log.info( ` 1. To start developing, run ${chalk.green( - `npx trigger.dev@${options.tag} dev` + `npx trigger.dev@${options.tag} dev${ + options.apiUrl === CLOUD_API_URL ? "" : ` -a ${options.apiUrl}` + }` )} in your project directory` ); log.info(` 2. Visit your ${projectDashboard} to view your newly created tasks.`); @@ -401,7 +405,8 @@ async function installPackages(dir: string, options: InitCommandOptions) { case "npm": { installSpinner.start(`Running npm install @trigger.dev/sdk@${options.tag}`); - await execa("npm", ["install", `@trigger.dev/sdk@${options.tag}`], { + // --save-exact: pin version, e.g. 3.0.0-beta.20 instead of ^3.0.0-beta.20 + await execa("npm", ["install", "--save-exact", `@trigger.dev/sdk@${options.tag}`], { cwd: projectDir, stdio: options.logLevel === "debug" ? "inherit" : "ignore", }); @@ -411,6 +416,7 @@ async function installPackages(dir: string, options: InitCommandOptions) { case "pnpm": { installSpinner.start(`Running pnpm add @trigger.dev/sdk@${options.tag}`); + // pins version by default await execa("pnpm", ["add", `@trigger.dev/sdk@${options.tag}`], { cwd: projectDir, stdio: options.logLevel === "debug" ? "inherit" : "ignore", @@ -421,6 +427,7 @@ async function installPackages(dir: string, options: InitCommandOptions) { case "yarn": { installSpinner.start(`Running yarn add @trigger.dev/sdk@${options.tag}`); + // pins version by default await execa("yarn", ["add", `@trigger.dev/sdk@${options.tag}`], { cwd: projectDir, stdio: options.logLevel === "debug" ? "inherit" : "ignore", diff --git a/packages/cli-v3/src/telemetry/tracing.ts b/packages/cli-v3/src/telemetry/tracing.ts index 4df9a3e3b..90364ebdd 100644 --- a/packages/cli-v3/src/telemetry/tracing.ts +++ b/packages/cli-v3/src/telemetry/tracing.ts @@ -9,9 +9,15 @@ import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION, } from "@opentelemetry/semantic-conventions"; +import { logger } from "../utilities/logger"; function initializeTracing(): NodeTracerProvider | undefined { - if (process.argv.includes("--skip-telemetry") || process.env.TRIGGER_DEV_SKIP_TELEMETRY) { + if ( + process.argv.includes("--skip-telemetry") || + process.env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat + process.env.TRIGGER_TELEMETRY_DISABLED + ) { + logger.debug("📉 Telemetry disabled"); return; } diff --git a/packages/core-apps/package.json b/packages/core-apps/package.json index 037500e7d..fcd278c80 100644 --- a/packages/core-apps/package.json +++ b/packages/core-apps/package.json @@ -28,10 +28,13 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/tsconfig": "workspace:*", "@types/node": "18", - "typescript": "^5.3.0", - "socket.io-client": "^4.7.4" + "socket.io-client": "^4.7.4", + "typescript": "^5.3.0" }, "engines": { "node": ">=18.0.0" + }, + "dependencies": { + "execa": "^8.0.1" } } \ No newline at end of file diff --git a/packages/core-apps/src/checkpoints.ts b/packages/core-apps/src/checkpoints.ts new file mode 100644 index 000000000..039a0bd16 --- /dev/null +++ b/packages/core-apps/src/checkpoints.ts @@ -0,0 +1,67 @@ +import { randomUUID } from "crypto"; +import { $, type ExecaChildProcess } from "execa"; + +export function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited { + return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca; +} + +export type CheckpointTestResult = + | { + ok: true; + } + | { + ok: false; + message: string; + error?: unknown; + }; + +export async function testDockerCheckpoint(): Promise { + try { + // Create a dummy container + const container = + await $`docker run -d --rm --name init-dummy-${randomUUID()} docker.io/library/busybox sleep 10`; + + // Checkpoint it + await $`docker checkpoint create ${container} init-check`; + } catch (error) { + if (!isExecaChildProcess(error)) { + return { + ok: false, + message: "No checkpoint support: Unknown error.", + error, + }; + } + + if (error.stderr.includes("criu")) { + if (error.stderr.includes("executable file not found")) { + return { + ok: false, + message: "No checkpoint support: Missing CRIU binary.", + }; + } + + return { + ok: false, + message: "No checkpoint support: Unknown CRIU error.", + error, + }; + } + + if (error.stderr.includes("experimental features enabled")) { + return { + ok: false, + message: "No checkpoint support: Please enable docker experimental features.", + }; + } + + return { + ok: false, + message: "No checkpoint support: Unknown execa error.", + error, + }; + } + + return { + ok: true, + }; +} diff --git a/packages/core-apps/src/index.ts b/packages/core-apps/src/index.ts index 7f18d773e..a4efbc947 100644 --- a/packages/core-apps/src/index.ts +++ b/packages/core-apps/src/index.ts @@ -1,3 +1,4 @@ export * from "./http"; export * from "./logger"; export * from "./provider"; +export * from "./checkpoints"; diff --git a/packages/core-apps/src/provider.ts b/packages/core-apps/src/provider.ts index 159c81dbc..14ed71033 100644 --- a/packages/core-apps/src/provider.ts +++ b/packages/core-apps/src/provider.ts @@ -62,10 +62,14 @@ export interface TaskOperationsRestoreOptions { } export interface TaskOperations { + init: () => Promise; + + // CRUD index: (opts: TaskOperationsIndexOptions) => Promise; create: (opts: TaskOperationsCreateOptions) => Promise; restore: (opts: TaskOperationsRestoreOptions) => Promise; + // unimplemented delete: (...args: any[]) => Promise; get: (...args: any[]) => Promise; } @@ -303,7 +307,8 @@ export class ProviderShell implements Provider { return httpServer; } - listen() { + async listen() { this.#httpServer.listen(this.#httpPort, this.options.host ?? "0.0.0.0"); + await this.tasks.init(); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b8642b0e..e483fd136 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1811,6 +1811,10 @@ importers: version: 5.3.2 packages/core-apps: + dependencies: + execa: + specifier: ^8.0.1 + version: 8.0.1 devDependencies: '@trigger.dev/core': specifier: workspace:*