diff --git a/.env.example b/.env.example index 47f77ba50..1ce00e294 100644 --- a/.env.example +++ b/.env.example @@ -45,4 +45,8 @@ CLOUD_LINEAR_CLIENT_ID= CLOUD_LINEAR_CLIENT_SECRET= CLOUD_SLACK_APP_HOST= CLOUD_SLACK_CLIENT_ID= -CLOUD_SLACK_CLIENT_SECRET= \ No newline at end of file +CLOUD_SLACK_CLIENT_SECRET= + +# v3 variables +PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32` +COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32` \ No newline at end of file diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml new file mode 100644 index 000000000..ed72c84b7 --- /dev/null +++ b/.github/workflows/publish-dev.yml @@ -0,0 +1,83 @@ +name: "🚢 Publish Container Images (dev)" + +on: + push: + tags: + - "dev-*" + paths: + - ".github/workflows/publish.yml" + - "packages/**" + - "!packages/**/*.md" + - "!packages/**/*.eslintrc" + - "apps/**" + - "!apps/**/*.md" + - "!apps/**/*.eslintrc" + - "integrations/**" + - "!integrations/**/*.md" + - "!integrations/**/*.eslintrc" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "turbo.json" + - "docker/Dockerfile" + - "docker/scripts/**" + - "tests/**" + +permissions: + id-token: write + packages: write + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +env: + AWS_REGION: us-east-1 + +jobs: + build: + strategy: + matrix: + package: [coordinator, kubernetes-provider] + runs-on: buildjet-4vcpu-ubuntu-2204 + env: + DOCKER_BUILDKIT: "1" + steps: + - uses: actions/checkout@v4 + + - name: Generate build ID + id: prep + run: | + sha=${GITHUB_SHA::7} + ts=$(date +%s) + echo "BUILD_ID=${{ matrix.package }}-${sha}-${ts}" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # ..to avoid rate limits when pulling images + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: 🚢 Build Container Image + run: | + docker build -t dev_image -f ./apps/${{ matrix.package }}/Containerfile . + + # ..to push image + - name: 🐙 Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: 🐙 Push to GitHub Container Registry + run: | + docker tag dev_image $REGISTRY/$REPOSITORY:$IMAGE_TAG + docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG + env: + REGISTRY: ghcr.io/triggerdotdev + REPOSITORY: dev + IMAGE_TAG: ${{ steps.prep.outputs.BUILD_ID }} diff --git a/apps/coordinator/.env.example b/apps/coordinator/.env.example new file mode 100644 index 000000000..2a416edcf --- /dev/null +++ b/apps/coordinator/.env.example @@ -0,0 +1,4 @@ +HTTP_SERVER_PORT=8020 + +PLATFORM_ENABLED=true +PLATFORM_WS_PORT=3030 \ No newline at end of file diff --git a/apps/coordinator/.gitignore b/apps/coordinator/.gitignore new file mode 100644 index 000000000..5c84119d6 --- /dev/null +++ b/apps/coordinator/.gitignore @@ -0,0 +1,3 @@ +dist/ +node_modules/ +.env \ No newline at end of file diff --git a/apps/coordinator/Containerfile b/apps/coordinator/Containerfile new file mode 100644 index 000000000..e1c5e2e56 --- /dev/null +++ b/apps/coordinator/Containerfile @@ -0,0 +1,63 @@ +# syntax=docker/dockerfile:labs + +FROM node:18.18.2-bullseye-slim@sha256:21479df46c3173ee0cefc6b264928e10239152c4f74df872ca9369be01a245b7 AS node-18 + +WORKDIR /app + +FROM node-18 AS pruner + +COPY --chown=node:node . . +RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker +RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + + +FROM node-18 AS base + +RUN apt-get update \ + && apt-get install -y buildah ca-certificates dumb-init \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=node:node .gitignore .gitignore +COPY --from=pruner --chown=node:node /app/out/json/ . +COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml +COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml + +FROM base AS dev-deps +RUN corepack enable +ENV NODE_ENV development + +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile + +FROM base AS builder +RUN corepack enable + +COPY --from=pruner --chown=node:node /app/out/full/ . +COPY --from=dev-deps --chown=node:node /app/ . +COPY --chown=node:node turbo.json turbo.json + +RUN pnpm run -r --filter '@trigger.dev/core*' build +RUN pnpm run -r --filter coordinator build:bundle + +FROM alpine AS cri-tools + +WORKDIR /cri-tools + +ARG CRICTL_VERSION=v1.29.0 +ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb +ADD --checksum=${CRICTL_CHECKSUM} \ + https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz . +RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz + +FROM base AS runner + +RUN corepack enable +ENV NODE_ENV production + +COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin +COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.cjs ./index.cjs + +EXPOSE 8000 + +USER node + +CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.cjs" ] diff --git a/apps/coordinator/README.md b/apps/coordinator/README.md new file mode 100644 index 000000000..fa6da2462 --- /dev/null +++ b/apps/coordinator/README.md @@ -0,0 +1,3 @@ +# Coordinator + +Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things. diff --git a/apps/coordinator/package.json b/apps/coordinator/package.json new file mode 100644 index 000000000..ef8c7a875 --- /dev/null +++ b/apps/coordinator/package.json @@ -0,0 +1,34 @@ +{ + "name": "coordinator", + "private": true, + "version": "0.0.1", + "description": "", + "main": "dist/index.cjs", + "type": "module", + "scripts": { + "build": "npm run build:bundle", + "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node", + "build:image": "docker build -f Containerfile . -t coordinator", + "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", + "start": "tsx src/index.ts", + "typecheck": "tsc --noEmit" + }, + "keywords": [], + "author": "", + "license": "MIT", + "dependencies": { + "@trigger.dev/core": "workspace:*", + "@trigger.dev/core-apps": "workspace:*", + "execa": "^8.0.1", + "prom-client": "^15.1.0", + "socket.io": "^4.7.4", + "socket.io-client": "^4.7.4" + }, + "devDependencies": { + "@types/node": "^18", + "dotenv": "^16.4.2", + "esbuild": "^0.19.11", + "tsx": "^4.7.0", + "typescript": "^5.3.3" + } +} diff --git a/apps/coordinator/src/index.ts b/apps/coordinator/src/index.ts new file mode 100644 index 000000000..428cf8b1b --- /dev/null +++ b/apps/coordinator/src/index.ts @@ -0,0 +1,506 @@ +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import { $ } from "execa"; +import { Namespace } from "socket.io"; +import { Server } from "socket.io"; +import { Socket, io } from "socket.io-client"; +import { DefaultEventsMap } from "socket.io/dist/typed-events"; +import { + CoordinatorToPlatformEvents, + CoordinatorToProdWorkerEvents, + PlatformToCoordinatorEvents, + ProdWorkerSocketData, + ProdWorkerToCoordinatorEvents, +} from "@trigger.dev/core/v3"; +import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps"; + +import { collectDefaultMetrics, register, Gauge } from "prom-client"; +collectDefaultMetrics(); + +const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020); +const NODE_NAME = process.env.NODE_NAME || "coordinator"; +const REGISTRY_FQDN = process.env.REGISTRY_FQDN || "localhost:5000"; +const REPO_NAME = process.env.REPO_NAME || "checkpoints"; +const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints"; +const REGISTRY_TLS_VERIFY = process.env.REGISTRY_TLS_VERIFY === "false" ? "false" : "true"; + +const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ?? "true"); +const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1"; +const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030; +const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret"; + +const logger = new SimpleLogger(`[${NODE_NAME}]`); + +class Checkpointer { + #initialized = false; + #canCheckpoint = false; + #dockerMode = true; + + #logger = new SimpleLogger("[checkptr]"); + + async initialize() { + if (this.#initialized) { + return; + } + + try { + await $`criu --version`; + } catch (error) { + this.#logger.error("No checkpoint support: Missing CRIU binary"); + this.#canCheckpoint = false; + this.#initialized = true; + return; + } + + if (this.#dockerMode) { + try { + await $`docker checkpoint`; + } catch (error) { + this.#logger.error( + "No checkpoint support: Docker needs to have experimental features enabled" + ); + this.#canCheckpoint = false; + this.#initialized = true; + return; + } + } + + this.#logger.log( + `Full checkpoint support with docker ${this.#dockerMode ? "enabled" : "disabled"}` + ); + + this.#initialized = true; + this.#canCheckpoint = true; + } + + async checkpointAndPush(podName: string) { + await this.initialize(); + + if (!this.#canCheckpoint) { + return; + } + + try { + const { path } = await this.#checkpointContainer(podName); + const { tag } = await this.#buildImage(path, podName); + const { destination } = await this.#pushImage(tag); + + if (this.#dockerMode) { + this.#logger.log("checkpoint created:", { podName, path }); + } else { + this.#logger.log("checkpointed and pushed image to:", destination); + } + + return { + path, + tag, + destination: this.#dockerMode ? path : destination, + docker: this.#dockerMode, + }; + } catch (error) { + this.#logger.error("checkpoint failed", error); + return; + } + } + + async #checkpointContainer(podName: string) { + await this.initialize(); + + if (!this.#canCheckpoint) { + throw new Error("No checkpoint support"); + } + + if (this.#dockerMode) { + this.#logger.log("Checkpointing:", podName); + + const path = randomUUID(); + + try { + this.#logger.debug(await $`docker checkpoint create --leave-running ${podName} ${path}`); + } catch (error: any) { + this.#logger.error(error.stderr); + } + + return { path }; + } + + const containerId = this.#logger.debug( + // @ts-expect-error + await $`crictl ps` + .pipeStdout($({ stdin: "pipe" })`grep ${podName}`) + .pipeStdout($({ stdin: "pipe" })`cut -f1 ${"-d "}`) + ); + + if (!containerId.stdout) { + throw new Error("could not find container id"); + } + + const exportPath = `${CHECKPOINT_PATH}/${podName}.tar`; + + this.#logger.debug(await $`crictl checkpoint --export=${exportPath} ${containerId}`); + + return { + path: exportPath, + }; + } + + async #buildImage(checkpointPath: string, tag: string) { + await this.initialize(); + + if (!this.#canCheckpoint) { + throw new Error("No checkpoint support"); + } + + if (this.#dockerMode) { + // Nothing to do here + return { tag }; + } + + const container = this.#logger.debug(await $`buildah from scratch`); + this.#logger.debug(await $`buildah add ${container} ${checkpointPath} /`); + this.#logger.debug( + await $`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}` + ); + this.#logger.debug(await $`buildah commit ${container} ${REGISTRY_FQDN}/${REPO_NAME}:${tag}`); + this.#logger.debug(await $`buildah rm ${container}`); + + return { + tag, + }; + } + + async #pushImage(tag: string) { + await this.initialize(); + + if (!this.#canCheckpoint) { + throw new Error("No checkpoint support"); + } + + if (this.#dockerMode) { + // Nothing to do here + return { destination: "" }; + } + + const destination = `${REGISTRY_FQDN}/${REPO_NAME}:${tag}`; + this.#logger.debug(await $`buildah push --tls-verify=${REGISTRY_TLS_VERIFY} ${destination}`); + + return { + destination, + }; + } +} + +class TaskCoordinator { + #httpServer: ReturnType; + #checkpointer = new Checkpointer(); + + #prodWorkerNamespace: Namespace< + ProdWorkerToCoordinatorEvents, + CoordinatorToProdWorkerEvents, + DefaultEventsMap, + ProdWorkerSocketData + >; + #platformSocket?: Socket; + + constructor( + private port: number, + private host = "0.0.0.0" + ) { + this.#httpServer = this.#createHttpServer(); + this.#checkpointer.initialize(); + + const io = new Server(this.#httpServer); + this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io); + + this.#platformSocket = this.#createPlatformSocket(); + + const connectedTasksTotal = new Gauge({ + name: "daemon_connected_tasks_total", // don't change this without updating dashboard config + help: "The number of tasks currently connected.", + collect: () => { + connectedTasksTotal.set(this.#prodWorkerNamespace.sockets.size); + }, + }); + register.registerMetric(connectedTasksTotal); + } + + #createPlatformSocket() { + if (!PLATFORM_ENABLED) { + console.log("INFO: platform connection disabled"); + return; + } + + const socket: Socket = io( + `ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/coordinator`, + { + transports: ["websocket"], + auth: { + token: PLATFORM_SECRET, + }, + } + ); + + const logger = new SimpleLogger(`[platform][${socket.id ?? "NO_ID"}]`); + + socket.on("connect", () => { + logger.log("connect"); + }); + + socket.on("connect_error", (err) => { + logger.error(`connect_error: ${err.message}`); + }); + + socket.on("disconnect", () => { + logger.log("disconnect"); + }); + + socket.on("RESUME", async (message) => { + logger.log("[RESUME]", message); + + const taskSocket = await this.#getAttemptSocket(message.attemptId); + + if (!taskSocket) { + logger.log("Socket for attempt not found", { attemptId: message.attemptId }); + return; + } + + taskSocket.emit("RESUME", message); + }); + + return socket; + } + + async #getAttemptSocket(attemptId: string) { + const sockets = await this.#prodWorkerNamespace.fetchSockets(); + + for (const socket of sockets) { + if (socket.data.attemptId === attemptId) { + return socket; + } + } + } + + #createProdWorkerNamespace(io: Server) { + const namespace: Namespace< + ProdWorkerToCoordinatorEvents, + CoordinatorToProdWorkerEvents, + DefaultEventsMap, + ProdWorkerSocketData + > = io.of("/prod-worker"); + + namespace.on("connection", async (socket) => { + const logger = new SimpleLogger(`[task][${socket.id}]`); + + this.#platformSocket?.emit("LOG", { + version: "v1", + metadata: { + projectRef: socket.data.projectRef, + attemptId: socket.data.attemptId, + }, + text: "connected", + }); + + logger.log("connected"); + + socket.on("disconnect", (reason, description) => { + logger.log("disconnect", { reason, description }); + + this.#platformSocket?.emit("LOG", { + version: "v1", + metadata: { + projectRef: socket.data.projectRef, + attemptId: socket.data.attemptId, + }, + text: "disconnect", + }); + }); + + socket.on("error", (error) => { + logger.error({ error }); + }); + + socket.on("LOG", (message, callback) => { + logger.log("[LOG]", message.text); + + callback(); + + this.#platformSocket?.emit("LOG", { + version: "v1", + metadata: { attemptId: socket.data.attemptId }, + text: message.text, + }); + }); + + socket.on("READY_FOR_EXECUTION", async (message) => { + logger.log("[READY_FOR_EXECUTION]", message); + + const executionAck = await this.#platformSocket?.emitWithAck("READY_FOR_EXECUTION", { + version: "v1", + attemptId: message.attemptId, + }); + + if (!executionAck) { + logger.error("no execution ack", { attemptId: socket.data.attemptId }); + return; + } + + if (!executionAck.success) { + logger.error("execution unsuccessful", { attemptId: socket.data.attemptId }); + return; + } + + // FIXME: shouldn't wait for completion here + const completionAck = await socket.emitWithAck("EXECUTE_TASK_RUN", { + version: "v1", + payload: executionAck.payload, + }); + + logger.log("completed task", { completionId: completionAck.completion.id }); + + this.#platformSocket?.emit("TASK_RUN_COMPLETED", { + version: "v1", + execution: executionAck.payload.execution, + completion: completionAck.completion, + }); + }); + + socket.on("TASK_HEARTBEAT", (message) => { + logger.log("[TASK_HEARTBEAT]", message); + + this.#platformSocket?.emit("TASK_HEARTBEAT", message); + }); + + socket.on("WAIT_FOR_BATCH", (message) => { + logger.log("[WAIT_FOR_BATCH]", message); + + // this.#checkpointer.checkpointAndPush(socket.data.podName); + }); + + socket.on("WAIT_FOR_DURATION", async (message, callback) => { + logger.log("[WAIT_FOR_DURATION]", message); + + const checkpoint = await this.#checkpointer.checkpointAndPush(socket.data.podName); + + if (!checkpoint) { + logger.error("Failed to checkpoint", { podName: socket.data.podName }); + callback({ success: false }); + return; + } + + this.#platformSocket?.emit("CHECKPOINT_CREATED", { + version: "v1", + attemptId: socket.data.attemptId, + docker: checkpoint.docker, + location: checkpoint.destination, + reason: "WAIT_FOR_DURATION", + }); + + callback({ success: true }); + }); + + socket.on("WAIT_FOR_TASK", (message) => { + logger.log("[WAIT_FOR_TASK]", message); + + // this.#checkpointer.checkpointAndPush(socket.data.podName); + }); + + socket.on("INDEX_TASKS", async (message, callback) => { + logger.log("[INDEX_TASKS]", message); + + const workerAck = await this.#platformSocket?.emitWithAck("CREATE_WORKER", { + version: "v1", + projectRef: socket.data.projectRef, + envId: socket.data.envId, + metadata: { + cliPackageVersion: socket.data.cliPackageVersion, + contentHash: socket.data.contentHash, + packageVersion: message.packageVersion, + tasks: message.tasks, + }, + }); + + if (!workerAck) { + logger.debug("no worker ack while indexing", message); + } + + callback({ success: !!workerAck?.success }); + }); + }); + + // auth middleware + namespace.use(async (socket, next) => { + const logger = new SimpleLogger(`[task][${socket.id}][auth]`); + + function setSocketDataFromHeader(dataKey: keyof typeof socket.data, headerName: string) { + const value = socket.handshake.headers[headerName]; + if (!value) { + logger.error(`missing required header: ${headerName}`); + throw new Error("missing header"); + } + socket.data[dataKey] = Array.isArray(value) ? value[0] : value; + } + + try { + setSocketDataFromHeader("podName", "x-pod-name"); + setSocketDataFromHeader("contentHash", "x-trigger-content-hash"); + setSocketDataFromHeader("cliPackageVersion", "x-trigger-cli-package-version"); + setSocketDataFromHeader("projectRef", "x-trigger-project-ref"); + setSocketDataFromHeader("attemptId", "x-trigger-attempt-id"); + setSocketDataFromHeader("envId", "x-trigger-env-id"); + } catch (error) { + return socket.disconnect(true); + } + + logger.log("success", socket.data); + + next(); + }); + + return namespace; + } + + #createHttpServer() { + const httpServer = createServer(async (req, res) => { + logger.log(`[${req.method}]`, req.url); + + const reply = new HttpReply(res); + + switch (req.url) { + case "/health": { + return reply.text("ok"); + } + case "/metrics": { + return reply.text(await register.metrics(), 200, register.contentType); + } + case "/whoami": { + return reply.text(NODE_NAME); + } + case "/checkpoint": { + const body = await getTextBody(req); + await this.#checkpointer.checkpointAndPush(body); + return reply.text(`sent restore request: ${body}`); + } + default: { + return reply.empty(404); + } + } + }); + + httpServer.on("clientError", (err, socket) => { + socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); + }); + + httpServer.on("listening", () => { + logger.log("server listening on port", HTTP_SERVER_PORT); + }); + + return httpServer; + } + + listen() { + this.#httpServer.listen(this.port, this.host); + } +} + +const coordinator = new TaskCoordinator(HTTP_SERVER_PORT); +coordinator.listen(); diff --git a/apps/coordinator/tsconfig.json b/apps/coordinator/tsconfig.json new file mode 100644 index 000000000..a491b740b --- /dev/null +++ b/apps/coordinator/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "commonjs", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "paths": { + "@trigger.dev/core/v3": ["../core/src/v3"], + "@trigger.dev/core/v3/*": ["../core/src/v3/*"], + "@trigger.dev/core-apps": ["../core-apps/src"], + "@trigger.dev/core-apps/*": ["../core-apps/src/*"] + } + } +} diff --git a/apps/docker-provider/.env.example b/apps/docker-provider/.env.example new file mode 100644 index 000000000..6083fd511 --- /dev/null +++ b/apps/docker-provider/.env.example @@ -0,0 +1,4 @@ +HTTP_SERVER_PORT=8050 + +PLATFORM_WS_PORT=3030 +PLATFORM_SECRET=provider-secret \ No newline at end of file diff --git a/apps/docker-provider/.gitignore b/apps/docker-provider/.gitignore new file mode 100644 index 000000000..5c84119d6 --- /dev/null +++ b/apps/docker-provider/.gitignore @@ -0,0 +1,3 @@ +dist/ +node_modules/ +.env \ No newline at end of file diff --git a/apps/docker-provider/Containerfile b/apps/docker-provider/Containerfile new file mode 100644 index 000000000..56a2b0d51 --- /dev/null +++ b/apps/docker-provider/Containerfile @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:labs + +FROM node:18-slim AS base + +RUN apt-get update \ + && apt-get install -y dumb-init + +FROM base + +WORKDIR /app + +COPY --chown=node dist/index.cjs /app/ + +EXPOSE 8000 + +ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.cjs" ] diff --git a/apps/docker-provider/README.md b/apps/docker-provider/README.md new file mode 100644 index 000000000..647db280a --- /dev/null +++ b/apps/docker-provider/README.md @@ -0,0 +1,3 @@ +# Docker provider + +The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions. diff --git a/apps/docker-provider/package.json b/apps/docker-provider/package.json new file mode 100644 index 000000000..2590629db --- /dev/null +++ b/apps/docker-provider/package.json @@ -0,0 +1,32 @@ +{ + "name": "docker-provider", + "private": true, + "version": "0.0.1", + "description": "", + "main": "dist/index.cjs", + "type": "module", + "scripts": { + "build": "npm run build:bundle", + "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node", + "build:image": "docker build -f Containerfile . -t docker-provider", + "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", + "start": "tsx src/index.ts", + "typecheck": "tsc --noEmit" + }, + "keywords": [], + "author": "", + "license": "MIT", + "dependencies": { + "@trigger.dev/core": "workspace:*", + "@trigger.dev/core-apps": "workspace:*", + "execa": "^8.0.1", + "socket.io-client": "^4.7.4" + }, + "devDependencies": { + "@types/node": "^18.19.8", + "dotenv": "^16.4.2", + "esbuild": "^0.19.11", + "tsx": "^4.7.0", + "typescript": "^5.3.3" + } +} diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts new file mode 100644 index 000000000..78a8f3160 --- /dev/null +++ b/apps/docker-provider/src/index.ts @@ -0,0 +1,335 @@ +import { createServer } from "node:http"; +import { $ } from "execa"; +import { io, Socket } from "socket.io-client"; +import { + clientWebsocketMessages, + Machine, + MessageCatalogToSocketIoEvents, + ProviderClientToServerEvents, + ProviderServerToClientEvents, + serverWebsocketMessages, + ZodMessageHandler, + ZodMessageSender, +} from "@trigger.dev/core/v3"; +import { HttpReply, SimpleLogger, getTextBody, getRandomPortNumber } from "@trigger.dev/core-apps"; + +const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber()); +const MACHINE_NAME = process.env.MACHINE_NAME || "local"; + +const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1"; +const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030; +const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret"; + +const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020; + +const logger = new SimpleLogger(`[${MACHINE_NAME}]`); + +interface TaskOperations { + create: (...args: any[]) => Promise; + restore: (...args: any[]) => Promise; + delete: (...args: any[]) => Promise; + get: (...args: any[]) => Promise; + index: (...args: any[]) => Promise; +} + +class DockerTaskOperations implements TaskOperations { + async index(opts: { contentHash: string; imageTag: string; envId: string }) { + const containerName = this.#getIndexContainerName(opts.contentHash); + + const { exitCode } = logger.debug( + await $`docker run --rm -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e INDEX_TASKS=true --network=host --pull=never --name=${containerName} ${opts.imageTag}` + ); + + if (exitCode !== 0) { + throw new Error("docker run command failed"); + } + } + + async create(opts: { attemptId: string; image: string; machine: Machine; envId: string }) { + const containerName = this.#getRunContainerName(opts.attemptId); + + const { exitCode } = logger.debug( + await $`docker run -d -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e TRIGGER_ATTEMPT_ID=${opts.attemptId} --network=host --pull=never --name=${containerName} ${opts.image}` + ); + + if (exitCode !== 0) { + throw new Error("docker run command failed"); + } + } + + async restore(opts: { + attemptId: string; + runId: string; + image: string; + name: string; + checkpointId: string; + machine: Machine; + }) { + const containerName = this.#getRunContainerName(opts.attemptId); + + const { exitCode } = logger.debug( + await $`docker start --checkpoint=${opts.checkpointId} ${containerName}` + ); + + if (exitCode !== 0) { + throw new Error("docker start command failed"); + } + } + + async delete(opts: { runId: string }) { + logger.log("noop: delete"); + } + + async get(opts: { runId: string }) { + logger.log("noop: get"); + } + + #getIndexContainerName(contentHash: string) { + return `task-index-${contentHash}`; + } + + #getRunContainerName(attemptId: string) { + return `task-run-${attemptId}`; + } +} + +interface Provider { + tasks: TaskOperations; +} + +type DockerProviderOptions = { + tasks: DockerTaskOperations; + host?: string; + port: number; +}; + +class DockerProvider implements Provider { + tasks: DockerTaskOperations; + + #httpServer: ReturnType; + #platformSocket: Socket; + + constructor(private options: DockerProviderOptions) { + this.tasks = options.tasks; + this.#httpServer = this.#createHttpServer(); + this.#platformSocket = this.#createPlatformSocket(); + this.#createSharedQueueSocket(); + } + + #createSharedQueueSocket() { + const socket: Socket< + MessageCatalogToSocketIoEvents, + MessageCatalogToSocketIoEvents + > = io(`ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/shared-queue`, { + transports: ["websocket"], + auth: { + token: PLATFORM_SECRET, + }, + }); + + const logger = new SimpleLogger(`[shared-queue][${socket.id ?? "NO_ID"}]`); + + socket.on("connect_error", (err) => { + logger.error(`connect_error: ${err.message}`); + }); + + socket.on("connect", () => { + logger.log("connect"); + }); + + socket.on("disconnect", () => { + logger.log("disconnect"); + }); + + const sender = new ZodMessageSender({ + schema: clientWebsocketMessages, + sender: async (message) => { + return new Promise((resolve, reject) => { + try { + const { type, ...payload } = message; + socket.emit(type, payload as any); + resolve(); + } catch (err) { + reject(err); + } + }); + }, + }); + + const handler = new ZodMessageHandler({ + schema: serverWebsocketMessages, + messages: { + SERVER_READY: async (payload) => { + logger.log("received SERVER_READY", payload); + + // TODO: create new schema without worker requirement + await sender.send("READY_FOR_TASKS", { + backgroundWorkerId: "placeholder", + }); + }, + BACKGROUND_WORKER_MESSAGE: async (payload) => { + logger.log("received BACKGROUND_WORKER_MESSAGE", payload); + + if (payload.data.type === "SCHEDULE_ATTEMPT") { + this.tasks.create({ + envId: payload.data.envId, + attemptId: payload.data.id, + image: payload.data.image, + machine: {}, + }); + } + }, + }, + }); + handler.registerHandlers(socket, logger.log.bind(logger)); + + return socket; + } + + #createPlatformSocket() { + const socket: Socket = io( + `ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/provider`, + { + transports: ["websocket"], + auth: { + token: PLATFORM_SECRET, + }, + extraHeaders: { + "x-trigger-provider-type": "docker", + }, + } + ); + + const logger = new SimpleLogger(`[platform][${socket.id ?? "NO_ID"}]`); + + socket.on("connect_error", (err) => { + logger.error(`connect_error: ${err.message}`); + }); + + socket.on("connect", () => { + logger.log("connect"); + }); + + socket.on("disconnect", () => { + logger.log("disconnect"); + }); + + socket.on("GET", async (message) => { + logger.log("[GET]", message); + + this.tasks.get({ runId: message.name }); + }); + + socket.on("DELETE", async (message, callback) => { + logger.log("[DELETE]", message); + + callback({ + message: "delete request received", + }); + + this.tasks.delete({ runId: message.name }); + }); + + socket.on("INDEX", async (message) => { + logger.log("[INDEX]", message); + try { + await this.tasks.index({ + contentHash: message.contentHash, + imageTag: message.imageTag, + envId: message.envId, + }); + } catch (error) { + logger.error("task index failed", error); + } + }); + + socket.on("RESTORE", async (message) => { + logger.log("[RESTORE]", message); + // await this.tasks.restore({}); + }); + + socket.on("HEALTH", async (message) => { + logger.log("[HEALTH]", message); + }); + + return socket; + } + + #createHttpServer() { + const httpServer = createServer(async (req, res) => { + logger.log(`[${req.method}]`, req.url); + + const reply = new HttpReply(res); + + switch (req.url) { + case "/health": { + return reply.text("ok"); + } + case "/whoami": { + return reply.text(`${MACHINE_NAME}`); + } + case "/close": { + this.#platformSocket.close(); + return reply.text("platform socket closed"); + } + case "/delete": { + const body = await getTextBody(req); + + await this.tasks.delete({ runId: body }); + + return reply.text(`sent delete request: ${body}`); + } + case "/invoke": { + const body = await getTextBody(req); + + await this.tasks.create({ + attemptId: body, + envId: "placeholder", + image: body, + machine: { + cpu: "1", + memory: "100Mi", + }, + }); + + return reply.text(`sent restore request: ${body}`); + } + case "/restore": { + const body = await getTextBody(req); + + const items = body.split("&"); + const image = items[0]; + const baseImageTag = items[1] ?? image; + + // await this.tasks.restore({}); + + return reply.text(`sent restore request: ${body}`); + } + default: { + return reply.empty(404); + } + } + }); + + httpServer.on("clientError", (err, socket) => { + socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); + }); + + httpServer.on("listening", () => { + logger.log("server listening on port", this.options.port); + }); + + return httpServer; + } + + listen() { + this.#httpServer.listen(this.options.port, this.options.host ?? "0.0.0.0"); + } +} + +const provider = new DockerProvider({ + port: HTTP_SERVER_PORT, + tasks: new DockerTaskOperations(), +}); + +provider.listen(); diff --git a/apps/docker-provider/tsconfig.json b/apps/docker-provider/tsconfig.json new file mode 100644 index 000000000..a491b740b --- /dev/null +++ b/apps/docker-provider/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "commonjs", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "paths": { + "@trigger.dev/core/v3": ["../core/src/v3"], + "@trigger.dev/core/v3/*": ["../core/src/v3/*"], + "@trigger.dev/core-apps": ["../core-apps/src"], + "@trigger.dev/core-apps/*": ["../core-apps/src/*"] + } + } +} diff --git a/apps/kubernetes-provider/.env.example b/apps/kubernetes-provider/.env.example new file mode 100644 index 000000000..352a6ca8e --- /dev/null +++ b/apps/kubernetes-provider/.env.example @@ -0,0 +1,7 @@ +HTTP_SERVER_PORT=8060 + +PLATFORM_WS_PORT=8003 +PLATFORM_SECRET=provider-secret + +REGISTRY_FQDN=docker.io +REPO_NAME=task \ No newline at end of file diff --git a/apps/kubernetes-provider/.gitignore b/apps/kubernetes-provider/.gitignore new file mode 100644 index 000000000..5c84119d6 --- /dev/null +++ b/apps/kubernetes-provider/.gitignore @@ -0,0 +1,3 @@ +dist/ +node_modules/ +.env \ No newline at end of file diff --git a/apps/kubernetes-provider/Containerfile b/apps/kubernetes-provider/Containerfile new file mode 100644 index 000000000..078bb9c24 --- /dev/null +++ b/apps/kubernetes-provider/Containerfile @@ -0,0 +1,48 @@ +FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine + +WORKDIR /app + +FROM node-18-alpine AS pruner + +COPY --chown=node:node . . +RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker +RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + + +FROM node-18-alpine AS base + +RUN apk add --no-cache dumb-init + +COPY --chown=node:node .gitignore .gitignore +COPY --from=pruner --chown=node:node /app/out/json/ . +COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml +COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml + +FROM base AS dev-deps +RUN corepack enable +ENV NODE_ENV development + +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile +RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile + +FROM base AS builder +RUN corepack enable + +COPY --from=pruner --chown=node:node /app/out/full/ . +COPY --from=dev-deps --chown=node:node /app/ . +COPY --chown=node:node turbo.json turbo.json + +RUN pnpm run -r --filter '@trigger.dev/core*' build +RUN pnpm run -r --filter kubernetes-provider build:bundle + +FROM base AS runner + +RUN corepack enable +ENV NODE_ENV production + +COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.cjs ./index.cjs + +EXPOSE 8000 + +USER node + +CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.cjs" ] diff --git a/apps/kubernetes-provider/README.md b/apps/kubernetes-provider/README.md new file mode 100644 index 000000000..829c8f215 --- /dev/null +++ b/apps/kubernetes-provider/README.md @@ -0,0 +1,3 @@ +# Kubernetes provider + +The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions. diff --git a/apps/kubernetes-provider/package.json b/apps/kubernetes-provider/package.json new file mode 100644 index 000000000..44830957a --- /dev/null +++ b/apps/kubernetes-provider/package.json @@ -0,0 +1,31 @@ +{ + "name": "kubernetes-provider", + "private": true, + "version": "0.0.1", + "description": "", + "main": "dist/index.cjs", + "type": "module", + "scripts": { + "build": "npm run build:bundle", + "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node", + "build:image": "docker build -f Containerfile . -t kubernetes-provider", + "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", + "start": "tsx src/index.ts", + "typecheck": "tsc --noEmit" + }, + "keywords": [], + "author": "", + "license": "MIT", + "dependencies": { + "@kubernetes/client-node": "^0.20.0", + "@trigger.dev/core": "workspace:*", + "@trigger.dev/core-apps": "workspace:*", + "socket.io-client": "^4.7.4" + }, + "devDependencies": { + "dotenv": "^16.4.2", + "esbuild": "^0.19.11", + "tsx": "^4.7.0", + "typescript": "^5.3.3" + } +} diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts new file mode 100644 index 000000000..e94c32769 --- /dev/null +++ b/apps/kubernetes-provider/src/index.ts @@ -0,0 +1,561 @@ +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import k8s, { BatchV1Api, CoreV1Api, V1Job, V1Pod } from "@kubernetes/client-node"; +import { io, Socket } from "socket.io-client"; +import { + Machine, + ProviderClientToServerEvents, + ProviderServerToClientEvents, +} from "@trigger.dev/core/v3"; +import { HttpReply, SimpleLogger, getTextBody } from "@trigger.dev/core-apps"; + +const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; + +const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8000); +const NODE_NAME = process.env.NODE_NAME || "some-node"; +const POD_NAME = process.env.POD_NAME || "k8s-provider"; + +const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1"; +const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 5080; +const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret"; + +const REGISTRY_FQDN = process.env.REGISTRY_FQDN || "localhost:5000"; +const REPO_NAME = process.env.REPO_NAME || "test"; + +const logger = new SimpleLogger(`[${NODE_NAME}]`); + +type Namespace = { + metadata: { + name: string; + }; +}; + +interface TaskOperations { + create: (...args: any[]) => Promise; + restore: (...args: any[]) => Promise; + delete: (...args: any[]) => Promise; + get: (...args: any[]) => Promise; + index: (...args: any[]) => Promise; +} + +class KubernetesTaskOperations implements TaskOperations { + #namespace: Namespace; + #k8sApi: { + core: CoreV1Api; + batch: BatchV1Api; + }; + + constructor(namespace = "default") { + this.#namespace = { + metadata: { + name: namespace, + }, + }; + + this.#k8sApi = this.#createK8sApi(); + } + + async index(opts: { contentHash: string; imageTag: string }) { + await this.#createJob( + { + metadata: { + name: `task-index-${opts.contentHash}`, + namespace: this.#namespace.metadata.name, + }, + spec: { + completions: 1, + template: { + metadata: { + labels: { + app: "task-index", + }, + }, + spec: { + restartPolicy: "Never", + imagePullSecrets: [ + { + name: "registry-trigger", + }, + ], + containers: [ + { + name: opts.contentHash, + image: opts.imageTag, + ports: [ + { + containerPort: 8000, + }, + ], + resources: { + limits: { + cpu: "100m", + memory: "50Mi", + }, + }, + env: [ + { + name: "DEBUG", + value: "true", + }, + { + name: "INDEX_TASKS", + value: "true", + }, + { + name: "HTTP_SERVER_PORT", + value: "8000", + }, + { + name: "POD_NAME", + valueFrom: { + fieldRef: { + fieldPath: "metadata.name", + }, + }, + }, + { + name: "COORDINATOR_HOST", + valueFrom: { + fieldRef: { + fieldPath: "status.hostIP", + }, + }, + }, + { + name: "MACHINE_NAME", + valueFrom: { + fieldRef: { + fieldPath: "spec.nodeName", + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + this.#namespace + ); + } + + async create(opts: { runId: string; image: string; machine: Machine }) { + await this.#createPod( + { + metadata: { + name: `${opts.runId}-${randomUUID().slice(0, 5)}`, + namespace: this.#namespace.metadata.name, + }, + spec: { + restartPolicy: "Never", + containers: [ + { + name: opts.runId, + image: this.#getImageFromRunId(opts.runId), + ports: [ + { + containerPort: 8000, + }, + ], + // resources: { + // limits: opts.machine, + // }, + env: [ + { + name: "DEBUG", + value: "true", + }, + { + name: "POD_NAME", + valueFrom: { + fieldRef: { + fieldPath: "metadata.name", + }, + }, + }, + { + name: "COORDINATOR_HOST", + valueFrom: { + fieldRef: { + fieldPath: "status.hostIP", + }, + }, + }, + { + name: "NODE_NAME", + valueFrom: { + fieldRef: { + fieldPath: "spec.nodeName", + }, + }, + }, + ], + }, + ], + }, + }, + this.#namespace + ); + } + + async restore(opts: { + runId: string; + image: string; + name: string; + checkpointId: string; + machine: Machine; + }) { + await this.#createPod( + { + metadata: { + name: opts.name, + namespace: this.#namespace.metadata.name, + }, + spec: { + imagePullSecrets: [ + { + name: "regcred", + }, + ], + initContainers: [ + { + name: "pull-base-image", + image: this.#getRestoreImage(opts.runId, opts.checkpointId), + command: ["sleep", "0"], + }, + ], + containers: [ + { + name: opts.runId, + image: this.#getImageFromRunId(opts.runId), + ports: [ + { + containerPort: 8000, + }, + ], + // resources: { + // limits: opts.machine, + // }, + lifecycle: { + postStart: { + httpGet: { + path: "/connect", + port: 8000, + }, + }, + }, + env: [ + { + name: "DEBUG", + value: "true", + }, + { + name: "POD_NAME", + valueFrom: { + fieldRef: { + fieldPath: "metadata.name", + }, + }, + }, + { + name: "COORDINATOR_HOST", + valueFrom: { + fieldRef: { + fieldPath: "status.hostIP", + }, + }, + }, + { + name: "NODE_NAME", + valueFrom: { + fieldRef: { + fieldPath: "spec.nodeName", + }, + }, + }, + ], + }, + ], + }, + }, + this.#namespace + ); + } + + async delete(opts: { runId: string }) { + await this.#deletePod({ + podName: opts.runId, + namespace: this.#namespace, + }); + } + + async get(opts: { runId: string }) { + await this.#getPod(opts.runId, this.#namespace); + } + + #getImageFromRunId(runId: string) { + return `${REGISTRY_FQDN}/${REPO_NAME}:${runId}`; + } + + #getRestoreImage(runId: string, checkpointId: string) { + return `${REGISTRY_FQDN}/${REPO_NAME}:${checkpointId}`; + } + + #createK8sApi() { + const kubeConfig = new k8s.KubeConfig(); + + if (RUNTIME_ENV === "local") { + kubeConfig.loadFromDefault(); + } else if (RUNTIME_ENV === "kubernetes") { + kubeConfig.loadFromCluster(); + } else { + throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`); + } + + return { + core: kubeConfig.makeApiClient(k8s.CoreV1Api), + batch: kubeConfig.makeApiClient(k8s.BatchV1Api), + }; + } + + async #createPod(pod: V1Pod, namespace: Namespace) { + try { + const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod); + logger.debug(res.body); + } catch (err: any) { + if ("body" in err) { + logger.error(err.body); + } else { + logger.error(err); + } + } + } + + async #deletePod(opts: { podName: string; namespace: Namespace }) { + try { + const res = await this.#k8sApi.core.deleteNamespacedPod( + opts.podName, + opts.namespace.metadata.name + ); + logger.debug(res.body); + } catch (err: any) { + if ("body" in err) { + logger.error(err.body); + } else { + logger.error(err); + } + } + } + + async #getPod(podName: string, namespace: Namespace) { + try { + const res = await this.#k8sApi.core.readNamespacedPod(podName, namespace.metadata.name); + logger.debug(res.body); + return res.body; + } catch (err: any) { + if ("body" in err) { + logger.error(err.body); + } else { + logger.error(err); + } + } + } + + async #createJob(job: V1Job, namespace: Namespace) { + try { + const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job); + logger.debug(res.body); + } catch (err: any) { + if ("body" in err) { + logger.error(err.body); + } else { + logger.error(err); + } + } + } +} + +interface Provider { + tasks: TaskOperations; +} + +type KubernetesProviderOptions = { + tasks: KubernetesTaskOperations; + host?: string; + port: number; +}; + +class KubernetesProvider implements Provider { + tasks: KubernetesTaskOperations; + + #httpServer: ReturnType; + #platformSocket: Socket; + + constructor(private options: KubernetesProviderOptions) { + this.tasks = options.tasks; + this.#httpServer = this.#createHttpServer(); + this.#platformSocket = this.#createPlatformSocket(); + } + + #createPlatformSocket() { + const socket: Socket = io( + `ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/provider`, + { + transports: ["websocket"], + auth: { + token: PLATFORM_SECRET, + }, + extraHeaders: { + "x-trigger-provider-type": "kubernetes", + }, + } + ); + + const logger = new SimpleLogger(`[platform][${socket.id ?? "NO_ID"}]`); + + socket.on("connect_error", (err) => { + logger.error(`connect_error: ${err.message}`); + }); + + socket.on("connect", () => { + logger.log("connect"); + }); + + socket.on("disconnect", () => { + logger.log("disconnect"); + }); + + socket.on("GET", async (message) => { + logger.log("[GET]", message); + this.tasks.get({ runId: message.name }); + }); + + socket.on("DELETE", async (message, callback) => { + logger.log("[DELETE]", message); + + callback({ + message: "delete request received", + }); + + this.tasks.delete({ runId: message.name }); + }); + + socket.on("INDEX", async (message) => { + logger.log("[INDEX]", message); + + await this.tasks.index({ + contentHash: message.contentHash, + imageTag: message.imageTag, + }); + }); + + socket.on("INVOKE", async (message) => { + logger.log("[INVOKE]", message); + + await this.tasks.create({ + runId: message.name, + image: message.name, + machine: message.machine, + }); + }); + + socket.on("RESTORE", async (message) => { + logger.log("[RESTORE]", message); + + // await this.tasks.restore({}); + }); + + socket.on("HEALTH", async (message) => { + logger.log("[HEALTH]", message); + }); + + return socket; + } + + #createHttpServer() { + const httpServer = createServer(async (req, res) => { + logger.log(`[${req.method}]`, req.url); + + const reply = new HttpReply(res); + + switch (req.url) { + case "/health": { + return reply.text("ok"); + } + case "/whoami": { + return reply.text(`${POD_NAME}`); + } + case "/close": { + this.#platformSocket.close(); + return reply.text("platform socket closed"); + } + case "/delete": { + const body = await getTextBody(req); + + await this.tasks.delete({ runId: body }); + + return reply.text(`sent delete request: ${body}`); + } + case "/invoke": { + const body = await getTextBody(req); + + await this.tasks.create({ + runId: body, + image: body, + machine: { + cpu: "1", + memory: "100Mi", + }, + }); + + return reply.text(`sent restore request: ${body}`); + } + case "/restore": { + const body = await getTextBody(req); + + const items = body.split("&"); + const image = items[0]; + const baseImageTag = items[1] ?? image; + + await this.tasks.restore({ + runId: image, + name: `${image}-restore`, + image, + checkpointId: baseImageTag, + machine: { + cpu: "1", + memory: "100Mi", + }, + }); + + return reply.text(`sent restore request: ${body}`); + } + default: { + return reply.empty(404); + } + } + }); + + httpServer.on("clientError", (err, socket) => { + socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); + }); + + httpServer.on("listening", () => { + logger.log("server listening on port", this.options.port); + }); + + return httpServer; + } + + listen() { + this.#httpServer.listen(this.options.port, this.options.host ?? "0.0.0.0"); + } +} + +const provider = new KubernetesProvider({ + port: HTTP_SERVER_PORT, + tasks: new KubernetesTaskOperations(), +}); + +provider.listen(); diff --git a/apps/kubernetes-provider/tsconfig.json b/apps/kubernetes-provider/tsconfig.json new file mode 100644 index 000000000..a491b740b --- /dev/null +++ b/apps/kubernetes-provider/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "commonjs", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "paths": { + "@trigger.dev/core/v3": ["../core/src/v3"], + "@trigger.dev/core/v3/*": ["../core/src/v3/*"], + "@trigger.dev/core-apps": ["../core-apps/src"], + "@trigger.dev/core-apps/*": ["../core-apps/src/*"] + } + } +} diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 12831e723..8930fcef6 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -196,3 +196,4 @@ function logError(error: unknown, request?: Request) { const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer); export { wss } from "./v3/handleWebsockets.server"; +export { socketIo } from "./v3/handleSocketIo.server"; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 47065b0c9..e7051e6cf 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -72,6 +72,10 @@ const EnvironmentSchema = z.object({ V3_ENABLED: z.string().default("false"), OTLP_EXPORTER_TRACES_URL: z.string().optional(), LOG_TELEMETRY: z.string().default("true"), + IMAGE_REGISTRY: z.string().default("docker.io"), + IMAGE_REPO: z.string().default("task"), + PROVIDER_SECRET: z.string().default("provider-secret"), + COORDINATOR_SECRET: z.string().default("coordinator-secret"), }); export type Environment = z.infer; diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts new file mode 100644 index 000000000..6cbcc3ffa --- /dev/null +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts @@ -0,0 +1,54 @@ +import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { CreateImageDetailsService } from "~/v3/services/createImageDetails.server"; + +const ParamsSchema = z.object({ + projectRef: z.string(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + // Ensure this is a POST request + if (request.method.toUpperCase() !== "POST") { + return { status: 405, body: "Method Not Allowed" }; + } + + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + // Next authenticate the request + const authenticationResult = await authenticateApiRequest(request); + + if (!authenticationResult) { + logger.info("Invalid or missing api key", { url: request.url }); + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const authenticatedEnv = authenticationResult.environment; + + const { projectRef } = parsedParams.data; + + const rawBody = await request.json(); + const body = CreateImageDetailsRequestBody.safeParse(rawBody); + + if (!body.success) { + return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 }); + } + + const service = new CreateImageDetailsService(); + + const imageDetails = await service.call(projectRef, authenticatedEnv, body.data); + + return json( + { + id: imageDetails.friendlyId, + contentHash: imageDetails.contentHash, + }, + { status: 200 } + ); +} diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts new file mode 100644 index 000000000..4bab1ac39 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts @@ -0,0 +1,62 @@ +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { GetProjectDevResponse } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; + +const ParamsSchema = z.object({ + projectRef: z.string(), +}); + +export async function loader({ request, params }: LoaderFunctionArgs) { + logger.info("projects get prod env", { url: request.url }); + + const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request); + + if (!authenticationResult) { + return json({ error: "Invalid or Missing Access Token" }, { status: 401 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid Params" }, { status: 400 }); + } + + const projectRef = parsedParams.data.projectRef; + + const project = await prisma.project.findUnique({ + where: { + externalRef: projectRef, + organization: { + members: { + some: { + userId: authenticationResult.userId, + }, + }, + }, + }, + include: { + environments: { + where: { + slug: "prod", + }, + }, + }, + }); + + if (!project) { + return json({ error: "Project not found" }, { status: 404 }); + } + + const prodEnvironment = project.environments[0]; + + const result: GetProjectDevResponse = { + apiKey: prodEnvironment.apiKey, + name: project.name, + }; + + return json(result); +} diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts index 92ac8a1ca..cf0fe76ef 100644 --- a/apps/webapp/app/services/worker.server.ts +++ b/apps/webapp/app/services/worker.server.ts @@ -29,6 +29,7 @@ import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralE import { ResumeRunService } from "./runs/resumeRun.server"; import { executionRateLimiter } from "./runExecutionRateLimiter.server"; import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server"; +import { IndexTasksService } from "~/v3/services/indexTasks.server"; const workerCatalog = { indexEndpoint: z.object({ @@ -102,6 +103,10 @@ const workerCatalog = { resumeRun: z.object({ id: z.string(), }), + // v3 tasks + indexTasks: z.object({ + id: z.string(), + }), }; const executionWorkerCatalog = { @@ -425,6 +430,16 @@ function getWorkerQueue() { handler: async (payload, job) => { const service = new ResumeRunService(); + return await service.call(payload.id); + }, + }, + // v3 tasks + indexTasks: { + priority: 0, + maxAttempts: 5, + handler: async (payload, job) => { + const service = new IndexTasksService(); + return await service.call(payload.id); }, }, diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 91e9d362b..455b5b1e2 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -4,6 +4,7 @@ import { $transaction, prisma } from "~/db.server"; import { getSecretStore } from "~/services/secrets/secretStore.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { EnvironmentVariable, ProjectEnvironmentVariable, Repository, Result } from "./repository"; +import { env } from "~/env.server"; function secretKeyProjectPrefix(projectId: string) { return `environmentvariable:${projectId}:`; @@ -402,7 +403,34 @@ export class EnvironmentVariablesRepository implements Repository { return this.getEnvironmentVariables(projectId, environmentId); } - async getEnvironmentVariables( + async #getTriggerEnvironmentVariables(environmentId: string): Promise { + const environment = await this.prismaClient.runtimeEnvironment.findFirst({ + where: { + id: environmentId, + }, + }); + + if (!environment) { + return []; + } + + if (environment.type === "DEVELOPMENT") { + return []; + } + + return [ + { + key: "TRIGGER_API_KEY", + value: environment.apiKey, + }, + { + key: "TRIGGER_API_URL", + value: env.APP_ORIGIN, + }, + ]; + } + + async #getSecretEnvironmentVariables( projectId: string, environmentId: string ): Promise { @@ -424,6 +452,16 @@ export class EnvironmentVariablesRepository implements Repository { }); } + async getEnvironmentVariables( + projectId: string, + environmentId: string + ): Promise { + const secretEnvVars = await this.#getSecretEnvironmentVariables(projectId, environmentId); + const triggerEnvVars = await this.#getTriggerEnvironmentVariables(environmentId); + + return [...secretEnvVars, ...triggerEnvVars]; + } + async delete(projectId: string, userId: string, options: { id: string }): Promise { const project = await this.prismaClient.project.findUnique({ where: { diff --git a/apps/webapp/app/v3/handleSocketIo.server.ts b/apps/webapp/app/v3/handleSocketIo.server.ts new file mode 100644 index 000000000..31f6e5740 --- /dev/null +++ b/apps/webapp/app/v3/handleSocketIo.server.ts @@ -0,0 +1,132 @@ +import { + CoordinatorToPlatformMessages, + PlatformToCoordinatorMessages, + PlatformToProviderMessages, + ProviderToPlatformMessages, + ZodNamespace, + clientWebsocketMessages, + serverWebsocketMessages, +} from "@trigger.dev/core/v3"; +import { Server } from "socket.io"; +import { env } from "~/env.server"; +import { singleton } from "~/utils/singleton"; +import { SharedSocketConnection } from "./sharedSocketConnection"; +import { CreateCheckpointService } from "./services/createCheckpoint.server"; +import { sharedQueueTasks } from "./marqs/sharedQueueConsumer.server"; +import { CompleteAttemptService } from "./services/completeAttempt.server"; +import { CreateBackgroundWorkerService } from "./services/createBackgroundWorker.server"; +import { logger } from "~/services/logger.server"; +import { findEnvironmentById } from "~/models/runtimeEnvironment.server"; + +export const socketIo = singleton("socketIo", initalizeIoServer); + +function initalizeIoServer() { + const io = new Server(); + + io.on("connection", (socket) => { + console.log(`[socket.io][${socket.id}] connection at url: ${socket.request.url}`); + }); + + const coordinatorNamespace = createCoordinatorNamespace(io); + const providerNamespace = createProviderNamespace(io); + const sharedQueueConsumerNamespace = createSharedQueueConsumerNamespace(io); + + return { + io, + coordinatorNamespace, + providerNamespace, + sharedQueueConsumerNamespace, + }; +} + +function createCoordinatorNamespace(io: Server) { + const coordinator = new ZodNamespace({ + io, + name: "coordinator", + authToken: env.COORDINATOR_SECRET, + clientMessages: CoordinatorToPlatformMessages, + serverMessages: PlatformToCoordinatorMessages, + messageHandler: { + READY_FOR_EXECUTION: async (message) => { + const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt(message.attemptId); + + if (!payload) { + return { success: false }; + } else { + return { success: true, payload }; + } + }, + TASK_RUN_COMPLETED: async (message) => { + const completeAttempt = new CompleteAttemptService(); + await completeAttempt.call(message.completion, message.execution); + }, + TASK_HEARTBEAT: async (message) => { + await sharedQueueTasks.taskHeartbeat(message.attemptFriendlyId); + }, + CHECKPOINT_CREATED: async (message) => { + const createCheckpoint = new CreateCheckpointService(); + await createCheckpoint.call(message); + }, + CREATE_WORKER: async (message) => { + try { + const environment = await findEnvironmentById(message.envId); + + if (!environment) { + logger.error("Environment not found", { id: message.envId }); + return { success: false }; + } + + const createCheckpoint = new CreateBackgroundWorkerService(); + await createCheckpoint.call(message.projectRef, environment, { + localOnly: true, + metadata: message.metadata, + }); + + return { success: true }; + } catch (error) { + logger.error("Error while creating worker", { error }); + return { success: false }; + } + }, + }, + }); + + return coordinator.namespace; +} + +function createProviderNamespace(io: Server) { + const provider = new ZodNamespace({ + io, + name: "provider", + authToken: env.PROVIDER_SECRET, + clientMessages: ProviderToPlatformMessages, + serverMessages: PlatformToProviderMessages, + }); + + return provider.namespace; +} + +function createSharedQueueConsumerNamespace(io: Server) { + const sharedQueue = new ZodNamespace({ + io, + name: "shared-queue", + authToken: env.PROVIDER_SECRET, + clientMessages: clientWebsocketMessages, + serverMessages: serverWebsocketMessages, + onConnection: async (socket, handler, sender, logger) => { + const sharedSocketConnection = new SharedSocketConnection( + sharedQueue.namespace, + socket, + logger + ); + + sharedSocketConnection.onClose.attach((closeEvent) => { + logger("Socket closed", { closeEvent }); + }); + + await sharedSocketConnection.initialize(); + }, + }); + + return sharedQueue.namespace; +} diff --git a/apps/webapp/app/v3/marqs.server.ts b/apps/webapp/app/v3/marqs.server.ts index cca3df6db..cfc79d082 100644 --- a/apps/webapp/app/v3/marqs.server.ts +++ b/apps/webapp/app/v3/marqs.server.ts @@ -29,7 +29,7 @@ const constants = { QUEUE_PART: "queue", CONCURRENCY_KEY_PART: "ck", MESSAGE_PART: "message", -}; +} as const; const MessagePayload = z.object({ version: z.literal("1"), @@ -259,6 +259,50 @@ export class MarQS { ); } + public async replaceMessage( + messageId: string, + messageData: Record, + timestamp?: number + ) { + return this.#trace( + "replaceMessage", + async (span) => { + const oldMessage = await this.#readMessage(messageId); + + if (!oldMessage) { + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: oldMessage.queue, + [SemanticAttributes.MESSAGE_ID]: oldMessage.messageId, + [SemanticAttributes.CONCURRENCY_KEY]: oldMessage.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: oldMessage.parentQueue, + }); + + await this.#callAcknowledgeMessage({ + messageKey: `${constants.MESSAGE_PART}:${messageId}`, + visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE, + concurrencyKey: `${oldMessage.queue}:${constants.CURRENT_CONCURRENCY_PART}`, + messageId, + }); + + const newMessage: MessagePayload = { + version: "1", + data: messageData, + queue: oldMessage.queue, + concurrencyKey: oldMessage.concurrencyKey, + timestamp: timestamp ?? Date.now(), + messageId, + parentQueue: oldMessage.parentQueue, + }; + + await this.#callEnqueueMessage(newMessage); + }, + { kind: SpanKind.CONSUMER } + ); + } + async #trace( name: string, fn: (span: Span, abort: () => void) => Promise, @@ -783,9 +827,9 @@ local currentTime = tonumber(ARGV[3]) local messageScore = tonumber(ARGV[4]) -- Check to see if the message is still in the visibilityQueue -local messageVisibility = redis.call('ZSCORE', visibilityQueue, messageId) +local messageVisibility = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0 -if messageVisibility == nil then +if messageVisibility == 0 then return end @@ -820,9 +864,9 @@ local milliseconds = tonumber(ARGV[2]) local maxVisibilityTimeout = tonumber(ARGV[3]) -- Get the current visibility timeout -local currentVisibilityTimeout = redis.call('ZSCORE', visibilityQueue, messageId) +local currentVisibilityTimeout = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0 -if currentVisibilityTimeout == nil then +if currentVisibilityTimeout == 0 then return end diff --git a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts new file mode 100644 index 000000000..4cc6afd9c --- /dev/null +++ b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts @@ -0,0 +1,851 @@ +import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api"; +import { + ProdTaskRunExecution, + ProdTaskRunExecutionPayload, + TaskRunError, + TaskRunExecution, + TaskRunExecutionResult, + TaskRunFailedExecutionResult, + TaskRunSuccessfulExecutionResult, + ZodMessageSender, + serverWebsocketMessages, +} from "@trigger.dev/core/v3"; +import { BackgroundWorker, BackgroundWorkerTask } from "@trigger.dev/database"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { marqs } from "../marqs.server"; +import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server"; +import { CancelAttemptService } from "../services/cancelAttempt.server"; +import { socketIo } from "../handleSocketIo.server"; +import { singleton } from "~/utils/singleton"; + +const tracer = trace.getTracer("sharedQueueConsumer"); + +const MessageBody = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("EXECUTE"), + taskIdentifier: z.string(), + }), + z.object({ + type: z.literal("RESUME"), + completedAttemptIds: z.string().array(), + }), +]); + +type BackgroundWorkerWithTasks = BackgroundWorker & { tasks: BackgroundWorkerTask[] }; + +export type SharedQueueConsumerOptions = { + maximumItemsPerTrace?: number; + traceTimeoutSeconds?: number; +}; + +export class SharedQueueConsumer { + private _backgroundWorkers: Map = new Map(); + private _deprecatedWorkers: Map = new Map(); + private _enabled = false; + private _options: Required; + private _perTraceCountdown: number | undefined; + private _lastNewTrace: Date | undefined; + private _currentSpanContext: Context | undefined; + private _taskFailures: number = 0; + private _taskSuccesses: number = 0; + private _currentSpan: Span | undefined; + private _endSpanInNextIteration = false; + private _tasks = sharedQueueTasks; + private _inProgressAttempts: Map = new Map(); // Keys are task attempt friendly IDs, values are TaskRun ids/queue message ids + + constructor( + private _sender: ZodMessageSender, + options: SharedQueueConsumerOptions = {} + ) { + this._options = { + maximumItemsPerTrace: options.maximumItemsPerTrace ?? 1_000, // 1k items per trace + traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds + }; + } + + // This method is called when a background worker is deprecated and will no longer be used unless a run is locked to it + public async deprecateBackgroundWorker(id: string) { + const backgroundWorker = this._backgroundWorkers.get(id); + + if (!backgroundWorker) { + return; + } + + this._deprecatedWorkers.set(id, backgroundWorker); + this._backgroundWorkers.delete(id); + } + + public async registerBackgroundWorker(id: string, envId?: string) { + if (!envId) { + logger.error("Environment ID is required for background worker registration", { + backgroundWorkerId: id, + }); + return; + } + + const backgroundWorker = await prisma.backgroundWorker.findUnique({ + where: { + friendlyId: id, + runtimeEnvironmentId: envId, + }, + include: { + tasks: true, + }, + }); + + if (!backgroundWorker) { + return; + } + + this._backgroundWorkers.set(backgroundWorker.id, backgroundWorker); + + logger.debug("Registered background worker", { backgroundWorker: backgroundWorker.id }); + + // Start reading from the queue if we haven't already + this.#enable(); + } + + public async start() { + this.#enable(); + } + + public async stop(reason: string = "Provider disconnected") { + if (!this._enabled) { + return; + } + + logger.debug("Stopping shared queue consumer"); + this._enabled = false; + + // TODO: think about automatic prod cancellation + + // We need to cancel all the in progress task run attempts and ack the messages so they will stop processing + // await this.#cancelInProgressAttempts(reason); + } + + async #cancelInProgressAttempts(reason: string) { + const service = new CancelAttemptService(); + + const cancelledAt = new Date(); + + const inProgressAttempts = new Map(this._inProgressAttempts); + + this._inProgressAttempts.clear(); + + for (const [attemptId, messageId] of inProgressAttempts) { + await this.#cancelInProgressAttempt(attemptId, messageId, service, cancelledAt, reason); + } + } + + async #cancelInProgressAttempt( + attemptId: string, + messageId: string, + cancelAttemptService: CancelAttemptService, + cancelledAt: Date, + reason: string + ) { + try { + await cancelAttemptService.call(attemptId, messageId, cancelledAt, reason); + } catch (e) { + logger.error("Failed to cancel in progress attempt", { + attemptId, + messageId, + error: e, + }); + } + } + + #enable() { + if (this._enabled) { + return; + } + + this._enabled = true; + this._perTraceCountdown = this._options.maximumItemsPerTrace; + this._lastNewTrace = new Date(); + this._taskFailures = 0; + this._taskSuccesses = 0; + + this.#doWork().finally(() => {}); + } + + async #doWork() { + if (!this._enabled) { + return; + } + + // Check if the trace has expired + if ( + this._perTraceCountdown === 0 || + Date.now() - this._lastNewTrace!.getTime() > this._options.traceTimeoutSeconds * 1000 || + this._currentSpanContext === undefined || + this._endSpanInNextIteration + ) { + if (this._currentSpan) { + this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures); + this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses); + + this._currentSpan.end(); + } + + // Create a new trace + this._currentSpan = tracer.startSpan( + "SharedQueueConsumer.doWork()", + { + kind: SpanKind.CONSUMER, + }, + ROOT_CONTEXT + ); + + // Get the span trace context + this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan); + + this._perTraceCountdown = this._options.maximumItemsPerTrace; + this._lastNewTrace = new Date(); + this._taskFailures = 0; + this._taskSuccesses = 0; + this._endSpanInNextIteration = false; + } + + return context.with(this._currentSpanContext ?? ROOT_CONTEXT, async () => { + await this.#doWorkInternal(); + this._perTraceCountdown = this._perTraceCountdown! - 1; + }); + } + + async #doWorkInternal() { + // Attempt to dequeue a message from the shared queue + // If no message is available, reschedule the worker to run again in 1 second + // If a message is available, find the BackgroundWorkerTask that matches the message's taskIdentifier + // If no matching task is found, nack the message and reschedule the worker to run again in 1 second + // If the matching task is found, create the task attempt and lock the task run, then send the task run to the client + // Store the message as a processing message + // If the websocket connection disconnects before the task run is completed, nack the message + // When the task run completes, ack the message + // Using a heartbeat mechanism, if the client keeps responding with a heartbeat, we'll keep the message processing and increase the visibility timeout. + + const message = await marqs?.dequeueMessageInSharedQueue(); + + if (!message) { + setTimeout(() => this.#doWork(), 1000); + return; + } + + console.log("dequeueMessageInSharedQueue()", message); + + const envId = this.#envIdFromQueue(message.queue); + + const environment = await prisma.runtimeEnvironment.findUnique({ + include: { + organization: true, + project: true, + }, + where: { + id: envId, + }, + }); + + if (!environment) { + logger.error("Environment not found", { + queueMessage: message.data, + envId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const messageBody = MessageBody.safeParse(message.data); + + if (!messageBody.success) { + logger.error("Failed to parse message", { + queueMessage: message.data, + error: messageBody.error, + env: environment, + }); + + await marqs?.acknowledgeMessage(message.messageId); + + setTimeout(() => this.#doWork(), 100); + return; + } + + switch (messageBody.data.type) { + case "EXECUTE": { + const existingTaskRun = await prisma.taskRun.findUnique({ + where: { + id: message.messageId, + }, + }); + + if (!existingTaskRun) { + logger.error("No existing task run", { + queueMessage: message.data, + messageId: message.messageId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const backgroundWorker = await prisma.backgroundWorker.findFirst({ + where: { + runtimeEnvironmentId: existingTaskRun.runtimeEnvironmentId, + projectId: existingTaskRun.projectId, + imageDetails: { + some: {}, + }, + }, + orderBy: { + updatedAt: "desc", + }, + include: { + tasks: true, + imageDetails: true, + }, + }); + + if (!backgroundWorker) { + logger.error("No matching background worker found for task run", { + queueMessage: message.data, + messageId: message.messageId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const backgroundTask = backgroundWorker.tasks.find( + (task) => task.slug === existingTaskRun.taskIdentifier + ); + + if (!backgroundTask) { + logger.warn("No matching background task found for task run", { + taskRun: existingTaskRun.id, + taskIdentifier: existingTaskRun.taskIdentifier, + backgroundWorker: backgroundWorker.id, + taskSlugs: backgroundWorker.tasks.map((task) => task.slug), + }); + + await marqs?.acknowledgeMessage(message.messageId); + + setTimeout(() => this.#doWork(), 100); + return; + } + + const lockedTaskRun = await prisma.taskRun.update({ + where: { + id: message.messageId, + }, + data: { + lockedAt: new Date(), + lockedById: backgroundTask.id, + }, + include: { + attempts: { + take: 1, + orderBy: { number: "desc" }, + }, + tags: true, + }, + }); + + if (!lockedTaskRun) { + logger.warn("Failed to lock task run", { + taskRun: existingTaskRun.id, + taskIdentifier: existingTaskRun.taskIdentifier, + backgroundWorker: backgroundWorker.id, + messageId: message.messageId, + }); + + await marqs?.acknowledgeMessage(message.messageId); + + setTimeout(() => this.#doWork(), 100); + return; + } + + const queue = await prisma.taskQueue.findUnique({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: environment.id, + name: lockedTaskRun.queue, + }, + }, + }); + + if (!queue) { + await marqs?.nackMessage(message.messageId); + setTimeout(() => this.#doWork(), 1000); + return; + } + + if (!this._enabled) { + await marqs?.nackMessage(message.messageId); + return; + } + + const taskRunAttempt = await prisma.taskRunAttempt.create({ + data: { + number: lockedTaskRun.attempts[0] ? lockedTaskRun.attempts[0].number + 1 : 1, + friendlyId: generateFriendlyId("attempt"), + taskRunId: lockedTaskRun.id, + startedAt: new Date(), + backgroundWorkerId: backgroundTask.workerId, + backgroundWorkerTaskId: backgroundTask.id, + status: "PENDING" as const, + queueId: queue.id, + }, + }); + + try { + await this._sender.send("BACKGROUND_WORKER_MESSAGE", { + backgroundWorkerId: backgroundWorker.friendlyId, + data: { + type: "SCHEDULE_ATTEMPT", + id: taskRunAttempt.id, + image: backgroundWorker.imageDetails[0].tag, + envId: environment.id, + }, + }); + + this._inProgressAttempts.set(taskRunAttempt.friendlyId, message.messageId); + } catch (e) { + if (e instanceof Error) { + this._currentSpan?.recordException(e); + } else { + this._currentSpan?.recordException(new Error(String(e))); + } + + this._endSpanInNextIteration = true; + + // We now need to unlock the task run and delete the task run attempt + await prisma.$transaction([ + prisma.taskRun.update({ + where: { + id: lockedTaskRun.id, + }, + data: { + lockedAt: null, + lockedById: null, + }, + }), + prisma.taskRunAttempt.delete({ + where: { + id: taskRunAttempt.id, + }, + }), + ]); + + // Finally we need to nack the message so it can be retried + await marqs?.nackMessage(message.messageId); + } finally { + setTimeout(() => this.#doWork(), 100); + } + break; + } + // Resume after dependency completed with no remaining retries + case "RESUME": { + if (messageBody.data.completedAttemptIds.length < 1) { + logger.error("No attempt IDs provided", { + queueMessage: message.data, + messageId: message.messageId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const resumableRun = await prisma.taskRun.findFirst({ + where: { + id: message.messageId, + }, + include: { + attempts: { + orderBy: { + createdAt: "desc", + }, + take: 1, + include: { + checkpoints: { + take: 1, + orderBy: { + createdAt: "desc", + }, + }, + }, + }, + }, + }); + + const resumableAttempt = resumableRun?.attempts[0]; + + if (!resumableAttempt) { + logger.error("Task run attempt to resume not found", { + queueMessage: message.data, + messageId: message.messageId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const backgroundWorker = await prisma.backgroundWorker.findFirst({ + where: { + runtimeEnvironmentId: resumableRun.runtimeEnvironmentId, + projectId: resumableRun.projectId, + imageDetails: { + some: {}, + }, + }, + orderBy: { + createdAt: "desc", + }, + include: { + tasks: true, + imageDetails: { + take: 1, + orderBy: { + updatedAt: "desc", + }, + }, + }, + }); + + if (!backgroundWorker) { + logger.error("No matching background worker found for task run", { + queueMessage: message.data, + messageId: message.messageId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const backgroundTask = backgroundWorker.tasks.find( + (task) => task.slug === resumableRun.taskIdentifier + ); + + if (!backgroundTask) { + logger.warn("No matching background task found for task run", { + taskRun: resumableRun.id, + taskIdentifier: resumableRun.taskIdentifier, + backgroundWorker: backgroundWorker.id, + taskSlugs: backgroundWorker.tasks.map((task) => task.slug), + }); + + await marqs?.acknowledgeMessage(message.messageId); + + setTimeout(() => this.#doWork(), 100); + return; + } + + const queue = await prisma.taskQueue.findUnique({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: environment.id, + name: resumableRun.queue, + }, + }, + }); + + if (!queue) { + await marqs?.nackMessage(message.messageId); + setTimeout(() => this.#doWork(), 1000); + return; + } + + if (!this._enabled) { + await marqs?.nackMessage(message.messageId); + return; + } + + const completions: TaskRunExecutionResult[] = []; + const executions: TaskRunExecution[] = []; + + for (const completedAttemptId of messageBody.data.completedAttemptIds) { + const completedAttempt = await prisma.taskRunAttempt.findUnique({ + where: { + id: completedAttemptId, + taskRun: { + lockedAt: { + not: null, + }, + lockedById: { + not: null, + }, + }, + }, + }); + + if (!completedAttempt) { + logger.error("Completed attempt not found", { + queueMessage: message.data, + messageId: message.messageId, + }); + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + const completion = await this._tasks.getCompletionPayloadFromAttempt(completedAttempt.id); + + if (!completion) { + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + completions.push(completion); + + const executionPayload = await this._tasks.getExecutionPayloadFromAttempt( + completedAttempt.id, + false + ); + + if (!executionPayload) { + await marqs?.acknowledgeMessage(message.messageId); + setTimeout(() => this.#doWork(), 100); + return; + } + + executions.push(executionPayload.execution); + } + + try { + const latestCheckpoint = resumableAttempt.checkpoints[0]; + + if (!latestCheckpoint) { + // No checkpoint means the task should still be running + // We can broadcast to all coordinators to resume immediately + socketIo.coordinatorNamespace.emit("RESUME", { + version: "v1", + attemptId: resumableAttempt.id, + image: backgroundWorker.imageDetails[0].tag, + completions, + executions, + }); + } else { + // There's a checkpoint we need to restore first + // TODO: Send RESUME message once the restored task has checked in + socketIo.providerNamespace.emit("RESTORE", { + version: "v1", + id: latestCheckpoint.id, + attemptId: latestCheckpoint.attemptId, + type: latestCheckpoint.type, + location: latestCheckpoint.location, + reason: latestCheckpoint.reason ?? undefined, + }); + } + } catch (e) { + if (e instanceof Error) { + this._currentSpan?.recordException(e); + } else { + this._currentSpan?.recordException(new Error(String(e))); + } + + this._endSpanInNextIteration = true; + + // Finally we need to nack the message so it can be retried + await marqs?.nackMessage(message.messageId); + } finally { + setTimeout(() => this.#doWork(), 100); + } + break; + } + } + } + + #envIdFromQueue(queueName: string) { + return queueName.split(":")[1]; + } +} + +class SharedQueueTasks { + async getCompletionPayloadFromAttempt(id: string): Promise { + const attempt = await prisma.taskRunAttempt.findUnique({ + where: { + id, + status: { + in: ["COMPLETED", "FAILED"], + }, + }, + include: { + backgroundWorker: true, + backgroundWorkerTask: true, + taskRun: { + include: { + runtimeEnvironment: { + include: { + organization: true, + project: true, + }, + }, + tags: true, + }, + }, + queue: true, + }, + }); + + if (!attempt) { + logger.error("No completed attempt found", { id }); + return; + } + + const ok = attempt.status === "COMPLETED"; + + if (ok) { + const success: TaskRunSuccessfulExecutionResult = { + ok, + id: attempt.friendlyId, + output: attempt.output ?? "", + outputType: attempt.outputType, + }; + return success; + } else { + const failure: TaskRunFailedExecutionResult = { + ok, + id: attempt.friendlyId, + error: attempt.error as TaskRunError, + }; + return failure; + } + } + + async getExecutionPayloadFromAttempt( + id: string, + setToExecuting = true + ): Promise { + const attempt = await prisma.taskRunAttempt.findUnique({ + where: { + id, + }, + include: { + backgroundWorker: true, + backgroundWorkerTask: true, + taskRun: { + include: { + runtimeEnvironment: { + include: { + organization: true, + project: true, + }, + }, + tags: true, + batchItem: { + include: { + batchTaskRun: true, + }, + }, + }, + }, + queue: true, + }, + }); + + if (!attempt) { + logger.error("No attempt found", { id }); + return; + } + + if (setToExecuting) { + await prisma.taskRunAttempt.update({ + where: { + id, + }, + data: { + status: "EXECUTING", + }, + }); + } + + const { backgroundWorkerTask, taskRun, queue } = attempt; + + const execution: ProdTaskRunExecution = { + task: { + id: backgroundWorkerTask.slug, + filePath: backgroundWorkerTask.filePath, + exportName: backgroundWorkerTask.exportName, + }, + attempt: { + id: attempt.friendlyId, + number: attempt.number, + startedAt: attempt.startedAt ?? attempt.createdAt, + backgroundWorkerId: attempt.backgroundWorkerId, + backgroundWorkerTaskId: attempt.backgroundWorkerTaskId, + status: "EXECUTING" as const, + }, + run: { + id: taskRun.friendlyId, + payload: taskRun.payload, + payloadType: taskRun.payloadType, + context: taskRun.context, + createdAt: taskRun.createdAt, + tags: taskRun.tags.map((tag) => tag.name), + isTest: taskRun.isTest, + }, + queue: { + id: queue.friendlyId, + name: queue.name, + }, + environment: { + id: taskRun.runtimeEnvironment.id, + slug: taskRun.runtimeEnvironment.slug, + type: taskRun.runtimeEnvironment.type, + }, + organization: { + id: taskRun.runtimeEnvironment.organization.id, + slug: taskRun.runtimeEnvironment.organization.slug, + name: taskRun.runtimeEnvironment.organization.title, + }, + project: { + id: taskRun.runtimeEnvironment.project.id, + ref: taskRun.runtimeEnvironment.project.externalRef, + slug: taskRun.runtimeEnvironment.project.slug, + name: taskRun.runtimeEnvironment.project.name, + }, + batch: taskRun.batchItem?.batchTaskRun + ? { id: taskRun.batchItem.batchTaskRun.friendlyId } + : undefined, + worker: { + id: attempt.backgroundWorkerId, + contentHash: attempt.backgroundWorker.contentHash, + version: attempt.backgroundWorker.version, + }, + }; + + const environmentRepository = new EnvironmentVariablesRepository(); + const variables = await environmentRepository.getEnvironmentVariables( + attempt.taskRun.runtimeEnvironment.projectId, + attempt.taskRun.runtimeEnvironmentId + ); + + const payload: ProdTaskRunExecutionPayload = { + execution, + traceContext: taskRun.traceContext as Record, + environment: variables.reduce((acc: Record, curr) => { + acc[curr.key] = curr.value; + return acc; + }, {}), + }; + + return payload; + } + + async taskHeartbeat(attemptFriendlyId: string, seconds: number = 60) { + const taskRunAttempt = await prisma.taskRunAttempt.findUnique({ + where: { friendlyId: attemptFriendlyId }, + }); + + if (!taskRunAttempt) { + return; + } + + await marqs?.heartbeatMessage(taskRunAttempt.taskRunId, seconds); + } +} + +export const sharedQueueTasks = singleton("sharedQueueTasks", () => new SharedQueueTasks()); diff --git a/apps/webapp/app/v3/services/cancelAttempt.server.ts b/apps/webapp/app/v3/services/cancelAttempt.server.ts index 9692fb0d1..468f93de8 100644 --- a/apps/webapp/app/v3/services/cancelAttempt.server.ts +++ b/apps/webapp/app/v3/services/cancelAttempt.server.ts @@ -4,14 +4,26 @@ import { marqs } from "../marqs.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; +import { PrismaClientOrTransaction, prisma } from "~/db.server"; + export class CancelAttemptService extends BaseService { public async call( attemptId: string, taskRunId: string, cancelledAt: Date, reason: string, - environment: AuthenticatedEnvironment + env?: AuthenticatedEnvironment ) { + let environment: AuthenticatedEnvironment | undefined = env; + + if (!environment) { + environment = await getAuthenticatedEnvironmentFromAttempt(attemptId); + + if (!environment) { + return; + } + } + return await this.traceWithEnv("call()", environment, async (span) => { span.setAttribute("taskRunId", taskRunId); span.setAttribute("attemptId", attemptId); @@ -56,3 +68,32 @@ export class CancelAttemptService extends BaseService { }); } } + +async function getAuthenticatedEnvironmentFromAttempt( + friendlyId: string, + prismaClient?: PrismaClientOrTransaction +) { + const taskRunAttempt = await (prismaClient ?? prisma).taskRunAttempt.findUnique({ + where: { + friendlyId, + }, + include: { + taskRun: { + include: { + runtimeEnvironment: { + include: { + organization: true, + project: true, + }, + }, + }, + }, + }, + }); + + if (!taskRunAttempt) { + return; + } + + return taskRunAttempt?.taskRun.runtimeEnvironment; +} diff --git a/apps/webapp/app/v3/services/completeAttempt.server.ts b/apps/webapp/app/v3/services/completeAttempt.server.ts index 9855821df..19bbbcea0 100644 --- a/apps/webapp/app/v3/services/completeAttempt.server.ts +++ b/apps/webapp/app/v3/services/completeAttempt.server.ts @@ -11,13 +11,14 @@ import { eventRepository } from "../eventRepository.server"; import { marqs } from "../marqs.server"; import { BaseService } from "./baseService.server"; import { Attributes } from "@opentelemetry/api"; +import { logger } from "~/services/logger.server"; export class CompleteAttemptService extends BaseService { public async call( completion: TaskRunExecutionResult, execution: TaskRunExecution, - env: AuthenticatedEnvironment - ): Promise<"ACKNOWLEDGED" | "RETRIED"> { + env?: AuthenticatedEnvironment + ): Promise<"ACKNOWLEDGED" | "RETRIED" | "FAILED"> { const taskRunAttempt = completion.ok ? await this._prisma.taskRunAttempt.update({ where: { friendlyId: completion.id }, @@ -28,7 +29,17 @@ export class CompleteAttemptService extends BaseService { outputType: completion.outputType, }, include: { - taskRun: true, + taskRun: { + include: { + batchItem: true, + dependency: { + include: { + dependentAttempt: true, + dependentBatchRun: true, + }, + }, + }, + }, backgroundWorkerTask: true, }, }) @@ -40,7 +51,17 @@ export class CompleteAttemptService extends BaseService { error: completion.error, }, include: { - taskRun: true, + taskRun: { + include: { + batchItem: true, + dependency: { + include: { + dependentAttempt: true, + dependentBatchRun: true, + }, + }, + }, + }, backgroundWorkerTask: true, }, }); @@ -53,7 +74,10 @@ export class CompleteAttemptService extends BaseService { } : undefined; + const environment = env ?? (await this.#getEnvironment(execution.environment.id)); + const retryAt = new Date(completion.retry.timestamp); + // Retry the task run await eventRepository.recordEvent( retryConfig?.maxAttempts @@ -61,7 +85,7 @@ export class CompleteAttemptService extends BaseService { : `Retry #${execution.attempt.number} delay`, { taskSlug: taskRunAttempt.taskRun.taskIdentifier, - environment: env, + environment, attributes: { metadata: this.#generateMetadataAttributesForNextAttempt(execution), properties: { @@ -85,10 +109,29 @@ export class CompleteAttemptService extends BaseService { } ); - await marqs?.nackMessage(taskRunAttempt.taskRunId, completion.retry.timestamp); + logger.debug("Retrying", { taskRun: taskRunAttempt.taskRun.friendlyId }); + + if (environment.type === "DEVELOPMENT") { + // This is already an EXECUTE message so we can just NACK + await marqs?.nackMessage(taskRunAttempt.taskRunId, completion.retry.timestamp); + } else { + // We have to replace a potential RESUME with EXECUTE to correctly retry the attempt + await marqs?.replaceMessage( + taskRunAttempt.taskRunId, + { + type: "EXECUTE", + taskIdentifier: taskRunAttempt.taskRun.taskIdentifier, + }, + completion.retry.timestamp + ); + } return "RETRIED"; - } else { + } + // Attempt succeeded or this was the last retry + else { + logger.debug("Completed attempt, ACKing message", taskRunAttempt); + await marqs?.acknowledgeMessage(taskRunAttempt.taskRunId); // Now we need to "complete" the task run event/span @@ -109,6 +152,124 @@ export class CompleteAttemptService extends BaseService { }); } + const { batchItem, dependency } = taskRunAttempt.taskRun; + + // This run is part of a batch so we should update its status + if (batchItem) { + logger.debug("Completing attempt with batch item", { batchItem }); + + await this._prisma.batchTaskRunItem.update({ + where: { + id: batchItem.id, + }, + data: { + status: completion.ok ? "COMPLETED" : "FAILED", + }, + }); + + const finalizedBatchRun = await this._prisma.batchTaskRun.findFirst({ + where: { + id: batchItem.batchTaskRunId, + dependentTaskAttemptId: { + not: null, + }, + items: { + every: { + status: { + not: "PENDING", + }, + }, + }, + }, + include: { + dependentTaskAttempt: true, + items: { + include: { + taskRun: { + include: { + attempts: { + orderBy: { + completedAt: "desc", + }, + take: 1, + select: { + id: true, + }, + }, + }, + }, + }, + }, + }, + }); + + // This batch has a dependent attempt and just finalized, we should resume that attempt + if (finalizedBatchRun && finalizedBatchRun.dependentTaskAttempt) { + const environment = + env ?? (await this.#getEnvironment(taskRunAttempt.taskRun.runtimeEnvironmentId)); + + if (!environment) { + logger.error("Environment not found", { + attemptId: taskRunAttempt.id, + envId: taskRunAttempt.taskRun.runtimeEnvironmentId, + }); + return "FAILED"; + } + + if (environment.type === "DEVELOPMENT") { + return "ACKNOWLEDGED"; + } + + await marqs?.replaceMessage(finalizedBatchRun.dependentTaskAttempt.taskRunId, { + type: "RESUME", + completedAttemptIds: finalizedBatchRun.items.map( + (item) => item.taskRun.attempts[0]?.id + ), + }); + } + } + + if (dependency) { + logger.debug("Completing attempt with dependency", { dependency }); + + const environment = + env ?? (await this.#getEnvironment(taskRunAttempt.taskRun.runtimeEnvironmentId)); + + if (!environment) { + logger.error("Environment not found", { + attemptId: taskRunAttempt.id, + envId: taskRunAttempt.taskRun.runtimeEnvironmentId, + }); + return "FAILED"; + } + + if (environment.type === "DEVELOPMENT") { + return "ACKNOWLEDGED"; + } + + if (dependency.dependentAttempt) { + const dependentRun = await this._prisma.taskRun.findFirst({ + where: { + id: dependency.dependentAttempt.taskRunId, + }, + }); + + if (!dependentRun) { + logger.error("Dependent task run does not exist", { + attemptId: taskRunAttempt.id, + envId: taskRunAttempt.taskRun.runtimeEnvironmentId, + taskRunId: dependency.taskRunId, + }); + return "FAILED"; + } + + await marqs?.replaceMessage(dependentRun.id, { + type: "RESUME", + completedAttemptIds: [taskRunAttempt.id], + }); + } + } + return "ACKNOWLEDGED"; } } @@ -123,4 +284,16 @@ export class CompleteAttemptService extends BaseService { return flattenAttributes(context, "ctx"); } + + async #getEnvironment(id: string) { + return await this._prisma.runtimeEnvironment.findUniqueOrThrow({ + where: { + id, + }, + include: { + project: true, + organization: true, + }, + }); + } } diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 5a512e2a8..3b3d911b7 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -5,6 +5,7 @@ import { logger } from "~/services/logger.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { marqs } from "../marqs.server"; import { BaseService } from "./baseService.server"; +import { $transaction } from "~/db.server"; export class CreateBackgroundWorkerService extends BaseService { public async call( @@ -15,100 +16,123 @@ export class CreateBackgroundWorkerService extends BaseService { return this.traceWithEnv("call", environment, async (span) => { span.setAttribute("projectRef", projectRef); - const project = await this._prisma.project.findUniqueOrThrow({ - where: { - externalRef: projectRef, - environments: { - some: { - id: environment.id, + const backgroundWorker = await $transaction(this._prisma, async (tx) => { + const project = await this._prisma.project.findUniqueOrThrow({ + where: { + externalRef: projectRef, + environments: { + some: { + id: environment.id, + }, }, }, - }, - include: { - backgroundWorkers: { - where: { - runtimeEnvironmentId: environment.id, + include: { + backgroundWorkers: { + where: { + runtimeEnvironmentId: environment.id, + }, + orderBy: { + createdAt: "desc", + }, + take: 1, }, - orderBy: { - createdAt: "desc", - }, - take: 1, }, - }, - }); + }); - const latestBackgroundWorker = project.backgroundWorkers[0]; + const latestBackgroundWorker = project.backgroundWorkers[0]; - if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) { - return latestBackgroundWorker; - } + if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) { + return latestBackgroundWorker; + } - const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version); + const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version); - logger.debug(`Creating background worker`, { - nextVersion, - lastVersion: project.backgroundWorkers[0]?.version, - }); + logger.debug(`Creating background worker`, { + nextVersion, + lastVersion: project.backgroundWorkers[0]?.version, + }); - const backgroundWorker = await this._prisma.backgroundWorker.create({ - data: { - friendlyId: generateFriendlyId("worker"), - version: nextVersion, - runtimeEnvironmentId: environment.id, - projectId: project.id, - metadata: body.metadata, - contentHash: body.metadata.contentHash, + const backgroundWorker = await this._prisma.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + version: nextVersion, + runtimeEnvironmentId: environment.id, + projectId: project.id, + metadata: body.metadata, + contentHash: body.metadata.contentHash, cliVersion: body.metadata.cliPackageVersion, sdkVersion: body.metadata.packageVersion, - }, + }, + }); + + if (environment.type !== "DEVELOPMENT") { + await this._prisma.imageDetails.update({ + where: { + projectId_runtimeEnvironmentId_contentHash: { + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + contentHash: backgroundWorker.contentHash, + }, + }, + data: { + backgroundWorkerId: backgroundWorker.id, + }, + }); + } + + for (const task of body.metadata.tasks) { + await this._prisma.backgroundWorkerTask.create({ + data: { + friendlyId: generateFriendlyId("task"), + projectId: project.id, + runtimeEnvironmentId: environment.id, + workerId: backgroundWorker.id, + slug: task.id, + filePath: task.filePath, + exportName: task.exportName, + retryConfig: task.retry, + queueConfig: task.queue, + }, + }); + + const queueName = task.queue?.name ?? `task/${task.id}`; + + const taskQueue = await this._prisma.taskQueue.upsert({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: environment.id, + name: queueName, + }, + }, + update: { + concurrencyLimit: task.queue?.concurrencyLimit, + rateLimit: task.queue?.rateLimit, + }, + create: { + friendlyId: generateFriendlyId("queue"), + name: queueName, + concurrencyLimit: task.queue?.concurrencyLimit, + runtimeEnvironmentId: environment.id, + projectId: project.id, + rateLimit: task.queue?.rateLimit, + type: task.queue?.name ? "NAMED" : "VIRTUAL", + }, + }); + + if (taskQueue.concurrencyLimit) { + await marqs?.updateQueueConcurrency( + environment, + taskQueue.name, + taskQueue.concurrencyLimit + ); + } + } + + return backgroundWorker; }); - for (const task of body.metadata.tasks) { - await this._prisma.backgroundWorkerTask.create({ - data: { - friendlyId: generateFriendlyId("task"), - projectId: project.id, - runtimeEnvironmentId: environment.id, - workerId: backgroundWorker.id, - slug: task.id, - filePath: task.filePath, - exportName: task.exportName, - retryConfig: task.retry, - queueConfig: task.queue, - }, - }); - - const queueName = task.queue?.name ?? `task/${task.id}`; - - const taskQueue = await this._prisma.taskQueue.upsert({ - where: { - runtimeEnvironmentId_name: { - runtimeEnvironmentId: environment.id, - name: queueName, - }, - }, - update: { - concurrencyLimit: task.queue?.concurrencyLimit, - rateLimit: task.queue?.rateLimit, - }, - create: { - friendlyId: generateFriendlyId("queue"), - name: queueName, - concurrencyLimit: task.queue?.concurrencyLimit, - runtimeEnvironmentId: environment.id, - projectId: project.id, - rateLimit: task.queue?.rateLimit, - type: task.queue?.name ? "NAMED" : "VIRTUAL", - }, - }); - - if (taskQueue.concurrencyLimit) { - await marqs?.updateQueueConcurrency( - environment, - taskQueue.name, - taskQueue.concurrencyLimit - ); - } + if (!backgroundWorker) { + throw new Error("Failed to create background worker"); } return backgroundWorker; diff --git a/apps/webapp/app/v3/services/createCheckpoint.server.ts b/apps/webapp/app/v3/services/createCheckpoint.server.ts new file mode 100644 index 000000000..02b228baa --- /dev/null +++ b/apps/webapp/app/v3/services/createCheckpoint.server.ts @@ -0,0 +1,46 @@ +import { CoordinatorToPlatformEvents } from "@trigger.dev/core/v3"; +import type { Checkpoint } from "@trigger.dev/database"; +import { PrismaClient, prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { marqs } from "../marqs.server"; + +export class CreateCheckpointService { + #prismaClient: PrismaClient; + + constructor(prismaClient: PrismaClient = prisma) { + this.#prismaClient = prismaClient; + } + + public async call( + params: Parameters[0] + ): Promise { + const attempt = await this.#prismaClient.taskRunAttempt.findUniqueOrThrow({ + where: { + id: params.attemptId, + }, + include: { + taskRun: true, + }, + }); + + logger.debug(`Creating checkpoint`, params); + + const checkpoint = await this.#prismaClient.checkpoint.create({ + data: { + friendlyId: generateFriendlyId("checkpoint"), + runtimeEnvironmentId: attempt.taskRun.runtimeEnvironmentId, + projectId: attempt.taskRun.projectId, + attemptId: attempt.id, + location: params.location, + type: params.docker ? "DOCKER" : "KUBERNETES", + reason: params.reason, + }, + }); + + // TODO: Can't heartbeat when checkpointed, so we ACK to prevent automatic requeue + // await marqs?.acknowledgeMessage(attempt.taskRunId); + + return checkpoint; + } +} diff --git a/apps/webapp/app/v3/services/createImageDetails.server.ts b/apps/webapp/app/v3/services/createImageDetails.server.ts new file mode 100644 index 000000000..ba55b8f37 --- /dev/null +++ b/apps/webapp/app/v3/services/createImageDetails.server.ts @@ -0,0 +1,75 @@ +import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3"; +import type { ImageDetails } from "@trigger.dev/database"; +import { PrismaClient, prisma } from "~/db.server"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { env } from "~/env.server"; +import { workerQueue } from "~/services/worker.server"; + +function escapeStringForRegex(rawString: string) { + return rawString.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&").replace(/-/g, "\\x2d"); +} + +export class CreateImageDetailsService { + #prismaClient: PrismaClient; + + constructor(prismaClient: PrismaClient = prisma) { + this.#prismaClient = prismaClient; + } + + public async call( + projectRef: string, + environment: AuthenticatedEnvironment, + body: CreateImageDetailsRequestBody + ): Promise { + const allowedTagPrefix = escapeStringForRegex(`${env.IMAGE_REGISTRY}/${env.IMAGE_REPO}:`); + + if (!body.metadata.imageTag.match(`^${allowedTagPrefix}`)) { + if (env.NODE_ENV !== "development") { + throw new Error("Forbidden image tag"); + } + } + + const project = await this.#prismaClient.project.findUniqueOrThrow({ + where: { + externalRef: projectRef, + environments: { + some: { + id: environment.id, + }, + }, + }, + }); + + logger.debug(`Creating image details`, { + imageTag: body.metadata.imageTag, + }); + + const imageDetails = await this.#prismaClient.imageDetails.upsert({ + where: { + projectId_runtimeEnvironmentId_contentHash: { + contentHash: body.metadata.contentHash, + runtimeEnvironmentId: environment.id, + projectId: project.id, + }, + }, + create: { + contentHash: body.metadata.contentHash, + friendlyId: generateFriendlyId("image"), + tag: body.metadata.imageTag, + runtimeEnvironmentId: environment.id, + projectId: project.id, + metadata: body.metadata, + }, + update: { + tag: body.metadata.imageTag, + metadata: body.metadata, + }, + }); + + await workerQueue.enqueue("indexTasks", { id: imageDetails.id }); + + return imageDetails; + } +} diff --git a/apps/webapp/app/v3/services/indexTasks.server.ts b/apps/webapp/app/v3/services/indexTasks.server.ts new file mode 100644 index 000000000..a0446cb5b --- /dev/null +++ b/apps/webapp/app/v3/services/indexTasks.server.ts @@ -0,0 +1,53 @@ +import { PrismaClient, prisma } from "~/db.server"; +import { socketIo } from "../handleSocketIo.server"; +import { logger } from "~/services/logger.server"; + +export type IndexTasksServiceOptions = { + idempotencyKey?: string; + triggerVersion?: string; + traceContext?: Record; +}; + +export class IndexTasksService { + #prismaClient: PrismaClient; + + constructor(prismaClient: PrismaClient = prisma) { + this.#prismaClient = prismaClient; + } + + public async call(imageDetailsId: string) { + const imageDetails = await this.#prismaClient.imageDetails.findUnique({ + where: { + id: imageDetailsId, + }, + }); + + if (!imageDetails) { + logger.error(`No image details with this ID: ${imageDetailsId}`); + return; + } + + if (imageDetails.backgroundWorkerId) { + logger.debug( + `Image details have already been indexed for ${imageDetails.friendlyId}. Refreshing worker timestamp.` + ); + await this.#prismaClient.backgroundWorker.update({ + where: { + id: imageDetails.backgroundWorkerId, + }, + data: { + updatedAt: new Date(), + }, + }); + return; + } + + // just broadcast for now - there should only ever be one provider connected + socketIo.providerNamespace.emit("INDEX", { + version: "v1", + contentHash: imageDetails.contentHash, + imageTag: imageDetails.tag, + envId: imageDetails.runtimeEnvironmentId, + }); + } +} diff --git a/apps/webapp/app/v3/services/triggerTask.server.ts b/apps/webapp/app/v3/services/triggerTask.server.ts index 04399f96a..3525200cb 100644 --- a/apps/webapp/app/v3/services/triggerTask.server.ts +++ b/apps/webapp/app/v3/services/triggerTask.server.ts @@ -123,7 +123,7 @@ export class TriggerTaskService extends BaseService { span.setAttribute("runId", taskRun.friendlyId); if (body.options?.dependentAttempt) { - const dependentAttempt = await tx.taskRun.findUnique({ + const dependentAttempt = await tx.taskRunAttempt.findUnique({ where: { friendlyId: body.options.dependentAttempt }, }); diff --git a/apps/webapp/app/v3/sharedSocketConnection.ts b/apps/webapp/app/v3/sharedSocketConnection.ts new file mode 100644 index 000000000..f47157c1a --- /dev/null +++ b/apps/webapp/app/v3/sharedSocketConnection.ts @@ -0,0 +1,94 @@ +import { + MessageCatalogToSocketIoEvents, + ZodMessageHandler, + ZodMessageSender, + clientWebsocketMessages, + serverWebsocketMessages, +} from "@trigger.dev/core/v3"; +import { Evt } from "evt"; +import { randomUUID } from "node:crypto"; +import { logger } from "~/services/logger.server"; +import { SharedQueueConsumer } from "./marqs/sharedQueueConsumer.server"; +import { DisconnectReason, Namespace, Socket } from "socket.io"; + +export class SharedSocketConnection { + public id: string; + public onClose: Evt = new Evt(); + + private _sender: ZodMessageSender; + private _sharedConsumer: SharedQueueConsumer; + private _messageHandler: ZodMessageHandler; + + constructor( + namespace: Namespace< + MessageCatalogToSocketIoEvents, + MessageCatalogToSocketIoEvents + >, + private socket: Socket< + MessageCatalogToSocketIoEvents, + MessageCatalogToSocketIoEvents + >, + logger?: (...args: any[]) => void + ) { + this.id = randomUUID(); + + this._sender = new ZodMessageSender({ + schema: serverWebsocketMessages, + sender: async (message) => { + return new Promise((resolve, reject) => { + try { + const { type, ...payload } = message; + namespace.emit(type, payload as any); + resolve(); + } catch (err) { + reject(err); + } + }); + }, + }); + + this._sharedConsumer = new SharedQueueConsumer(this._sender); + + socket.on("disconnect", this.#handleClose.bind(this)); + socket.on("error", this.#handleError.bind(this)); + + this._messageHandler = new ZodMessageHandler({ + schema: clientWebsocketMessages, + messages: { + READY_FOR_TASKS: async (payload) => { + this._sharedConsumer.start(); + }, + BACKGROUND_WORKER_DEPRECATED: async (payload) => { + // await this._sharedConsumer.deprecateBackgroundWorker(payload.backgroundWorkerId); + }, + BACKGROUND_WORKER_MESSAGE: async (payload) => { + switch (payload.data.type) { + case "TASK_RUN_COMPLETED": { + // handled in coordinator namespace + break; + } + case "TASK_HEARTBEAT": { + // handled in coordinator namespace + break; + } + } + }, + }, + }); + this._messageHandler.registerHandlers(this.socket, logger); + } + + async initialize() { + this._sender.send("SERVER_READY", { id: this.id }); + } + + async #handleClose(ev: DisconnectReason) { + await this._sharedConsumer.stop(); + + this.onClose.post(ev); + } + + async #handleError(ev: Error) { + logger.error("Websocket error", { ev }); + } +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 28972ea74..faedc35ed 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -142,6 +142,7 @@ "simple-oauth2": "^5.0.0", "simplur": "^3.0.1", "slug": "^6.0.0", + "socket.io": "^4.7.4", "sonner": "^1.0.3", "sqs-consumer": "^7.4.0", "tailwind-merge": "^1.12.0", @@ -201,6 +202,7 @@ "@typescript-eslint/parser": "^5.59.6", "autoprefixer": "^10.4.13", "datepicker": "link:@types/@react-aria/datepicker", + "engine.io": "^6.5.4", "esbuild": "^0.15.10", "eslint": "^8.24.0", "eslint-config-prettier": "^8.5.0", diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index c1693d9c8..98e354966 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -5,6 +5,8 @@ import morgan from "morgan"; import { createRequestHandler } from "@remix-run/express"; import { WebSocketServer } from "ws"; import { broadcastDevReady, logDevReady } from "@remix-run/server-runtime"; +import type { Server as IoServer } from "socket.io"; +import type { Server as EngineServer } from "engine.io"; const app = express(); @@ -53,6 +55,7 @@ app.all( const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000; if (process.env.HTTP_SERVER_DISABLED !== "true") { + const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo; const wss: WebSocketServer | undefined = build.entry.module.wss; const server = app.listen(port, () => { @@ -77,6 +80,9 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") { }); }); + socketIo?.io.attach(server); + server.removeAllListeners("upgrade"); // prevent duplicate upgrades from listeners created by io.attach() + server.on("upgrade", async (req, socket, head) => { console.log( `Attemping to upgrade connection at url ${req.url} with headers: ${JSON.stringify( @@ -86,6 +92,15 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") { const url = new URL(req.url ?? "", "http://localhost"); + // Upgrade socket.io connection + if (url.pathname.startsWith("/socket.io/")) { + console.log(`Socket.io client connected, upgrading their connection...`); + + // https://github.com/socketio/socket.io/issues/4693 + (socketIo?.io.engine as EngineServer).handleUpgrade(req, socket, head); + return; + } + // Only upgrade the connecting if the path is `/ws` if (url.pathname !== "/ws") { socket.destroy( diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 99d871c3e..c7fa6ab16 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -27,7 +27,7 @@ services: pgadmin: container_name: pgadmin - image: dpage/pgadmin4:7 + image: dpage/pgadmin4:8 restart: always environment: PGADMIN_DEFAULT_EMAIL: admin@example.com @@ -55,7 +55,9 @@ services: - 6379:6379 otel-collector: + container_name: otel-collector image: otel/opentelemetry-collector-contrib:latest + restart: always command: ["--config", "/etc/otel-collector-config.yaml"] volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 25c931e35..2f0e017ce 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -43,7 +43,9 @@ "@types/object-hash": "^3.0.6", "@types/react": "^18.2.48", "@types/ws": "^8.5.3", + "cpy-cli": "^5.0.0", "npm-run-all": "^4.1.5", + "npm-watch": "^0.11.0", "open": "^10.0.3", "p-retry": "^6.1.0", "rimraf": "^3.0.2", @@ -53,14 +55,23 @@ "vitest": "^0.34.4", "xdg-app-paths": "^8.3.0" }, + "watch": { + "build:prod-containerfile": "src/Containerfile.prod" + }, "scripts": { "typecheck": "tsc", "build": "npm run clean && run-p build:**", "build:main": "tsup", "build:facade": "tsup --config tsup.facade.config.ts", + "build:prod-facade": "tsup --config tsup.prod-facade.config.ts", + "build:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts", + "build:prod-containerfile": "cpy --flat src/Containerfile.prod dist/", "dev": "npm run clean && run-p dev:**", "dev:main": "tsup --watch", "dev:facade": "tsup --config tsup.facade.config.ts --watch", + "dev:prod-facade": "tsup --config tsup.prod-facade.config.ts --watch", + "dev:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts --watch", + "dev:prod-containerfile": "npm-watch", "clean": "rimraf dist", "start": "node dist/index.js", "test": "vitest" @@ -111,6 +122,7 @@ "react": "^18.2.0", "react-error-boundary": "^4.0.12", "simple-git": "^3.19.0", + "socket.io-client": "^4.7.4", "source-map-support": "^0.5.21", "supports-color": "^9.4.0", "terminal-link": "^3.0.0", diff --git a/packages/cli-v3/src/Containerfile.prod b/packages/cli-v3/src/Containerfile.prod new file mode 100644 index 000000000..9a710f728 --- /dev/null +++ b/packages/cli-v3/src/Containerfile.prod @@ -0,0 +1,18 @@ +FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS base + +RUN apk add --no-cache dumb-init + +FROM base + +ENV TRIGGER_CONTENT_HASH=__CONTENT_HASH__ +ENV TRIGGER_PROJECT_DIR=__PROJECT_DIR__ +ENV TRIGGER_PROJECT_REF=__PROJECT_REF__ +ENV TRIGGER_CLI_PACKAGE_VERSION=__CLI_PACKAGE_VERSION__ + +WORKDIR /app + +COPY --chown=node:node . . + +USER node + +CMD [ "dumb-init", "node", "index.mjs" ] \ No newline at end of file diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 214b143d1..989e84125 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -2,18 +2,24 @@ import { z } from "zod"; import { CreateAuthorizationCodeResponseSchema, GetPersonalAccessTokenResponseSchema, - GetProjectDevResponse, - CreateBackgroundWorkerRequestBody, WhoAmIResponseSchema, + CreateBackgroundWorkerRequestBody, CreateBackgroundWorkerResponse, + CreateImageDetailsRequestBody, + CreateImageDetailsResponse, + GetProjectDevResponse, GetEnvironmentVariablesResponseBody, } from "@trigger.dev/core/v3"; -export class ApiClient { +export class CliApiClient { + private readonly apiURL: string; + constructor( - private readonly apiURL: string, + apiURL: string, private readonly accessToken?: string - ) {} + ) { + this.apiURL = apiURL.replace(/\/$/, ""); + } async createAuthorizationCode() { return zodfetch( @@ -49,7 +55,7 @@ export class ApiClient { async createBackgroundWorker(projectRef: string, body: CreateBackgroundWorkerRequestBody) { if (!this.accessToken) { - throw new Error("indexProject: No access token"); + throw new Error("createBackgroundWorker: No access token"); } return zodfetch( @@ -66,6 +72,25 @@ export class ApiClient { ); } + async createImageDetails(projectRef: string, body: CreateImageDetailsRequestBody) { + if (!this.accessToken) { + throw new Error("createImageDetails: No access token"); + } + + return zodfetch( + CreateImageDetailsResponse, + `${this.apiURL}/api/v1/projects/${projectRef}/image-details`, + { + method: "POST", + headers: { + Authorization: `Bearer ${this.accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + } + ); + } + async getProjectDevEnv({ projectRef }: { projectRef: string }) { if (!this.accessToken) { throw new Error("getProjectDevEnv: No access token"); @@ -79,6 +104,19 @@ export class ApiClient { }); } + async getProjectProdEnv({ projectRef }: { projectRef: string }) { + if (!this.accessToken) { + throw new Error("getProjectDevEnv: No access token"); + } + + return zodfetch(GetProjectDevResponse, `${this.apiURL}/api/v1/projects/${projectRef}/prod`, { + headers: { + Authorization: `Bearer ${this.accessToken}`, + "Content-Type": "application/json", + }, + }); + } + async getEnvironmentVariables(projectRef: string) { if (!this.accessToken) { throw new Error("getEnvironmentVariables: No access token"); diff --git a/packages/cli-v3/src/cli/index.ts b/packages/cli-v3/src/cli/index.ts index 92449e0d8..54525b35d 100644 --- a/packages/cli-v3/src/cli/index.ts +++ b/packages/cli-v3/src/cli/index.ts @@ -8,6 +8,7 @@ import { configureWhoamiCommand } from "../commands/whoami.js"; import { COMMAND_NAME } from "../consts.js"; import { getVersion } from "../utilities/getVersion.js"; import { printInitialBanner } from "../utilities/initialBanner.js"; +import { configureBuildCommand } from "../commands/build.js"; export const program = new Command(); @@ -55,6 +56,8 @@ program } }); + +configureBuildCommand(program); configureDevCommand(program); program diff --git a/packages/cli-v3/src/commands/build.ts b/packages/cli-v3/src/commands/build.ts new file mode 100644 index 000000000..462cf6491 --- /dev/null +++ b/packages/cli-v3/src/commands/build.ts @@ -0,0 +1,316 @@ +import chalk from "chalk"; +import { Command } from "commander"; +import { build } from "esbuild"; +import { execa } from "execa"; +import { resolve as importResolve } from "import-meta-resolve"; +import { createHash } from "node:crypto"; +import fs, { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import * as packageJson from "../../package.json"; +import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; +import { logger } from "../utilities/logger.js"; +import { isLoggedIn } from "../utilities/session.js"; +import { CommonCommandOptions } from "../cli/common.js"; +import { getConfigPath, readConfig } from "../utilities/configFiles.js"; +import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles.js"; +import { ResolvedConfig } from "@trigger.dev/core/v3"; +import { CliApiClient } from "../apiClient"; + +const BuildCommandOptions = CommonCommandOptions.extend({ + registry: z.string(), + repo: z.string(), + skipTypecheck: z.boolean().optional(), +}); + +type BuildCommandOptions = z.infer; + +export function configureBuildCommand(program: Command) { + program + .command("build") + .description("Build your Trigger.dev tasks locally") + .argument("[path]", "The path to the project", ".") + .option("-r, --repo ", "The repo to push images to", "task") + .option("-R, --registry ", "The registry to push images to", "") + .option("-T, --skip-typecheck", "Whether to skip the pre-build typecheck") + .option( + "-l, --log-level ", + "The log level to use (debug, info, log, warn, error, none)", + "log" + ) + .action(async (path, options) => { + try { + await buildCommand(path, options); + } catch (e) { + //todo error reporting + throw e; + } + }); +} + +export async function buildCommand(dir: string, anyOptions: unknown) { + const options = BuildCommandOptions.safeParse(anyOptions); + + if (!options.success) { + throw new Error(`Invalid options: ${options.error}`); + } + + const authorization = await isLoggedIn(); + + if (!authorization.ok) { + if (authorization.error === "fetch failed") { + logger.error("Fetch failed. Platform down?"); + } else { + logger.error("You must login first. Use `trigger.dev login` to login."); + } + process.exitCode = 1; + return; + } + + await startBuild(dir, options.data, authorization.config); +} + +async function startBuild( + dir: string, + options: BuildCommandOptions, + authorization: { apiUrl: string; accessToken: string } +) { + try { + if (options.logLevel) { + logger.loggerLevel = options.logLevel; + } + + await printStandloneInitialBanner(true); + + const configPath = await getConfigPath(dir); + const config = await readConfig(configPath); + + const apiClient = new CliApiClient(authorization.apiUrl, authorization.accessToken); + + const prodEnv = await apiClient.getProjectProdEnv({ projectRef: config.project }); + + if (!prodEnv.success) { + throw new Error(prodEnv.error); + } + + const buildResult = await runBuild(config, options, { + apiUrl: authorization.apiUrl, + apiKey: prodEnv.data.apiKey, + }); + + const envClient = new CliApiClient(authorization.apiUrl, prodEnv.data.apiKey); + await envClient.createImageDetails(config.project, { + metadata: { + contentHash: buildResult.contentHash, + imageTag: buildResult.imageTag, + }, + }); + } catch (e) { + throw e; + } +} + +async function runBuild( + config: ResolvedConfig, + options: BuildCommandOptions, + auth: { apiUrl: string; apiKey: string } +) { + const taskFiles = await gatherTaskFiles(config); + + const prodFacade = readFileSync( + new URL(importResolve("./prod-facade.js", import.meta.url)).href.replace("file://", ""), + "utf-8" + ); + const entryPointContents = prodFacade.replace("__TASKS__", createTaskFileImports(taskFiles)); + + if (!options.skipTypecheck) { + logger.log(chalk.dim("⎔ Typecheck...")); + + const tscTypecheck = execa("npm", ["exec", "tsc", "--", "--noEmit"]); + + tscTypecheck.stdout?.on("data", (chunk) => logger.log(chunk.toString())); + tscTypecheck.stderr?.on("data", (chunk) => logger.error(chunk.toString())); + + try { + await new Promise((resolve, reject) => { + tscTypecheck.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code))); + }); + } catch (error) { + throw new Error("Typecheck failed."); + } + + logger.log(chalk.green(`Typecheck succeeded.\n`)); + } + + logger.log(chalk.dim("⎔ Bundling tasks...")); + + const result = await build({ + stdin: { + contents: entryPointContents, + resolveDir: process.cwd(), + sourcefile: "__entryPoint.ts", + }, + bundle: true, + metafile: true, + write: false, + minify: false, + sourcemap: true, + logLevel: "warning", + platform: "node", + format: "esm", + target: ["node18", "es2020"], + outdir: "out", + banner: { + js: "import { createRequire } from 'module';const require = createRequire(import.meta.url);", + }, + }); + + if (result.errors.length > 0) { + logger.error(result.errors); + throw new Error("Build failed"); + } + if (!result || !result.outputFiles) { + throw new Error("Build failed: no result"); + } + + const metaOutputKey = join("out", `stdin.js`); + + const metaOutput = result.metafile!.outputs[metaOutputKey]; + + if (!metaOutput) { + throw new Error(`Could not find metafile`); + } + + const outputFileKey = join(config.projectDir, metaOutputKey); + const outputFile = result.outputFiles.find((file) => file.path === outputFileKey); + + if (!outputFile) { + throw new Error(`Could not find output file for entry point ${metaOutput.entryPoint}`); + } + + const sourceMapFileKey = join(config.projectDir, `${metaOutputKey}.map`); + const sourceMapFile = result.outputFiles.find((file) => file.path === sourceMapFileKey); + + if (!sourceMapFile) { + throw new Error(`Could not find source map file for entry point ${metaOutput.entryPoint}`); + } + + const md5Hasher = createHash("md5"); + md5Hasher.update(Buffer.from(outputFile.contents.buffer)); + + const contentHash = md5Hasher.digest("hex"); + const buildContextPath = join(config.projectDir, ".trigger"); + + try { + // Clean build context dir first + await fs.promises.rm(buildContextPath, { recursive: true, force: true }); + } catch (err) { + } finally { + // ..then ensure it exists + await fs.promises.mkdir(buildContextPath, { recursive: true }); + } + + // Create a file at join(dir, ".trigger", path) with the fileContents + const fullPath = join(buildContextPath, `${contentHash}.mjs`); + await fs.promises.writeFile(fullPath, outputFile.text); + const sourceMapPath = `${fullPath}.map`; + await fs.promises.writeFile(sourceMapPath, sourceMapFile.text); + + const prodWorkerPath = new URL(importResolve("./prod-worker.mjs", import.meta.url)).href.replace( + "file://", + "" + ); + await fs.promises.copyFile(prodWorkerPath, join(buildContextPath, "index.mjs")); + + logger.log(chalk.green(`Bundling finished.\n`)); + + let localOnly = false; + + if (!options.registry) { + logger.log(chalk.yellow(`No registry specified, enabling local only mode.\n`)); + localOnly = true; + } + + const registryWithRepo = localOnly ? options.repo : `${options.registry}/${options.repo}`; + + if (!localOnly) { + logger.log(chalk.dim("⎔ Checking repo login...")); + + const dockerLogin = execa("docker", ["login", registryWithRepo]); + + dockerLogin.stdout?.on("data", (chunk) => logger.debug(chunk.toString())); + dockerLogin.stderr?.on("data", (chunk) => logger.error(chunk.toString())); + + try { + await new Promise((resolve, reject) => { + dockerLogin.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code))); + }); + } catch (error) { + throw new Error("Login failed. Please run `docker login` to authenticate."); + } + + logger.log(chalk.green(`Login succeeded.\n`)); + } + + logger.log(chalk.dim("⎔ Starting docker build...")); + + const containerfile = await fs.promises.readFile( + new URL(importResolve("./Containerfile.prod", import.meta.url)).href.replace("file://", ""), + "utf-8" + ); + + const containerfileContents = containerfile + .replace("__CONTENT_HASH__", contentHash) + .replace("__PROJECT_DIR__", config.projectDir) + .replace("__PROJECT_REF__", config.project) + .replace("__CLI_PACKAGE_VERSION__", packageJson.version); + + const containerfilePath = join(buildContextPath, "Containerfile"); + const imageTag = `${registryWithRepo}:${contentHash}`; + + await fs.promises.writeFile(containerfilePath, containerfileContents); + + const dockerBuild = execa("docker", [ + "build", + "-f", + containerfilePath, + "-t", + imageTag, + join(config.projectDir, ".trigger"), + ]); + dockerBuild.stdout?.pipe(process.stdout); + dockerBuild.stderr?.pipe(process.stderr); + + try { + await new Promise((resolve, reject) => { + dockerBuild.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code))); + }); + } catch (error) { + throw new Error("Build failed."); + } + + logger.log(chalk.green(`Build finished.\n`)); + + if (!localOnly) { + logger.log(chalk.dim("⎔ Pushing image...")); + + const dockerPush = execa("docker", ["push", imageTag]); + dockerPush.stdout?.pipe(process.stdout); + dockerPush.stderr?.pipe(process.stderr); + + try { + await new Promise((resolve, reject) => { + dockerPush.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code))); + }); + } catch (error) { + throw new Error("Push failed."); + } + + logger.log(chalk.green(`Push complete.\n`)); + } + + return { + contentHash, + imageTag, + }; +} diff --git a/packages/cli-v3/src/commands/dev.tsx b/packages/cli-v3/src/commands/dev.tsx index 728ebe363..9d4cbb07d 100644 --- a/packages/cli-v3/src/commands/dev.tsx +++ b/packages/cli-v3/src/commands/dev.tsx @@ -1,5 +1,6 @@ import { CreateBackgroundWorkerRequestBody, + ResolvedConfig, TaskResource, ZodMessageHandler, ZodMessageSender, @@ -10,48 +11,28 @@ import chalk from "chalk"; import { watch } from "chokidar"; import { Command } from "commander"; import { BuildContext, context } from "esbuild"; -import { findUp } from "find-up"; import { resolve as importResolve } from "import-meta-resolve"; import { Box, Text, render, useApp, useInput } from "ink"; import { createHash } from "node:crypto"; import fs, { readFileSync } from "node:fs"; import { ClientRequestArgs } from "node:http"; -import { basename, dirname, join, relative, resolve } from "node:path"; -import { pathToFileURL } from "node:url"; +import { basename, dirname, join } from "node:path"; import pThrottle from "p-throttle"; import { WebSocket } from "partysocket"; import React, { Suspense, useEffect } from "react"; import { ClientOptions, WebSocket as wsWebSocket } from "ws"; import { z } from "zod"; import * as packageJson from "../../package.json"; -import { ApiClient } from "../apiClient.js"; -import { CLOUD_API_URL } from "../consts.js"; import { BackgroundWorker, BackgroundWorkerCoordinator } from "../dev/backgroundWorker.js"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger.js"; -import { RequireKeys } from "../utilities/requiredKeys.js"; import { isLoggedIn } from "../utilities/session.js"; import { CommonCommandOptions } from "../cli/common.js"; +import { getConfigPath, readConfig } from "../utilities/configFiles"; +import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles"; +import { CliApiClient } from "../apiClient"; -const CONFIG_FILES = ["trigger.config.js", "trigger.config.mjs"]; - -const ConfigSchema = z.object({ - project: z.string(), - triggerDirectories: z.string().array().optional(), - triggerUrl: z.string().optional(), - projectDir: z.string().optional(), -}); - -type Config = z.infer; -type ResolvedConfig = RequireKeys; -type TaskFile = { - triggerDir: string; - filePath: string; - importPath: string; - importName: string; -}; - -let apiClient: ApiClient | undefined; +let apiClient: CliApiClient | undefined; const DevCommandOptions = CommonCommandOptions.extend({ debugger: z.boolean().default(false), @@ -92,7 +73,11 @@ export async function devCommand(dir: string, anyOptions: unknown) { const authorization = await isLoggedIn(); if (!authorization.ok) { - logger.error("You must login first. Use `trigger.dev login` to login."); + if (authorization.error === "fetch failed") { + logger.error("Fetch failed. Platform down?"); + } else { + logger.error("You must login first. Use `trigger.dev login` to login."); + } process.exitCode = 1; return; } @@ -143,7 +128,7 @@ async function startDev( const accessToken = authorization.accessToken; const apiUrl = authorization.apiUrl; - apiClient = new ApiClient(apiUrl, accessToken); + apiClient = new CliApiClient(apiUrl, accessToken); const devEnv = await apiClient.getProjectDevEnv({ projectRef: config.project }); @@ -151,7 +136,7 @@ async function startDev( throw new Error(devEnv.error); } - const environmentClient = new ApiClient(apiUrl, devEnv.data.apiKey); + const environmentClient = new CliApiClient(apiUrl, devEnv.data.apiKey); return ( - `import * as ${taskFile.importName} from "./${taskFile.importPath}"; TaskFileImports["${ - taskFile.importName - }"] = ${taskFile.importName}; TaskFiles["${taskFile.importName}"] = ${JSON.stringify( - taskFile - )};` - ) - .join("\n"); -} - -// Find all the top-level .js or .ts files in the trigger directories -async function gatherTaskFiles(config: ResolvedConfig): Promise> { - const taskFiles: Array = []; - - for (const triggerDir of config.triggerDirectories) { - const files = await fs.promises.readdir(triggerDir, { withFileTypes: true }); - for (const file of files) { - if (!file.isFile()) continue; - if (!file.name.endsWith(".js") && !file.name.endsWith(".ts")) continue; - - const fullPath = join(triggerDir, file.name); - - const filePath = relative(config.projectDir, fullPath); - const importPath = filePath.replace(/\.(js|ts)$/, ""); - const importName = importPath.replace(/\//g, "_"); - - taskFiles.push({ triggerDir, importPath, importName, filePath }); - } - } - - return taskFiles; -} - -async function getConfigPath(dir: string): Promise { - const path = await findUp(CONFIG_FILES, { cwd: dir }); - - if (!path) { - throw new Error("No config file found."); - } - - return path; -} - -async function readConfig(path: string): Promise { - try { - // import the config file - const userConfigModule = await import(`${pathToFileURL(path).href}?_ts=${Date.now()}`); - const rawConfig = await normalizeConfig(userConfigModule ? userConfigModule.default : {}); - const config = ConfigSchema.parse(rawConfig); - - return resolveConfig(path, config); - } catch (error) { - console.error(`Failed to load config file at ${path}`); - throw error; - } -} - -async function resolveConfig(path: string, config: Config): Promise { - if (!config.triggerDirectories) { - config.triggerDirectories = await findTriggerDirectories(path); - } - - config.triggerDirectories = resolveTriggerDirectories(config.triggerDirectories); - - if (!config.triggerUrl) { - config.triggerUrl = CLOUD_API_URL; - } - - if (!config.projectDir) { - config.projectDir = dirname(path); - } - - return config as ResolvedConfig; -} - -async function normalizeConfig(config: any): Promise { - if (typeof config === "function") { - config = config(); - } - - return await config; -} - -function resolveTriggerDirectories(dirs: string[]): string[] { - return dirs.map((dir) => resolve(dir)); -} - -const IGNORED_DIRS = ["node_modules", ".git", "dist", "build"]; - -async function findTriggerDirectories(filePath: string): Promise { - const dirPath = dirname(filePath); - return getTriggerDirectories(dirPath); -} - -async function getTriggerDirectories(dirPath: string): Promise { - const entries = await fs.promises.readdir(dirPath, { withFileTypes: true }); - const triggerDirectories: string[] = []; - - for (const entry of entries) { - if (!entry.isDirectory() || IGNORED_DIRS.includes(entry.name)) continue; - - const fullPath = join(dirPath, entry.name); - - if (entry.name === "trigger") { - triggerDirectories.push(fullPath); - } - - triggerDirectories.push(...(await getTriggerDirectories(fullPath))); - } - - return triggerDirectories; -} diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 11b8911e9..4934c3f24 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -1,12 +1,12 @@ import { intro, log, outro, select, spinner } from "@clack/prompts"; import open from "open"; import pRetry, { AbortError } from "p-retry"; -import { ApiClient } from "../apiClient.js"; import { ApiUrlOptionsSchema } from "../cli/index.js"; import { chalkLink } from "../utilities/colors.js"; import { readAuthConfigFile, writeAuthConfigFile } from "../utilities/configFiles.js"; import { logger } from "../utilities/logger.js"; import { whoAmI } from "./whoami.js"; +import { CliApiClient } from "../apiClient.js"; export async function loginCommand(options: any) { const result = ApiUrlOptionsSchema.safeParse(options); @@ -29,7 +29,7 @@ export type LoginResult = }; export async function login(apiUrl: string): Promise { - const apiClient = new ApiClient(apiUrl); + const apiClient = new CliApiClient(apiUrl); intro("Logging in to Trigger.dev"); @@ -118,7 +118,7 @@ export async function login(apiUrl: string): Promise { } } -async function getPersonalAccessToken(apiClient: ApiClient, authorizationCode: string) { +async function getPersonalAccessToken(apiClient: CliApiClient, authorizationCode: string) { const token = await apiClient.getPersonalAccessToken(authorizationCode); if (!token.success) { diff --git a/packages/cli-v3/src/commands/whoami.ts b/packages/cli-v3/src/commands/whoami.ts index 223a52e0f..a5685c096 100644 --- a/packages/cli-v3/src/commands/whoami.ts +++ b/packages/cli-v3/src/commands/whoami.ts @@ -1,5 +1,4 @@ import { note, spinner } from "@clack/prompts"; -import { ApiClient } from "../apiClient.js"; import { chalkLink } from "../utilities/colors.js"; import { logger } from "../utilities/logger.js"; import { isLoggedIn } from "../utilities/session.js"; @@ -7,6 +6,7 @@ import { Command } from "commander"; import { printInitialBanner } from "../utilities/initialBanner.js"; import { CommonCommandOptions } from "../cli/common.js"; import { z } from "zod"; +import { CliApiClient } from "../apiClient.js"; type WhoAmIResult = | { @@ -55,7 +55,11 @@ export async function whoAmI(options?: WhoamiCommandOptions): Promise { + process.send?.(message); + }, +}); + +const prodRuntimeManager = new ProdRuntimeManager(sender); + +runtime.setGlobalRuntimeManager(prodRuntimeManager); + +const otelTaskLogger = new OtelTaskLogger({ + logger: otelLogger, + tracer: tracer, + level: "info", +}); + +logger.setGlobalTaskLogger(otelTaskLogger); + +type TaskFileImport = Record; + +const TaskFileImports: Record = {}; +const TaskFiles: Record = {}; + +__TASKS__; +declare const __TASKS__: Record; + +class TaskExecutor { + constructor(public task: TaskMetadataWithFunctions) {} + + async determineRetrying( + execution: TaskRunExecution, + error: unknown + ): Promise { + if (!this.task.retry) { + return; + } + + const retry = this.task.retry; + + const delay = calculateNextRetryDelay(retry, execution.attempt.number); + + return typeof delay === "undefined" ? undefined : { timestamp: Date.now() + delay, delay }; + } + + async execute( + execution: TaskRunExecution, + worker: BackgroundWorkerProperties, + traceContext: Record + ) { + const parsedPayload = JSON.parse(execution.run.payload); + const ctx = TaskRunContext.parse(execution); + const attemptMessage = `Attempt ${execution.attempt.number}`; + + const output = await taskContextManager.runWith( + { + ctx, + payload: parsedPayload, + worker, + }, + async () => { + tracingSDK.asyncResourceDetector.resolveWithAttributes({ + ...taskContextManager.attributes, + [SemanticInternalAttributes.SDK_VERSION]: this.task.packageVersion, + [SemanticInternalAttributes.SDK_LANGUAGE]: "typescript", + }); + + return await tracer.startActiveSpan( + attemptMessage, + async (span) => { + return await consoleInterceptor.intercept(console, async () => { + const init = await this.#callTaskInit(parsedPayload, ctx); + + try { + const output = await this.#callRun(parsedPayload, ctx, init); + + span.setAttributes(flattenAttributes(output, SemanticInternalAttributes.OUTPUT)); + + return output; + } finally { + await this.#callTaskCleanup(parsedPayload, ctx, init); + } + }); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SemanticInternalAttributes.STYLE_ICON]: "attempt", + ...flattenAttributes(parsedPayload, SemanticInternalAttributes.PAYLOAD), + ...accessoryAttributes({ + items: [ + { + text: ctx.task.filePath, + }, + { + text: `${ctx.task.exportName}.run()`, + }, + ], + style: "codepath", + }), + }, + }, + tracer.extractContext(traceContext) + ); + } + ); + + return { output: JSON.stringify(output), outputType: "application/json" }; + } + + async #callRun(payload: unknown, ctx: TaskRunContext, init: unknown) { + const runFn = this.task.fns.run; + const middlewareFn = this.task.fns.middleware; + + if (!runFn) { + throw new Error("Task does not have a run function"); + } + + if (!middlewareFn) { + return runFn({ payload, ctx }); + } + + return middlewareFn({ payload, ctx, next: async () => runFn({ payload, ctx, init }) }); + } + + async #callTaskInit(payload: unknown, ctx: TaskRunContext) { + const initFn = this.task.fns.init; + + if (!initFn) { + return {}; + } + + return tracer.startActiveSpan("init", async (span) => { + return await initFn({ payload, ctx }); + }); + } + + async #callTaskCleanup(payload: unknown, ctx: TaskRunContext, init: unknown) { + const cleanupFn = this.task.fns.cleanup; + + if (!cleanupFn) { + return; + } + + return tracer.startActiveSpan("cleanup", async (span) => { + return await cleanupFn({ payload, ctx, init }); + }); + } +} + +function getTasks(): Array { + const result: Array = []; + + for (const [importName, taskFile] of Object.entries(TaskFiles)) { + const fileImports = TaskFileImports[importName]; + + for (const [exportName, task] of Object.entries(fileImports ?? {})) { + if ((task as any).__trigger) { + result.push({ + id: (task as any).__trigger.id, + exportName, + packageVersion: (task as any).__trigger.packageVersion, + filePath: (taskFile as any).filePath, + queue: (task as any).__trigger.queue, + retry: (task as any).__trigger.retry, + fns: (task as any).__trigger.fns, + }); + } + } + } + + return result; +} + +function getTaskMetadata(): Array { + const result = getTasks(); + + // Remove the functions from the metadata + return result.map((task) => { + const { fns, ...metadata } = task; + + return metadata; + }); +} + +const tasks = getTasks(); + +runtime.registerTasks(tasks); + +const taskExecutors: Map = new Map(); + +for (const task of tasks) { + taskExecutors.set(task.id, new TaskExecutor(task)); +} + +let _execution: TaskRunExecution | undefined; +let _isRunning = false; + +const handler = new ZodMessageHandler({ + schema: workerToChildMessages, + messages: { + EXECUTE_TASK_RUN: async ({ execution, traceContext, metadata }) => { + if (_isRunning) { + console.error("Worker is already running a task"); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.attempt.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_ALREADY_RUNNING, + }, + }, + }); + + return; + } + process.title = `trigger-prod-worker: ${execution.task.id} ${execution.run.id}`; + + const executor = taskExecutors.get(execution.task.id); + + if (!executor) { + console.error(`Could not find executor for task ${execution.task.id}`); + + await sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + ok: false, + id: execution.attempt.id, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR, + }, + }, + }); + + return; + } + + try { + _execution = execution; + _isRunning = true; + + const result = await executor.execute(execution, metadata, traceContext); + + return sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + id: execution.attempt.id, + ok: true, + ...result, + }, + }); + } catch (e) { + return sender.send("TASK_RUN_COMPLETED", { + execution, + result: { + id: execution.attempt.id, + ok: false, + error: parseError(e), + retry: await executor.determineRetrying(execution, e), + }, + }); + } finally { + _execution = undefined; + _isRunning = false; + } + }, + TASK_RUN_COMPLETED_NOTIFICATION: async ({ completion, execution }) => { + prodRuntimeManager.resumeTask(completion, execution); + }, + CLEANUP: async ({ flush, kill }) => { + if (kill) { + await tracingSDK.flush(); + // Now we need to exit the process + await sender.send("READY_TO_DISPOSE", undefined); + } else { + if (flush) { + await tracingSDK.flush(); + } + } + }, + }, +}); + +process.on("message", async (msg: any) => { + await handler.handleMessage(msg); +}); + +sender.send("TASKS_READY", { tasks: getTaskMetadata() }).catch((err) => { + console.error("Failed to send TASKS_READY message", err); +}); + +process.title = "trigger-prod-worker"; + +async function asyncHeartbeat(initialDelayInSeconds: number = 30, intervalInSeconds: number = 5) { + async function _doHeartbeat() { + while (true) { + if (_isRunning && _execution) { + try { + await sender.send("TASK_HEARTBEAT", { id: _execution.attempt.id }); + } catch (err) { + console.error("Failed to send HEARTBEAT message", err); + } + } + + await new Promise((resolve) => setTimeout(resolve, 1000 * intervalInSeconds)); + } + } + + // Wait for the initial delay + await new Promise((resolve) => setTimeout(resolve, 1000 * initialDelayInSeconds)); + + // Wait for 5 seconds before the next execution + return _doHeartbeat(); +} + +// Start the async interval after initial delay +asyncHeartbeat(5).catch((err) => { + console.error("Failed to start asyncHeartbeat", err); +}); diff --git a/packages/cli-v3/src/prod-worker.ts b/packages/cli-v3/src/prod-worker.ts new file mode 100644 index 000000000..2efef649f --- /dev/null +++ b/packages/cli-v3/src/prod-worker.ts @@ -0,0 +1,317 @@ +import { + CoordinatorToProdWorkerEvents, + ProdWorkerToCoordinatorEvents, + TaskResource, +} from "@trigger.dev/core/v3"; +import { HttpReply, getTextBody, SimpleLogger, getRandomPortNumber } from "@trigger.dev/core-apps"; +import { createServer } from "node:http"; +import { io, Socket } from "socket.io-client"; +import { ProdBackgroundWorker } from "./prod/backgroundWorker"; + +const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber()); +const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1"; +const COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080); +const MACHINE_NAME = process.env.MACHINE_NAME || "local"; +const POD_NAME = process.env.POD_NAME || "some-pod"; +const SHORT_HASH = process.env.TRIGGER_CONTENT_HASH!.slice(0, 9); + +const logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`); + +class ProdWorker { + private apiUrl = process.env.TRIGGER_API_URL!; + private apiKey = process.env.TRIGGER_API_KEY!; + private contentHash = process.env.TRIGGER_CONTENT_HASH!; + private projectDir = process.env.TRIGGER_PROJECT_DIR!; + private projectRef = process.env.TRIGGER_PROJECT_REF!; + private envId = process.env.TRIGGER_ENV_ID!; + private cliPackageVersion = process.env.TRIGGER_CLI_PACKAGE_VERSION!; + private attemptId = process.env.TRIGGER_ATTEMPT_ID || "index-only"; + + private executing = false; + private completed = false; + + #httpPort: number; + #backgroundWorker: ProdBackgroundWorker; + #httpServer: ReturnType; + #coordinatorSocket: Socket; + + constructor( + port: number, + private host = "0.0.0.0" + ) { + this.#coordinatorSocket = this.#createCoordinatorSocket(); + + this.#backgroundWorker = new ProdBackgroundWorker(this.#getWorkerEntryPath(this.contentHash), { + projectDir: this.projectDir, + env: { + TRIGGER_API_URL: this.apiUrl, + TRIGGER_API_KEY: this.apiKey, + }, + contentHash: this.contentHash, + }); + this.#backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => { + this.#coordinatorSocket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId }); + }); + this.#backgroundWorker.onWaitForBatch.attach((message) => { + this.#coordinatorSocket.emit("WAIT_FOR_BATCH", { version: "v1", ...message }); + }); + this.#backgroundWorker.onWaitForDuration.attach((message) => { + this.#coordinatorSocket.emit( + "WAIT_FOR_DURATION", + { version: "v1", ...message }, + ({ success }) => { + logger.log("WAIT_FOR_DURATION", { success }); + } + ); + }); + this.#backgroundWorker.onWaitForTask.attach((message) => { + this.#coordinatorSocket.emit("WAIT_FOR_TASK", { version: "v1", ...message }); + }); + + this.#httpPort = port; + this.#httpServer = this.#createHttpServer(); + } + + #createCoordinatorSocket() { + const socket: Socket = io( + `ws://${COORDINATOR_HOST}:${COORDINATOR_PORT}/prod-worker`, + { + transports: ["websocket"], + extraHeaders: { + "x-machine-name": MACHINE_NAME, + "x-pod-name": POD_NAME, + "x-trigger-content-hash": this.contentHash, + "x-trigger-cli-package-version": this.cliPackageVersion, + "x-trigger-project-ref": this.projectRef, + "x-trigger-attempt-id": this.attemptId, + "x-trigger-env-id": this.envId, + }, + } + ); + + const logger = new SimpleLogger(`[coordinator][${socket.id ?? "NO_ID"}]`); + + socket.on("connect_error", (err) => { + logger.error(`connect_error: ${err.message}`); + }); + + socket.on("connect", async () => { + logger.log("connect"); + + if (process.env.INDEX_TASKS === "true") { + const taskResources = await this.#initializeWorker(); + const { success } = await socket.emitWithAck("INDEX_TASKS", { + version: "v1", + ...taskResources, + }); + if (success) { + logger.log("indexing done, shutting down.."); + process.exit(0); + } else { + logger.log("indexing failure, shutting down.."); + process.exit(1); + } + } else { + socket.emit("READY_FOR_EXECUTION", { + version: "v1", + attemptId: process.env.TRIGGER_ATTEMPT_ID!, + }); + } + }); + + socket.on("disconnect", () => { + logger.log("disconnect"); + }); + + socket.on("RESUME", async (message) => { + logger.log("[RESUME]", message); + + for (let i = 0; i < message.completions.length; i++) { + const completion = message.completions[i]; + const execution = message.executions[i]; + + if (!completion || !execution) continue; + + this.#backgroundWorker.taskRunCompletedNotification(completion, execution); + } + }); + + socket.on("EXECUTE_TASK_RUN", async (message, callback) => { + logger.log("[EXECUTE_TASK_RUN]", { attempt: message.payload.execution.attempt }); + + if (this.executing || this.completed) { + return; + } + + this.executing = true; + const completion = await this.#backgroundWorker.executeTaskRun(message.payload); + + logger.log("completed", completion); + + // TODO: replace ack with emit + callback({ completion }); + + this.completed = true; + this.executing = false; + + setTimeout(() => { + process.exit(0); + }, 1000); + }); + + return socket; + } + + #createHttpServer() { + const httpServer = createServer(async (req, res) => { + logger.log(`[${req.method}]`, req.url); + + const reply = new HttpReply(res); + + switch (req.url) { + case "/complete": + setTimeout(() => process.exit(0), 1000); + return reply.text("ok"); + + case "/date": + const date = new Date(); + return reply.text(date.toString()); + + case "/fail": + setTimeout(() => process.exit(1), 1000); + return reply.text("ok"); + + case "/health": + return reply.text("ok"); + + case "/whoami": + return reply.text(this.contentHash); + + case "/wait": + this.#coordinatorSocket.emit( + "WAIT_FOR_DURATION", + { + version: "v1", + ms: 60_000, + }, + ({ success }) => { + logger.log("WAIT_FOR_DURATION", { success }); + } + ); + // this is required when C/Ring established connections + this.#coordinatorSocket.close(); + return reply.text("sent WAIT"); + + case "/connect": + this.#coordinatorSocket.connect(); + return reply.empty(); + + case "/close": + this.#coordinatorSocket.emitWithAck("LOG", { + version: "v1", + text: "close without delay", + }); + this.#coordinatorSocket.close(); + return reply.empty(); + + case "/close-delay": + this.#coordinatorSocket.emitWithAck("LOG", { + version: "v1", + text: "close with delay", + }); + setTimeout(() => { + this.#coordinatorSocket.close(); + }, 200); + return reply.empty(); + + case "/log": + this.#coordinatorSocket.emitWithAck("LOG", { + version: "v1", + text: await getTextBody(req), + }); + return reply.empty(); + + case "/preStop": + logger.log("should do preStop stuff, e.g. checkpoint and graceful shutdown"); + return reply.text("got preStop request"); + + case "/ready": + this.#coordinatorSocket.emit("READY_FOR_EXECUTION", { + version: "v1", + attemptId: this.attemptId, + }); + return reply.empty(); + + default: + return reply.empty(404); + } + }); + + httpServer.on("clientError", (err, socket) => { + socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); + }); + + httpServer.on("listening", () => { + logger.log("http server listening on port", this.#httpPort); + }); + + httpServer.on("error", (error) => { + // @ts-expect-error + if (error.code != "EADDRINUSE") { + return; + } + + logger.error(`port ${this.#httpPort} already in use, retrying with random port..`); + + this.#httpPort = getRandomPortNumber(); + + setTimeout(() => { + this.start(); + }, 100); + }); + + return httpServer; + } + + #getWorkerEntryPath(contentHash: string) { + return `${contentHash}.mjs`; + } + + async #initializeWorker() { + await this.#backgroundWorker.initialize(); + + let packageVersion: string | undefined; + + const taskResources: Array = []; + + if (!this.#backgroundWorker.tasks) { + throw new Error(`Background Worker started without tasks`); + } + + for (const task of this.#backgroundWorker.tasks) { + taskResources.push({ + id: task.id, + filePath: task.filePath, + exportName: task.exportName, + }); + + packageVersion = task.packageVersion; + } + + if (!packageVersion) { + throw new Error(`Background Worker started without package version`); + } + + return { + packageVersion, + tasks: taskResources, + }; + } + + start() { + this.#httpServer.listen(this.#httpPort, this.host); + } +} + +const prodWorker = new ProdWorker(HTTP_SERVER_PORT); +prodWorker.start(); diff --git a/packages/cli-v3/src/prod/backgroundWorker.ts b/packages/cli-v3/src/prod/backgroundWorker.ts new file mode 100644 index 000000000..feba8f7d6 --- /dev/null +++ b/packages/cli-v3/src/prod/backgroundWorker.ts @@ -0,0 +1,514 @@ +import { + BackgroundWorkerProperties, + CreateBackgroundWorkerResponse, + ProdTaskRunExecutionPayload, + SemanticInternalAttributes, + TaskMetadataWithFilePath, + TaskRunBuiltInError, + TaskRunErrorCodes, + TaskRunExecution, + TaskRunExecutionPayload, + TaskRunExecutionResult, + ZodMessageHandler, + ZodMessageSender, + childToWorkerMessages, + correctErrorStackTrace, + workerToChildMessages, +} from "@trigger.dev/core/v3"; +import { Evt } from "evt"; +import { ChildProcess, fork } from "node:child_process"; +import { safeDeleteFileSync } from "../utilities/fileSystem"; + +class UnexpectedExitError extends Error { + constructor(public code: number) { + super(`Unexpected exit with code ${code}`); + + this.name = "UnexpectedExitError"; + } +} + +class CleanupProcessError extends Error { + constructor() { + super("Cancelled"); + + this.name = "CleanupProcessError"; + } +} + +type BackgroundWorkerParams = { + env: Record; + projectDir: string; + contentHash: string; + debugOtel?: boolean; +}; + +export class ProdBackgroundWorker { + private _initialized: boolean = false; + private _handler = new ZodMessageHandler({ + schema: childToWorkerMessages, + }); + + public onTaskHeartbeat: Evt = new Evt(); + + public onWaitForBatch: Evt<{ version?: "v1"; id: string; runs: string[] }> = new Evt(); + public onWaitForDuration: Evt<{ version?: "v1"; ms: number }> = new Evt(); + public onWaitForTask: Evt<{ version?: "v1"; id: string }> = new Evt(); + + private _onClose: Evt = new Evt(); + + public tasks: Array = []; + + _taskRunProcesses: Map = new Map(); + + private _closed: boolean = false; + + constructor( + public path: string, + private params: BackgroundWorkerParams + ) {} + + close() { + if (this._closed) { + return; + } + + this._closed = true; + + this.onTaskHeartbeat.detach(); + + // We need to close all the task run processes + for (const taskRunProcess of this._taskRunProcesses.values()) { + taskRunProcess.cleanup(true); + } + + // Delete worker files + this._onClose.post(); + + safeDeleteFileSync(this.path); + safeDeleteFileSync(`${this.path}.map`); + } + + async initialize() { + if (this._initialized) { + throw new Error("Worker already initialized"); + } + + let resolved = false; + + this.tasks = await new Promise>((resolve, reject) => { + const child = fork(this.path, { + stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], + env: { + ...this.params.env, + }, + }); + + // Set a timeout to kill the child process if it doesn't respond + const timeout = setTimeout(() => { + if (resolved) { + return; + } + + resolved = true; + child.kill(); + reject(new Error("Worker timed out")); + }, 1000); + + child.on("message", async (msg: any) => { + const message = this._handler.parseMessage(msg); + + if (message.type === "TASKS_READY" && !resolved) { + clearTimeout(timeout); + resolved = true; + resolve(message.payload.tasks); + child.kill(); + } + }); + + child.stdout?.on("data", (data) => { + console.log(data.toString()); + }); + + child.stderr?.on("data", (data) => { + console.error(data.toString()); + }); + + child.on("exit", (code) => { + if (!resolved) { + clearTimeout(timeout); + resolved = true; + reject(new Error(`Worker exited with code ${code}`)); + } + }); + }); + + this._initialized = true; + } + + getMetadata(workerId: string, version: string): CreateBackgroundWorkerResponse { + return { + contentHash: this.params.contentHash, + id: workerId, + version: version, + }; + } + + // We need to notify all the task run processes that a task run has completed, + // in case they are waiting for it through triggerAndWait + async taskRunCompletedNotification( + completion: TaskRunExecutionResult, + execution: TaskRunExecution + ) { + for (const taskRunProcess of this._taskRunProcesses.values()) { + taskRunProcess.taskRunCompletedNotification(completion, execution); + } + } + + async #initializeTaskRunProcess(payload: ProdTaskRunExecutionPayload): Promise { + const metadata = this.getMetadata( + payload.execution.worker.id, + payload.execution.worker.version + ); + + if (!this._taskRunProcesses.has(payload.execution.run.id)) { + const taskRunProcess = new TaskRunProcess( + this.path, + { + ...this.params.env, + ...(payload.environment ?? {}), + }, + metadata, + this.params + ); + + taskRunProcess.onExit.attach(() => { + this._taskRunProcesses.delete(payload.execution.run.id); + }); + + taskRunProcess.onTaskHeartbeat.attach((id) => { + this.onTaskHeartbeat.post(id); + }); + + taskRunProcess.onWaitForBatch.attach((message) => { + this.onWaitForBatch.post(message); + }); + + taskRunProcess.onWaitForDuration.attach((message) => { + this.onWaitForDuration.post(message); + }); + + taskRunProcess.onWaitForTask.attach((message) => { + this.onWaitForTask.post(message); + }); + + await taskRunProcess.initialize(); + + this._taskRunProcesses.set(payload.execution.run.id, taskRunProcess); + } + + return this._taskRunProcesses.get(payload.execution.run.id) as TaskRunProcess; + } + + // We need to fork the process before we can execute any tasks + async executeTaskRun(payload: ProdTaskRunExecutionPayload): Promise { + try { + const taskRunProcess = await this.#initializeTaskRunProcess(payload); + + const result = await taskRunProcess.executeTaskRun(payload); + + // Kill the worker if the task was successful or if it's not going to be retried); + await taskRunProcess.cleanup(result.ok || result.retry === undefined); + + if (result.ok) { + return result; + } + + const error = result.error; + + if (error.type === "BUILT_IN_ERROR") { + const mappedError = await this.#correctError(error, payload.execution); + + return { + ...result, + error: mappedError, + }; + } + + return result; + } catch (e) { + if (e instanceof CleanupProcessError) { + return { + id: payload.execution.attempt.id, + ok: false, + retry: undefined, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_EXECUTION_ABORTED, + }, + }; + } + + if (e instanceof UnexpectedExitError) { + return { + id: payload.execution.attempt.id, + ok: false, + retry: undefined, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE, + }, + }; + } + + return { + id: payload.execution.attempt.id, + ok: false, + retry: undefined, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.TASK_EXECUTION_FAILED, + }, + }; + } + } + + async #correctError( + error: TaskRunBuiltInError, + execution: TaskRunExecution + ): Promise { + return { + ...error, + stackTrace: correctErrorStackTrace(error.stackTrace, this.params.projectDir), + }; + } +} + +class TaskRunProcess { + private _handler = new ZodMessageHandler({ + schema: childToWorkerMessages, + }); + private _sender: ZodMessageSender; + private _child: ChildProcess | undefined; + private _attemptPromises: Map< + string, + { resolver: (value: TaskRunExecutionResult) => void; rejecter: (err?: any) => void } + > = new Map(); + private _attemptStatuses: Map = new Map(); + private _currentExecution: TaskRunExecution | undefined; + private _isBeingKilled: boolean = false; + + public onTaskHeartbeat: Evt = new Evt(); + public onExit: Evt = new Evt(); + + public onWaitForBatch: Evt<{ version?: "v1"; id: string; runs: string[] }> = new Evt(); + public onWaitForDuration: Evt<{ version?: "v1"; ms: number }> = new Evt(); + public onWaitForTask: Evt<{ version?: "v1"; id: string }> = new Evt(); + + constructor( + private path: string, + private env: NodeJS.ProcessEnv, + private metadata: BackgroundWorkerProperties, + private worker: BackgroundWorkerParams + ) { + this._sender = new ZodMessageSender({ + schema: workerToChildMessages, + sender: async (message) => { + if (this._child?.connected && !this._isBeingKilled && !this._child.killed) { + this._child?.send?.(message); + } + }, + }); + } + + async initialize() { + this._child = fork(this.path, { + stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], + env: { + ...this.env, + OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({ + [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectDir, + }), + ...(this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}), + }, + }); + + this._child.on("message", this.#handleMessage.bind(this)); + this._child.on("exit", this.#handleExit.bind(this)); + this._child.stdout?.on("data", this.#handleLog.bind(this)); + this._child.stderr?.on("data", this.#handleStdErr.bind(this)); + } + + async cleanup(kill: boolean = false) { + if (kill && this._isBeingKilled) { + return; + } + + await this._sender.send("CLEANUP", { + flush: true, + kill, + }); + + this._isBeingKilled = kill; + } + + async executeTaskRun(payload: TaskRunExecutionPayload): Promise { + let resolver: (value: TaskRunExecutionResult) => void; + let rejecter: (err?: any) => void; + + const promise = new Promise((resolve, reject) => { + resolver = resolve; + rejecter = reject; + }); + + this._attemptStatuses.set(payload.execution.attempt.id, "PENDING"); + + // @ts-expect-error - We know that the resolver and rejecter are defined + this._attemptPromises.set(payload.execution.attempt.id, { resolver, rejecter }); + + const { execution, traceContext } = payload; + + this._currentExecution = execution; + + await this._sender.send("EXECUTE_TASK_RUN", { + execution, + traceContext, + metadata: this.metadata, + }); + + const result = await promise; + + this._currentExecution = undefined; + + return result; + } + + taskRunCompletedNotification(completion: TaskRunExecutionResult, execution: TaskRunExecution) { + if (!completion.ok && typeof completion.retry !== "undefined") { + return; + } + + this._sender.send("TASK_RUN_COMPLETED_NOTIFICATION", { + completion, + execution, + }); + } + + async #handleMessage(msg: any) { + const message = this._handler.parseMessage(msg); + + switch (message.type) { + case "TASK_RUN_COMPLETED": { + const { result, execution } = message.payload; + + const promiseStatus = this._attemptStatuses.get(execution.attempt.id); + + if (promiseStatus !== "PENDING") { + return; + } + + this._attemptStatuses.set(execution.attempt.id, "RESOLVED"); + + const attemptPromise = this._attemptPromises.get(execution.attempt.id); + + if (!attemptPromise) { + return; + } + + const { resolver } = attemptPromise; + + resolver(result); + + break; + } + case "READY_TO_DISPOSE": { + this.#kill(); + + break; + } + case "TASK_HEARTBEAT": { + this.onTaskHeartbeat.post(message.payload.id); + + break; + } + case "TASKS_READY": { + break; + } + case "WAIT_FOR_BATCH": { + this.onWaitForBatch.post(message.payload); + + break; + } + case "WAIT_FOR_DURATION": { + this.onWaitForDuration.post(message.payload); + + break; + } + case "WAIT_FOR_TASK": { + this.onWaitForTask.post(message.payload); + + break; + } + } + } + + async #handleExit(code: number) { + // Go through all the attempts currently pending and reject them + for (const [id, status] of this._attemptStatuses.entries()) { + if (status === "PENDING") { + this._attemptStatuses.set(id, "REJECTED"); + + const attemptPromise = this._attemptPromises.get(id); + + if (!attemptPromise) { + continue; + } + + const { rejecter } = attemptPromise; + + if (this._isBeingKilled) { + rejecter(new CleanupProcessError()); + } else { + rejecter(new UnexpectedExitError(code)); + } + } + } + + this.onExit.post(code); + } + + #handleLog(data: Buffer) { + if (!this._currentExecution) { + return; + } + + console.log( + `[${this.metadata.version}][${this._currentExecution.run.id}.${ + this._currentExecution.attempt.number + }] ${data.toString()}` + ); + } + + #handleStdErr(data: Buffer) { + if (this._isBeingKilled) { + return; + } + + if (!this._currentExecution) { + console.error(`[${this.metadata.version}] ${data.toString()}`); + + return; + } + + console.error( + `[${this.metadata.version}][${this._currentExecution.run.id}.${ + this._currentExecution.attempt.number + }] ${data.toString()}` + ); + } + + #kill() { + if (this._child && !this._child.killed) { + this._child?.kill(); + } + } +} diff --git a/packages/cli-v3/src/types.ts b/packages/cli-v3/src/types.ts index 4070b8384..a84e8e9b8 100644 --- a/packages/cli-v3/src/types.ts +++ b/packages/cli-v3/src/types.ts @@ -8,3 +8,10 @@ export type TaskMetadataWithFunctions = TaskMetadataWithFilePath & { middleware?: (params: any) => Promise; }; }; + +export type TaskFile = { + triggerDir: string; + filePath: string; + importPath: string; + importName: string; +}; diff --git a/packages/cli-v3/src/utilities/configFiles.ts b/packages/cli-v3/src/utilities/configFiles.ts index eac5d34ac..b8542d44d 100644 --- a/packages/cli-v3/src/utilities/configFiles.ts +++ b/packages/cli-v3/src/utilities/configFiles.ts @@ -1,12 +1,17 @@ import { mkdirSync, writeFileSync } from "node:fs"; -import path from "node:path"; +import path, { dirname } from "node:path"; import xdgAppPaths from "xdg-app-paths"; import { z } from "zod"; import { readJSONFileSync } from "./fileSystem.js"; import { logger } from "./logger.js"; +import { findUp } from "find-up"; +import { CLOUD_API_URL, CONFIG_FILES } from "../consts.js"; +import { pathToFileURL } from "node:url"; +import { findTriggerDirectories, resolveTriggerDirectories } from "./taskFiles.js"; +import { Config, ResolvedConfig } from "@trigger.dev/core/v3"; function getGlobalConfigFolderPath() { - const configDir = xdgAppPaths(".trigger").config(); + const configDir = xdgAppPaths("trigger").config(); return configDir; } @@ -20,7 +25,7 @@ export const UserAuthConfigSchema = z.object({ export type UserAuthConfig = z.infer; function getAuthConfigFilePath() { - return path.join(getGlobalConfigFolderPath(), "config", "default.json"); + return path.join(getGlobalConfigFolderPath(), "default.json"); } export function writeAuthConfigFile(config: UserAuthConfig) { @@ -45,3 +50,53 @@ export function readAuthConfigFile(): UserAuthConfig | undefined { return undefined; } } + +export async function getConfigPath(dir: string): Promise { + const path = await findUp(CONFIG_FILES, { cwd: dir }); + + if (!path) { + throw new Error("No config file found."); + } + + return path; +} + +export async function readConfig(path: string): Promise { + try { + // import the config file + const userConfigModule = await import(`${pathToFileURL(path).href}?_ts=${Date.now()}`); + const rawConfig = await normalizeConfig(userConfigModule ? userConfigModule.default : {}); + const config = Config.parse(rawConfig); + + return resolveConfig(path, config); + } catch (error) { + console.error(`Failed to load config file at ${path}`); + throw error; + } +} + +export async function resolveConfig(path: string, config: Config): Promise { + if (!config.triggerDirectories) { + config.triggerDirectories = await findTriggerDirectories(path); + } + + config.triggerDirectories = resolveTriggerDirectories(config.triggerDirectories); + + if (!config.triggerUrl) { + config.triggerUrl = CLOUD_API_URL; + } + + if (!config.projectDir) { + config.projectDir = dirname(path); + } + + return config as ResolvedConfig; +} + +export async function normalizeConfig(config: any): Promise { + if (typeof config === "function") { + config = config(); + } + + return await config; +} diff --git a/packages/cli-v3/src/utilities/session.ts b/packages/cli-v3/src/utilities/session.ts index 71ce8d036..5c5308004 100644 --- a/packages/cli-v3/src/utilities/session.ts +++ b/packages/cli-v3/src/utilities/session.ts @@ -1,4 +1,4 @@ -import { ApiClient } from "../apiClient.js"; +import { CliApiClient } from "../apiClient.js"; import { readAuthConfigFile } from "./configFiles.js"; export async function isLoggedIn() { @@ -8,7 +8,7 @@ export async function isLoggedIn() { return { ok: false as const, error: "You must login first" }; } - const apiClient = new ApiClient(config.apiUrl, config.accessToken); + const apiClient = new CliApiClient(config.apiUrl, config.accessToken); const userData = await apiClient.whoAmI(); if (!userData.success) { diff --git a/packages/cli-v3/src/utilities/taskFiles.ts b/packages/cli-v3/src/utilities/taskFiles.ts new file mode 100644 index 000000000..175e4f6bb --- /dev/null +++ b/packages/cli-v3/src/utilities/taskFiles.ts @@ -0,0 +1,70 @@ +import fs from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; +import { TaskFile } from "../types"; +import { ResolvedConfig } from "@trigger.dev/core/v3"; + +export function createTaskFileImports(taskFiles: TaskFile[]) { + return taskFiles + .map( + (taskFile) => + `import * as ${taskFile.importName} from "./${taskFile.importPath}"; TaskFileImports["${ + taskFile.importName + }"] = ${taskFile.importName}; TaskFiles["${taskFile.importName}"] = ${JSON.stringify( + taskFile + )};` + ) + .join("\n"); +} + +// Find all the top-level .js or .ts files in the trigger directories +export async function gatherTaskFiles(config: ResolvedConfig): Promise> { + const taskFiles: Array = []; + + for (const triggerDir of config.triggerDirectories) { + const files = await fs.promises.readdir(triggerDir, { withFileTypes: true }); + for (const file of files) { + if (!file.isFile()) continue; + if (!file.name.endsWith(".js") && !file.name.endsWith(".ts")) continue; + + const fullPath = join(triggerDir, file.name); + + const filePath = relative(config.projectDir, fullPath); + const importPath = filePath.replace(/\.(js|ts)$/, ""); + const importName = importPath.replace(/\//g, "_"); + + taskFiles.push({ triggerDir, importPath, importName, filePath }); + } + } + + return taskFiles; +} + +export function resolveTriggerDirectories(dirs: string[]): string[] { + return dirs.map((dir) => resolve(dir)); +} + +const IGNORED_DIRS = ["node_modules", ".git", "dist", "build"]; + +export async function findTriggerDirectories(filePath: string): Promise { + const dirPath = dirname(filePath); + return getTriggerDirectories(dirPath); +} + +async function getTriggerDirectories(dirPath: string): Promise { + const entries = await fs.promises.readdir(dirPath, { withFileTypes: true }); + const triggerDirectories: string[] = []; + + for (const entry of entries) { + if (!entry.isDirectory() || IGNORED_DIRS.includes(entry.name)) continue; + + const fullPath = join(dirPath, entry.name); + + if (entry.name === "trigger") { + triggerDirectories.push(fullPath); + } + + triggerDirectories.push(...(await getTriggerDirectories(fullPath))); + } + + return triggerDirectories; +} diff --git a/packages/cli-v3/tsconfig.json b/packages/cli-v3/tsconfig.json index b4976837f..7830ccaca 100644 --- a/packages/cli-v3/tsconfig.json +++ b/packages/cli-v3/tsconfig.json @@ -25,7 +25,9 @@ "jsx": "react", "paths": { "@trigger.dev/core/v3": ["../core/src/v3"], - "@trigger.dev/core/v3/*": ["../core/src/v3/*"] + "@trigger.dev/core/v3/*": ["../core/src/v3/*"], + "@trigger.dev/core-apps": ["../core-apps/src"], + "@trigger.dev/core-apps/*": ["../core-apps/src/*"] } }, "exclude": ["node_modules"] diff --git a/packages/cli-v3/tsup.prod-facade.config.ts b/packages/cli-v3/tsup.prod-facade.config.ts new file mode 100644 index 000000000..2c03c4520 --- /dev/null +++ b/packages/cli-v3/tsup.prod-facade.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + clean: false, + dts: true, + tsconfig: "tsconfig.json", + splitting: false, + entry: ["src/prod-facade.ts"], + format: ["esm"], + minify: false, + metafile: false, + sourcemap: true, + target: "esnext", + outDir: "dist", + noExternal: ["zod", /traceloop/, /opentelemetry/], +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index d1fcb7bc9..d6ca2ca4f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -39,7 +39,7 @@ "@types/gradient-string": "^1.1.2", "@types/inquirer": "^9.0.3", "@types/mock-fs": "^4.13.1", - "@types/node": "16", + "@types/node": "^18", "@types/node-fetch": "^2.6.2", "@types/ws": "^8.5.3", "rimraf": "^3.0.2", diff --git a/packages/core-apps/package.json b/packages/core-apps/package.json new file mode 100644 index 000000000..bebf33384 --- /dev/null +++ b/packages/core-apps/package.json @@ -0,0 +1,40 @@ +{ + "name": "@trigger.dev/core-apps", + "description": "Backend core code used across apps", + "private": true, + "license": "MIT", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "module": "./dist/index.mjs", + "files": [ + "dist" + ], + "exports": { + ".": { + "import": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + }, + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + }, + "./package.json": "./package.json" + }, + "scripts": { + "clean": "rimraf dist", + "build": "npm run clean && npm run build:tsup", + "build:tsup": "tsup --dts-resolve", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@trigger.dev/tsconfig": "workspace:*", + "@trigger.dev/tsup": "workspace:*", + "@types/node": "18", + "rimraf": "^3.0.2", + "tsup": "^8.0.1", + "typescript": "^5.3.0" + }, + "engines": { + "node": ">=18.0.0" + } +} \ No newline at end of file diff --git a/packages/core-apps/src/http.ts b/packages/core-apps/src/http.ts new file mode 100644 index 000000000..60eeb67a0 --- /dev/null +++ b/packages/core-apps/src/http.ts @@ -0,0 +1,39 @@ +import { IncomingMessage, RequestListener } from "http"; + +export const getTextBody = (req: IncomingMessage) => + new Promise((resolve) => { + let body = ""; + req.on("readable", () => { + const chunk = req.read(); + if (chunk) { + body += chunk; + } + }); + req.on("end", () => { + resolve(body); + }); + }); + +export class HttpReply { + constructor(private response: Parameters[1]) {} + + empty(status?: number) { + return this.response.writeHead(status ?? 200).end(); + } + + text(text: string, status?: number, contentType?: string) { + return this.response + .writeHead(status ?? 200, { "Content-Type": contentType || "text/plain" }) + .end(text.endsWith("\n") ? text : `${text}\n`); + } +} + +function getRandomInteger(min: number, max: number) { + const intMin = Math.ceil(min); + const intMax = Math.floor(max); + return Math.floor(Math.random() * (intMax - intMin + 1)) + intMin; +} + +export function getRandomPortNumber() { + return getRandomInteger(8000, 9999); +} diff --git a/packages/core-apps/src/index.ts b/packages/core-apps/src/index.ts new file mode 100644 index 000000000..b121c86fe --- /dev/null +++ b/packages/core-apps/src/index.ts @@ -0,0 +1,2 @@ +export * from "./http"; +export * from "./logger"; diff --git a/packages/core-apps/src/logger.ts b/packages/core-apps/src/logger.ts new file mode 100644 index 000000000..7e1e99feb --- /dev/null +++ b/packages/core-apps/src/logger.ts @@ -0,0 +1,35 @@ +export class SimpleLogger { + #debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? ""); + + constructor(private prefix?: string) {} + + log(arg0: TFirstArg, ...argN: any[]) { + console.log(...this.#getPrefixedArgs(arg0, ...argN)); + + return arg0; + } + + debug(arg0: TFirstArg, ...argN: any[]) { + if (!this.#debugEnabled) { + return arg0; + } + + console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN)); + + return arg0; + } + + error(arg0: TFirstArg, ...argN: any[]) { + console.error(...this.#getPrefixedArgs(arg0, ...argN)); + + return arg0; + } + + #getPrefixedArgs(...args: any[]) { + if (!this.prefix) { + return args; + } + + return [this.prefix, ...args]; + } +} diff --git a/packages/core-apps/tsconfig.build.json b/packages/core-apps/tsconfig.build.json new file mode 100644 index 000000000..f444efc19 --- /dev/null +++ b/packages/core-apps/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "@trigger.dev/tsconfig/node18.json", + "include": ["src/globals.d.ts", "./src/**/*.ts", "tsup.config.ts"], + "compilerOptions": { + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declaration": false, + "declarationMap": false + }, + "exclude": ["node_modules"] +} diff --git a/packages/core-apps/tsconfig.json b/packages/core-apps/tsconfig.json new file mode 100644 index 000000000..ceb26c729 --- /dev/null +++ b/packages/core-apps/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "@trigger.dev/tsconfig/node18.json", + "include": ["src/globals.d.ts", "./src/**/*.ts", "tsup.config.ts", "./test/**/*.ts"], + "compilerOptions": { + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declaration": false, + "declarationMap": false, + "lib": ["DOM", "DOM.Iterable"], + "paths": { + "@trigger.dev/core/v3": ["../core/src/v3"], + "@trigger.dev/core/v3/*": ["../core/src/v3/*"], + "@trigger.dev/tsup/*": ["../../config-packages/tsup/src/*"], + "@trigger.dev/tsup": ["../../config-packages/tsup/src/index"] + } + }, + "exclude": ["node_modules"] +} diff --git a/packages/core-apps/tsup.config.ts b/packages/core-apps/tsup.config.ts new file mode 100644 index 000000000..1914192e5 --- /dev/null +++ b/packages/core-apps/tsup.config.ts @@ -0,0 +1,6 @@ +import { packageOptions, defineConfig } from "@trigger.dev/tsup"; + +export default defineConfig({ + ...packageOptions, + config: "tsconfig.build.json", +}); diff --git a/packages/core/package.json b/packages/core/package.json index 89400f021..db797578f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -74,6 +74,7 @@ "@opentelemetry/sdk-trace-node": "^1.21.0", "@opentelemetry/semantic-conventions": "^1.21.0", "humanize-duration": "^3.27.3", + "socket.io": "^4.7.4", "ulidx": "^2.2.1", "zod": "3.22.3", "zod-error": "1.5.0" @@ -83,7 +84,7 @@ "@trigger.dev/tsup": "workspace:*", "@types/humanize-duration": "^3.27.1", "@types/jest": "^29.5.3", - "@types/node": "16", + "@types/node": "^18", "jest": "^29.6.2", "rimraf": "^3.0.2", "ts-jest": "^29.1.1", diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index 251f4719e..ccefb2c8b 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1,15 +1,14 @@ import { context, propagation } from "@opentelemetry/api"; import { zodfetch } from "../../zodfetch"; -import { - BatchTriggerTaskRequestBody, - BatchTriggerTaskResponse, - GetBatchResponseBody, - TriggerTaskRequestBody, - TriggerTaskResponse, -} from "../schemas/api"; import { taskContextManager } from "../tasks/taskContextManager"; import { SafeAsyncLocalStorage } from "../utils/safeAsyncLocalStorage"; import { getEnvVar } from "../utils/getEnv"; +import { + TriggerTaskRequestBody, + TriggerTaskResponse, + BatchTriggerTaskRequestBody, + BatchTriggerTaskResponse, +} from "../schemas"; export type TriggerOptions = { spanParentAsLink?: boolean; @@ -19,10 +18,14 @@ export type TriggerOptions = { * Trigger.dev v3 API client */ export class ApiClient { + private readonly baseUrl: string; + constructor( - private readonly baseUrl: string, + baseUrl: string, private readonly accessToken: string - ) {} + ) { + this.baseUrl = baseUrl.replace(/\/$/, ""); + } triggerTask(taskId: string, body: TriggerTaskRequestBody, options?: TriggerOptions) { return zodfetch(TriggerTaskResponse, `${this.baseUrl}/api/v1/tasks/${taskId}/trigger`, { diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index da5327040..42c4738a1 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -3,9 +3,11 @@ import { BatchTriggerTaskRequestBody, TriggerTaskRequestBody } from "./schemas"; export * from "./schemas"; export * from "./apiClient"; export * from "./zodMessageHandler"; +export * from "./zodNamespace"; export * from "./errors"; export * from "./runtime-api"; export * from "./logger-api"; +export * from "./types"; export { SemanticInternalAttributes } from "./semanticInternalAttributes"; export { iconStringForSeverity } from "./icons"; export { @@ -28,6 +30,7 @@ export function parseBatchTriggerTaskRequestBody(body: unknown) { export { taskContextManager, TaskContextSpanProcessor } from "./tasks/taskContextManager"; export type { RuntimeManager } from "./runtime/manager"; export { DevRuntimeManager } from "./runtime/devRuntimeManager"; +export { ProdRuntimeManager } from "./runtime/prodRuntimeManager"; export { TriggerTracer } from "./tracer"; export type { TaskLogger } from "./logger/taskLogger"; diff --git a/packages/core/src/v3/runtime/devRuntimeManager.ts b/packages/core/src/v3/runtime/devRuntimeManager.ts index 6d63d10bd..ac8b76af7 100644 --- a/packages/core/src/v3/runtime/devRuntimeManager.ts +++ b/packages/core/src/v3/runtime/devRuntimeManager.ts @@ -82,9 +82,16 @@ export class DevRuntimeManager implements RuntimeManager { resumeTask(completion: TaskRunExecutionResult, execution: TaskRunExecution): void { const wait = this._taskWaits.get(execution.run.id); - if (wait) { - wait.resolve(completion); - this._taskWaits.delete(execution.run.id); + if (!wait) { + return; } + + if (completion.ok) { + wait.resolve(completion); + } else { + wait.reject(completion); + } + + this._taskWaits.delete(execution.run.id); } } diff --git a/packages/core/src/v3/runtime/prodRuntimeManager.ts b/packages/core/src/v3/runtime/prodRuntimeManager.ts new file mode 100644 index 000000000..d4e751824 --- /dev/null +++ b/packages/core/src/v3/runtime/prodRuntimeManager.ts @@ -0,0 +1,114 @@ +import { + BatchTaskRunExecutionResult, + TaskMetadataWithFilePath, + TaskRunContext, + TaskRunExecution, + TaskRunExecutionResult, + childToWorkerMessages, +} from "../schemas"; +import { ZodMessageSender } from "../zodMessageHandler"; +import { RuntimeManager } from "./manager"; + +export class ProdRuntimeManager implements RuntimeManager { + _taskWaits: Map< + string, + { resolve: (value: TaskRunExecutionResult) => void; reject: (err?: any) => void } + > = new Map(); + + _batchWaits: Map< + string, + { resolve: (value: BatchTaskRunExecutionResult) => void; reject: (err?: any) => void } + > = new Map(); + + _tasks: Map = new Map(); + + constructor(private sender: ZodMessageSender) {} + + disable(): void { + // do nothing + } + + registerTasks(tasks: TaskMetadataWithFilePath[]): void { + for (const task of tasks) { + this._tasks.set(task.id, task); + } + } + + getTaskMetadata(id: string): TaskMetadataWithFilePath | undefined { + return this._tasks.get(id); + } + + async waitForDuration(ms: number): Promise { + if (ms > 30_000) { + // TODO: sender with ack support + await this.sender.send("WAIT_FOR_DURATION", { ms }); + // TODO: resolve after resume signal instead + } + + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } + + async waitUntil(date: Date): Promise { + return this.waitForDuration(date.getTime() - Date.now()); + } + + async waitForTask(params: { id: string; ctx: TaskRunContext }): Promise { + const promise = new Promise((resolve, reject) => { + this._taskWaits.set(params.id, { resolve, reject }); + }); + + await this.sender.send("WAIT_FOR_TASK", { + id: params.id, + }); + + return await promise; + } + + async waitForBatch(params: { + id: string; + runs: string[]; + ctx: TaskRunContext; + }): Promise { + if (!params.runs.length) { + return Promise.resolve({ id: params.id, items: [] }); + } + + const promise = Promise.all( + params.runs.map((runId) => { + return new Promise((resolve, reject) => { + this._taskWaits.set(runId, { resolve, reject }); + }); + }) + ); + + await this.sender.send("WAIT_FOR_BATCH", { + id: params.id, + runs: params.runs, + }); + + const results = await promise; + + return { + id: params.id, + items: results, + }; + } + + resumeTask(completion: TaskRunExecutionResult, execution: TaskRunExecution): void { + const wait = this._taskWaits.get(execution.run.id); + + if (!wait) { + return; + } + + if (completion.ok) { + wait.resolve(completion); + } else { + wait.reject(completion); + } + + this._taskWaits.delete(execution.run.id); + } +} diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 010e69df4..3eeb266c1 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { BackgroundWorkerMetadata } from "./resources"; +import { BackgroundWorkerMetadata, ImageDetailsMetadata } from "./resources"; import { QueueOptions } from "./messages"; export const WhoAmIResponseSchema = z.object({ @@ -88,3 +88,16 @@ export const GetEnvironmentVariablesResponseBody = z.object({ export type GetEnvironmentVariablesResponseBody = z.infer< typeof GetEnvironmentVariablesResponseBody >; + +export const CreateImageDetailsRequestBody = z.object({ + metadata: ImageDetailsMetadata, +}); + +export type CreateImageDetailsRequestBody = z.infer; + +export const CreateImageDetailsResponse = z.object({ + id: z.string(), + contentHash: z.string(), +}); + +export type CreateImageDetailsResponse = z.infer; diff --git a/packages/core/src/v3/schemas/index.ts b/packages/core/src/v3/schemas/index.ts index 371d4cef7..8a5076544 100644 --- a/packages/core/src/v3/schemas/index.ts +++ b/packages/core/src/v3/schemas/index.ts @@ -3,6 +3,7 @@ export * from "./api"; export * from "./resources"; export * from "./common"; export * from "./messages"; +export * from "./schemas"; export * from "./style"; export * from "./fetch"; export * from "./eventFilter"; diff --git a/packages/core/src/v3/schemas/messages.ts b/packages/core/src/v3/schemas/messages.ts index 22c637e83..daa7f07b8 100644 --- a/packages/core/src/v3/schemas/messages.ts +++ b/packages/core/src/v3/schemas/messages.ts @@ -9,11 +9,35 @@ export const TaskRunExecutionPayload = z.object({ export type TaskRunExecutionPayload = z.infer; +export const ProdTaskRunExecution = TaskRunExecution.extend({ + worker: z.object({ + id: z.string(), + contentHash: z.string(), + version: z.string(), + }), +}); + +export type ProdTaskRunExecution = z.infer; + +export const ProdTaskRunExecutionPayload = z.object({ + execution: ProdTaskRunExecution, + traceContext: z.record(z.unknown()), + environment: z.record(z.string()).optional(), +}); + +export type ProdTaskRunExecutionPayload = z.infer; + export const BackgroundWorkerServerMessages = z.discriminatedUnion("type", [ z.object({ type: z.literal("EXECUTE_RUNS"), payloads: z.array(TaskRunExecutionPayload), }), + z.object({ + type: z.literal("SCHEDULE_ATTEMPT"), + id: z.string(), + image: z.string(), + envId: z.string(), + }), ]); export type BackgroundWorkerServerMessages = z.infer; @@ -178,4 +202,17 @@ export const childToWorkerMessages = { id: z.string(), }), READY_TO_DISPOSE: z.undefined(), + WAIT_FOR_DURATION: z.object({ + version: z.literal("v1").default("v1"), + ms: z.number(), + }), + WAIT_FOR_TASK: z.object({ + version: z.literal("v1").default("v1"), + id: z.string(), + }), + WAIT_FOR_BATCH: z.object({ + version: z.literal("v1").default("v1"), + id: z.string(), + runs: z.string().array(), + }), }; diff --git a/packages/core/src/v3/schemas/resources.ts b/packages/core/src/v3/schemas/resources.ts index a861cde14..612d8a2db 100644 --- a/packages/core/src/v3/schemas/resources.ts +++ b/packages/core/src/v3/schemas/resources.ts @@ -19,3 +19,10 @@ export const BackgroundWorkerMetadata = z.object({ }); export type BackgroundWorkerMetadata = z.infer; + +export const ImageDetailsMetadata = z.object({ + contentHash: z.string(), + imageTag: z.string(), +}); + +export type ImageDetailsMetadata = z.infer; diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts new file mode 100644 index 000000000..2f3de8c2a --- /dev/null +++ b/packages/core/src/v3/schemas/schemas.ts @@ -0,0 +1,117 @@ +import { z } from "zod"; +import { RequireKeys } from "../types"; +import { TaskRunExecution, TaskRunExecutionResult } from "./common"; +import { ProdTaskRunExecution } from "./messages"; +import { TaskResource } from "./resources"; + +export const Config = z.object({ + project: z.string(), + triggerDirectories: z.string().array().optional(), + triggerUrl: z.string().optional(), + projectDir: z.string().optional(), +}); + +export type Config = z.infer; +export type ResolvedConfig = RequireKeys< + Config, + "triggerDirectories" | "triggerUrl" | "projectDir" +>; + +export const Machine = z.object({ + cpu: z.string().default("1").optional(), + memory: z.string().default("500Mi").optional(), +}); + +export type Machine = z.infer; + +export const ProviderToPlatformMessages = { + LOG: z.object({ + version: z.literal("v1").default("v1"), + data: z.string(), + }), +}; + +export const PlatformToProviderMessages = { + HEALTH: z.object({ + // TODO: callback: (ack: { status: "ok" }) => void + version: z.literal("v1").default("v1"), + }), + INDEX: z.object({ + version: z.literal("v1").default("v1"), + imageTag: z.string(), + contentHash: z.string(), + envId: z.string(), + }), + INVOKE: z.object({ + version: z.literal("v1").default("v1"), + name: z.string(), + machine: Machine, + }), + RESTORE: z.object({ + version: z.literal("v1").default("v1"), + id: z.string(), + attemptId: z.string(), + type: z.enum(["DOCKER", "KUBERNETES"]), + location: z.string(), + reason: z.string().optional(), + }), + DELETE: z.object({ + // TODO: callback: (ack: { message: string }) => void + version: z.literal("v1").default("v1"), + name: z.string(), + }), + GET: z.object({ + version: z.literal("v1").default("v1"), + name: z.string(), + }), +}; + +export const CoordinatorToPlatformMessages = { + LOG: z.object({ + version: z.literal("v1").default("v1"), + metadata: z.any(), + text: z.string(), + }), + CREATE_WORKER: z.object({ + version: z.literal("v1").default("v1"), + projectRef: z.string(), + envId: z.string(), + metadata: z.object({ + cliPackageVersion: z.string(), + contentHash: z.string(), + packageVersion: z.string(), + tasks: TaskResource.array(), + }), + }), + // TODO: callback: (ack: { success: false } | { success: true; payload: ProdTaskRunExecutionPayload }) => void + READY_FOR_EXECUTION: z.object({ + version: z.literal("v1").default("v1"), + attemptId: z.string(), + }), + TASK_RUN_COMPLETED: z.object({ + version: z.literal("v1").default("v1"), + execution: ProdTaskRunExecution, + completion: TaskRunExecutionResult, + }), + TASK_HEARTBEAT: z.object({ + version: z.literal("v1").default("v1"), + attemptFriendlyId: z.string(), + }), + CHECKPOINT_CREATED: z.object({ + version: z.literal("v1").default("v1"), + attemptId: z.string(), + docker: z.boolean(), + location: z.string(), + reason: z.string().optional(), + }), +}; + +export const PlatformToCoordinatorMessages = { + RESUME: z.object({ + version: z.literal("v1").default("v1"), + attemptId: z.string(), + image: z.string(), + completions: TaskRunExecutionResult.array(), + executions: TaskRunExecution.array(), + }), +}; diff --git a/packages/core/src/v3/types/index.ts b/packages/core/src/v3/types/index.ts new file mode 100644 index 000000000..90fc43be6 --- /dev/null +++ b/packages/core/src/v3/types/index.ts @@ -0,0 +1,2 @@ +export * from "./socketIo"; +export * from "./utils"; diff --git a/packages/core/src/v3/types/socketIo.ts b/packages/core/src/v3/types/socketIo.ts new file mode 100644 index 000000000..7075a7611 --- /dev/null +++ b/packages/core/src/v3/types/socketIo.ts @@ -0,0 +1,129 @@ +import { + TaskResource, + ProdTaskRunExecutionPayload, + TaskRunExecutionResult, + TaskRunExecution, + ProdTaskRunExecution, +} from "../schemas"; + +export type VersionedMessage = { version: "v1" } & TMessage; + +// provider <--> platform +export interface ProviderClientToServerEvents { + LOG: (message: VersionedMessage<{ data: string }>) => void; +} + +export interface ProviderServerToClientEvents { + HEALTH: (message: VersionedMessage<{}>, callback: (ack: { status: "ok" }) => void) => void; + INDEX: ( + message: VersionedMessage<{ imageTag: string; contentHash: string; envId: string }> + ) => void; + RESTORE: ( + message: VersionedMessage<{ + id: string; + attemptId: string; + type: "DOCKER" | "KUBERNETES"; + location: string; + reason?: string; + }> + ) => void; + DELETE: ( + message: VersionedMessage<{ name: string }>, + callback: (ack: { message: string }) => void + ) => void; + GET: (message: VersionedMessage<{ name: string }>) => void; +} + +// coordinator <--> prod worker +export interface ProdWorkerToCoordinatorEvents { + LOG: (message: VersionedMessage<{ text: string }>, callback: () => {}) => void; + INDEX_TASKS: ( + message: VersionedMessage<{ + tasks: TaskResource[]; + packageVersion: string; + }>, + callback: (params: { success: boolean }) => {} + ) => void; + READY_FOR_EXECUTION: (message: VersionedMessage<{ attemptId: string }>) => void; + TASK_HEARTBEAT: (message: VersionedMessage<{ attemptFriendlyId: string }>) => void; + WAIT_FOR_BATCH: (message: VersionedMessage<{ id: string; runs: string[] }>) => void; + WAIT_FOR_DURATION: ( + message: VersionedMessage<{ ms: number }>, + callback: (ack: { success: boolean }) => void + ) => void; + WAIT_FOR_TASK: (message: VersionedMessage<{ id: string }>) => void; +} + +export interface CoordinatorToProdWorkerEvents { + RESUME: ( + message: VersionedMessage<{ + attemptId: string; + image: string; + completions: TaskRunExecutionResult[]; + executions: TaskRunExecution[]; + }> + ) => void; + EXECUTE_TASK_RUN: ( + message: VersionedMessage<{ payload: ProdTaskRunExecutionPayload }>, + callback: (ack: { completion: TaskRunExecutionResult }) => void + ) => void; +} + +export interface ProdWorkerSocketData { + cliPackageVersion: string; + contentHash: string; + projectRef: string; + envId: string; + attemptId: string; + podName: string; +} + +// coordinator <--> platform +export interface CoordinatorToPlatformEvents { + LOG: (message: VersionedMessage<{ metadata: any; text: string }>) => void; + CREATE_WORKER: ( + message: VersionedMessage<{ + projectRef: string; + envId: string; + metadata: { + cliPackageVersion: string; + contentHash: string; + packageVersion: string; + tasks: TaskResource[]; + }; + }>, + callback: (ack: { success: boolean }) => void + ) => void; + READY_FOR_EXECUTION: ( + message: VersionedMessage<{ attemptId: string }>, + callback: ( + ack: { success: false } | { success: true; payload: ProdTaskRunExecutionPayload } + ) => void + ) => void; + TASK_RUN_COMPLETED: ( + message: VersionedMessage<{ + execution: ProdTaskRunExecution; + completion: TaskRunExecutionResult; + }> + ) => void; + TASK_HEARTBEAT: (message: VersionedMessage<{ attemptFriendlyId: string }>) => void; + CHECKPOINT_CREATED: ( + message: VersionedMessage<{ + attemptId: string; + docker: boolean; + location: string; + reason?: string; + }> + ) => void; +} + +export interface PlatformToCoordinatorEvents { + RESUME: ( + message: VersionedMessage<{ + attemptId: string; + image: string; + completions: TaskRunExecutionResult[]; + executions: TaskRunExecution[]; + }> + ) => void; +} diff --git a/packages/cli-v3/src/utilities/requiredKeys.ts b/packages/core/src/v3/types/utils.ts similarity index 100% rename from packages/cli-v3/src/utilities/requiredKeys.ts rename to packages/core/src/v3/types/utils.ts diff --git a/packages/core/src/v3/zodMessageHandler.ts b/packages/core/src/v3/zodMessageHandler.ts index 599f56ea1..2072cb618 100644 --- a/packages/core/src/v3/zodMessageHandler.ts +++ b/packages/core/src/v3/zodMessageHandler.ts @@ -5,7 +5,7 @@ export interface ZodMessageCatalogSchema { } export type ZodMessageHandlers = Partial<{ - [K in keyof TCatalogSchema]: (payload: z.infer) => Promise; + [K in keyof TCatalogSchema]: (payload: z.infer) => Promise; }>; export type ZodMessageHandlerOptions = { @@ -31,6 +31,10 @@ const messageSchema = z.object({ payload: z.unknown(), }); +interface EventEmitterLike { + on(eventName: string | symbol, listener: (...args: any[]) => void): this; +} + export class ZodMessageHandler { #schema: TMessageCatalog; #handlers: ZodMessageHandlers | undefined; @@ -50,10 +54,13 @@ export class ZodMessageHandler const handler = this.#handlers[parsedMessage.type]; if (!handler) { - throw new Error(`Unknown message type: ${String(parsedMessage.type)}`); + console.error(`No handler for message type: ${String(parsedMessage.type)}`); + return; } - await handler(parsedMessage.payload); + const ack = await handler(parsedMessage.payload); + + return ack; } public parseMessage(message: unknown): MessageFromCatalog { @@ -80,6 +87,35 @@ export class ZodMessageHandler payload: parsedPayload.data, }; } + + public registerHandlers(emitter: EventEmitterLike, logger?: (...args: any[]) => void) { + const log = logger ?? console.log; + + if (!this.#handlers) { + log("No handlers provided"); + return; + } + + for (const eventName of Object.keys(this.#schema)) { + emitter.on(eventName, async (message: any, callback?: any): Promise => { + log(`handling ${eventName}`, message); + + let ack; + + if ("payload" in message) { + ack = await this.handleMessage({ type: eventName, ...message }); + } else { + // Handle messages not sent by ZodMessageSender + const { version, ...payload } = message; + ack = await this.handleMessage({ type: eventName, version, payload }); + } + + if (callback && typeof callback === "function") { + callback(ack); + } + }); + } + } } type ZodMessageSenderCallback = (message: { @@ -121,3 +157,7 @@ export class ZodMessageSender { await this.#sender({ type, payload, version: "v1" }); } } + +export type MessageCatalogToSocketIoEvents = { + [K in keyof TCatalog]: (message: z.infer) => void; +}; diff --git a/packages/core/src/v3/zodNamespace.ts b/packages/core/src/v3/zodNamespace.ts new file mode 100644 index 000000000..63a9c5a73 --- /dev/null +++ b/packages/core/src/v3/zodNamespace.ts @@ -0,0 +1,158 @@ +import { DisconnectReason, Namespace, Server, Socket } from "socket.io"; +import { + ZodMessageCatalogSchema, + ZodMessageHandlerOptions, + MessageCatalogToSocketIoEvents, + ZodMessageHandler, + ZodMessageSender, +} from "./zodMessageHandler"; + +interface ExtendedError extends Error { + data?: any; +} + +export type ZodSocket< + TClientMessages extends ZodMessageCatalogSchema, + TServerMessages extends ZodMessageCatalogSchema, +> = Socket< + MessageCatalogToSocketIoEvents, + MessageCatalogToSocketIoEvents +>; + +interface ZodNamespaceOptions< + TClientMessages extends ZodMessageCatalogSchema, + TServerMessages extends ZodMessageCatalogSchema, +> { + io: Server; + name: string; + clientMessages: TClientMessages; + serverMessages: TServerMessages; + messageHandler?: ZodMessageHandlerOptions["messages"]; + authToken?: string; + preAuth?: ( + socket: ZodSocket, + next: (err?: ExtendedError) => void + ) => void; + postAuth?: ( + socket: ZodSocket, + next: (err?: ExtendedError) => void + ) => void; + onConnection?: ( + socket: ZodSocket, + handler: ZodMessageHandler, + sender: ZodMessageSender, + logger: (...args: any[]) => void + ) => Promise; + onDisconnect?: ( + socket: ZodSocket, + reason: DisconnectReason, + description: any, + logger: (...args: any[]) => void + ) => Promise; + onError?: ( + socket: ZodSocket, + err: Error, + logger: (...args: any[]) => void + ) => Promise; +} + +export class ZodNamespace< + TClientMessages extends ZodMessageCatalogSchema, + TServerMessages extends ZodMessageCatalogSchema, +> { + #handler: ZodMessageHandler; + sender: ZodMessageSender; + + io: Server; + namespace: Namespace< + MessageCatalogToSocketIoEvents, + MessageCatalogToSocketIoEvents + >; + + constructor(opts: ZodNamespaceOptions) { + this.#handler = new ZodMessageHandler({ + schema: opts.clientMessages, + messages: opts.messageHandler, + }); + + this.io = opts.io; + + this.namespace = this.io.of(opts.name); + + this.sender = new ZodMessageSender({ + schema: opts.serverMessages, + sender: async (message) => { + return new Promise((resolve, reject) => { + try { + // @ts-expect-error + this.namespace.emit(message.type, message.payload); + resolve(); + } catch (err) { + reject(err); + } + }); + }, + }); + + if (opts.preAuth) { + this.namespace.use(opts.preAuth); + } + + if (opts.authToken) { + this.namespace.use((socket, next) => { + const logger = createLogger(`[${opts.name}][${socket.id}][auth]`); + + const { auth } = socket.handshake; + + if (!("token" in auth)) { + logger("no token"); + return socket.disconnect(true); + } + + if (auth.token !== opts.authToken) { + logger("invalid token"); + return socket.disconnect(true); + } + + logger("success"); + + next(); + }); + } + + if (opts.postAuth) { + this.namespace.use(opts.postAuth); + } + + this.namespace.on("connection", async (socket) => { + const logger = createLogger(`[${opts.name}][${socket.id}]`); + logger("connection"); + + this.#handler.registerHandlers(socket, logger); + + socket.on("disconnect", async (reason, description) => { + logger("disconnect", { reason, description }); + + if (opts.onDisconnect) { + await opts.onDisconnect(socket, reason, description, logger); + } + }); + + socket.on("error", async (error) => { + logger("error", error); + + if (opts.onError) { + await opts.onError(socket, error, logger); + } + }); + + if (opts.onConnection) { + await opts.onConnection(socket, this.#handler, this.sender, logger); + } + }); + } +} + +function createLogger(prefix: string) { + return (...args: any[]) => console.log(prefix, ...args); +} diff --git a/packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql b/packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql new file mode 100644 index 000000000..8b0ed3e14 --- /dev/null +++ b/packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql @@ -0,0 +1,30 @@ +-- CreateTable +CREATE TABLE "ImageDetails" ( + "id" TEXT NOT NULL, + "friendlyId" TEXT NOT NULL, + "contentHash" TEXT NOT NULL, + "tag" TEXT NOT NULL, + "backgroundWorkerId" TEXT, + "projectId" TEXT NOT NULL, + "runtimeEnvironmentId" TEXT NOT NULL, + "metadata" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "ImageDetails_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "ImageDetails_friendlyId_key" ON "ImageDetails"("friendlyId"); + +-- CreateIndex +CREATE UNIQUE INDEX "ImageDetails_projectId_runtimeEnvironmentId_tag_key" ON "ImageDetails"("projectId", "runtimeEnvironmentId", "tag"); + +-- AddForeignKey +ALTER TABLE "ImageDetails" ADD CONSTRAINT "ImageDetails_backgroundWorkerId_fkey" FOREIGN KEY ("backgroundWorkerId") REFERENCES "BackgroundWorker"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ImageDetails" ADD CONSTRAINT "ImageDetails_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ImageDetails" ADD CONSTRAINT "ImageDetails_runtimeEnvironmentId_fkey" FOREIGN KEY ("runtimeEnvironmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql b/packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql new file mode 100644 index 000000000..d53196da1 --- /dev/null +++ b/packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql @@ -0,0 +1,11 @@ +/* + Warnings: + + - A unique constraint covering the columns `[projectId,runtimeEnvironmentId,contentHash]` on the table `ImageDetails` will be added. If there are existing duplicate values, this will fail. + +*/ +-- DropIndex +DROP INDEX "ImageDetails_projectId_runtimeEnvironmentId_tag_key"; + +-- CreateIndex +CREATE UNIQUE INDEX "ImageDetails_projectId_runtimeEnvironmentId_contentHash_key" ON "ImageDetails"("projectId", "runtimeEnvironmentId", "contentHash"); diff --git a/packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql b/packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql new file mode 100644 index 000000000..428ab2eb8 --- /dev/null +++ b/packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql @@ -0,0 +1,30 @@ +-- CreateEnum +CREATE TYPE "CheckpointType" AS ENUM ('DOCKER', 'KUBERNETES'); + +-- CreateTable +CREATE TABLE "Checkpoint" ( + "id" TEXT NOT NULL, + "friendlyId" TEXT NOT NULL, + "type" "CheckpointType" NOT NULL, + "location" TEXT NOT NULL, + "reason" TEXT, + "attemptId" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "runtimeEnvironmentId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "Checkpoint_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Checkpoint_friendlyId_key" ON "Checkpoint"("friendlyId"); + +-- AddForeignKey +ALTER TABLE "Checkpoint" ADD CONSTRAINT "Checkpoint_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "TaskRunAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Checkpoint" ADD CONSTRAINT "Checkpoint_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Checkpoint" ADD CONSTRAINT "Checkpoint_runtimeEnvironmentId_fkey" FOREIGN KEY ("runtimeEnvironmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index df1cec219..ad45c12c4 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -389,6 +389,8 @@ model RuntimeEnvironment { taskQueues TaskQueue[] batchTaskRuns BatchTaskRun[] environmentVariableValues EnvironmentVariableValue[] + imageDetails ImageDetails[] + checkpoints Checkpoint[] @@unique([projectId, slug, orgMemberId]) @@unique([projectId, shortcode]) @@ -432,6 +434,8 @@ model Project { taskTags TaskTag[] taskQueues TaskQueue[] environmentVariables EnvironmentVariable[] + imageDetails ImageDetails[] + checkpoints Checkpoint[] } enum ProjectVersion { @@ -1513,9 +1517,10 @@ model BackgroundWorker { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - tasks BackgroundWorkerTask[] - attempts TaskRunAttempt[] - lockedRuns TaskRun[] + tasks BackgroundWorkerTask[] + attempts TaskRunAttempt[] + lockedRuns TaskRun[] + imageDetails ImageDetails[] @@unique([projectId, runtimeEnvironmentId, version]) } @@ -1670,6 +1675,8 @@ model TaskRunAttempt { taskRunDependency TaskRunDependency? @relation("dependentAttempt") batchTaskRunDependency BatchTaskRun? + checkpoints Checkpoint[] + @@unique([taskRunId, number]) } @@ -1884,3 +1891,55 @@ model EnvironmentVariableValue { @@unique([variableId, environmentId]) } + +model ImageDetails { + id String @id @default(cuid()) + + friendlyId String @unique + + contentHash String + tag String + + backgroundWorker BackgroundWorker? @relation(fields: [backgroundWorkerId], references: [id], onDelete: Cascade, onUpdate: Cascade) + backgroundWorkerId String? + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + runtimeEnvironmentId String + + metadata Json + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([projectId, runtimeEnvironmentId, contentHash]) +} + +model Checkpoint { + id String @id @default(cuid()) + + friendlyId String @unique + + type CheckpointType + location String + reason String? + + attempt TaskRunAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade, onUpdate: Cascade) + attemptId String + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + runtimeEnvironmentId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +enum CheckpointType { + DOCKER + KUBERNETES +} diff --git a/packages/emails/package.json b/packages/emails/package.json index 48f855354..9604d15cd 100644 --- a/packages/emails/package.json +++ b/packages/emails/package.json @@ -29,7 +29,7 @@ }, "devDependencies": { "@trigger.dev/tsconfig": "workspace:*", - "@types/node": "16", + "@types/node": "^18", "@types/react": "18.2.17", "typescript": "^4.9.4" }, diff --git a/packages/otlp-importer/README.md b/packages/otlp-importer/README.md index 0e5f7e507..f22a137bb 100644 --- a/packages/otlp-importer/README.md +++ b/packages/otlp-importer/README.md @@ -1,17 +1,25 @@ +# OTLP Importer + ## Getting started Install dependencies: -```sh +```sh Mac brew install protobuf ``` -# Submodules +```sh Linux +apt install -y protobuf-compiler +``` + +Alternatively, follow the [manual install instructions](https://github.com/protocolbuffers/protobuf?tab=readme-ov-file#protobuf-compiler-installation) for the protobuf compiler. + +## Submodules **Submodule is always pointing to certain revision number. So updating the submodule repo will not have impact on your code. Knowing this if you want to change the submodule to point to a different version (when for example proto has changed) here is how to do it:** -## Updating submodule to point to certain revision number +### Updating submodule to point to certain revision number 1. Make sure you are in the same folder as this instruction diff --git a/packages/otlp-importer/package.json b/packages/otlp-importer/package.json index 0d4848760..c436a2f25 100644 --- a/packages/otlp-importer/package.json +++ b/packages/otlp-importer/package.json @@ -40,7 +40,7 @@ "@trigger.dev/tsconfig": "workspace:*", "@trigger.dev/tsup": "workspace:*", "@types/jest": "^29.5.3", - "@types/node": "16", + "@types/node": "^18", "jest": "^29.6.2", "rimraf": "^3.0.2", "ts-jest": "^29.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 489f33fac..ec59a1fe7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -55,6 +55,77 @@ importers: vite-tsconfig-paths: 4.0.5 vitest: 0.28.5 + apps/coordinator: + specifiers: + '@trigger.dev/core': workspace:* + '@trigger.dev/core-apps': workspace:* + '@types/node': ^18 + dotenv: ^16.4.2 + esbuild: ^0.19.11 + execa: ^8.0.1 + prom-client: ^15.1.0 + socket.io: ^4.7.4 + socket.io-client: ^4.7.4 + tsx: ^4.7.0 + typescript: ^5.3.3 + dependencies: + '@trigger.dev/core': link:../../packages/core + '@trigger.dev/core-apps': link:../../packages/core-apps + execa: 8.0.1 + prom-client: 15.1.0 + socket.io: 4.7.4 + socket.io-client: 4.7.4 + devDependencies: + '@types/node': 18.17.1 + dotenv: 16.4.4 + esbuild: 0.19.11 + tsx: 4.7.1 + typescript: 5.3.3 + + apps/docker-provider: + specifiers: + '@trigger.dev/core': workspace:* + '@trigger.dev/core-apps': workspace:* + '@types/node': ^18.19.8 + dotenv: ^16.4.2 + esbuild: ^0.19.11 + execa: ^8.0.1 + socket.io-client: ^4.7.4 + tsx: ^4.7.0 + typescript: ^5.3.3 + dependencies: + '@trigger.dev/core': link:../../packages/core + '@trigger.dev/core-apps': link:../../packages/core-apps + execa: 8.0.1 + socket.io-client: 4.7.4 + devDependencies: + '@types/node': 18.19.20 + dotenv: 16.4.4 + esbuild: 0.19.11 + tsx: 4.7.1 + typescript: 5.3.3 + + apps/kubernetes-provider: + specifiers: + '@kubernetes/client-node': ^0.20.0 + '@trigger.dev/core': workspace:* + '@trigger.dev/core-apps': workspace:* + dotenv: ^16.4.2 + esbuild: ^0.19.11 + socket.io-client: ^4.7.4 + tsx: ^4.7.0 + typescript: ^5.3.3 + dependencies: + '@kubernetes/client-node': 0.20.0 + '@trigger.dev/core': link:../../packages/core + '@trigger.dev/core-apps': link:../../packages/core-apps + socket.io-client: 4.7.4 + devDependencies: + dotenv: 16.4.4 + esbuild: 0.19.11 + tsx: 4.7.1 + typescript: 5.3.3 + apps/proxy: specifiers: '@aws-sdk/client-sqs': ^3.445.0 @@ -196,6 +267,7 @@ importers: cuid: ^2.1.8 datepicker: link:@types/@react-aria/datepicker emails: workspace:* + engine.io: ^6.5.4 esbuild: ^0.15.10 eslint: ^8.24.0 eslint-config-prettier: ^8.5.0 @@ -245,6 +317,7 @@ importers: simple-oauth2: ^5.0.0 simplur: ^3.0.1 slug: ^6.0.0 + socket.io: ^4.7.4 sonner: ^1.0.3 sqs-consumer: ^7.4.0 storybook: ^7.0.7 @@ -379,6 +452,7 @@ importers: simple-oauth2: 5.0.0 simplur: 3.0.1 slug: 6.1.0 + socket.io: 4.7.4 sonner: 1.0.3_biqbaboplfbrettd7655fr4n2y sqs-consumer: 7.5.0_hzguu36iioy52ghs7zxwltx2ia tailwind-merge: 1.12.0 @@ -437,6 +511,7 @@ importers: '@typescript-eslint/parser': 5.59.6_a3vgt4pcadxos4gox5ryckz5rq autoprefixer: 10.4.13_postcss@8.4.21 datepicker: link:@types/@react-aria/datepicker + engine.io: 6.5.4 esbuild: 0.15.18 eslint: 8.31.0 eslint-config-prettier: 8.6.0_eslint@8.31.0 @@ -865,7 +940,7 @@ importers: '@types/gradient-string': ^1.1.2 '@types/inquirer': ^9.0.3 '@types/mock-fs': ^4.13.1 - '@types/node': '16' + '@types/node': ^18 '@types/node-fetch': ^2.6.2 '@types/ws': ^8.5.3 boxen: ^7.1.1 @@ -942,7 +1017,7 @@ importers: '@types/gradient-string': 1.1.2 '@types/inquirer': 9.0.3 '@types/mock-fs': 4.13.1 - '@types/node': 16.18.11 + '@types/node': 18.17.1 '@types/node-fetch': 2.6.2 '@types/ws': 8.5.4 rimraf: 3.0.2 @@ -983,6 +1058,7 @@ importers: chokidar: ^3.5.3 cli-table3: ^0.6.3 commander: ^9.4.1 + cpy-cli: ^5.0.0 degit: ^2.8.4 dotenv: ^16.4.4 esbuild: ^0.19.11 @@ -998,6 +1074,7 @@ importers: node-fetch: ^3.3.0 npm-check-updates: ^16.12.2 npm-run-all: ^4.1.5 + npm-watch: ^0.11.0 object-hash: ^3.0.0 open: ^10.0.3 p-retry: ^6.1.0 @@ -1009,6 +1086,7 @@ importers: react-error-boundary: ^4.0.12 rimraf: ^3.0.2 simple-git: ^3.19.0 + socket.io-client: ^4.7.4 source-map-support: ^0.5.21 supports-color: ^9.4.0 terminal-link: ^3.0.0 @@ -1067,6 +1145,7 @@ importers: react: 18.2.0 react-error-boundary: 4.0.12_react@18.2.0 simple-git: 3.19.0_supports-color@9.4.0 + socket.io-client: 4.7.4_supports-color@9.4.0 source-map-support: 0.5.21 supports-color: 9.4.0 terminal-link: 3.0.0 @@ -1082,7 +1161,9 @@ importers: '@types/object-hash': 3.0.6 '@types/react': 18.2.48 '@types/ws': 8.5.4 + cpy-cli: 5.0.0 npm-run-all: 4.1.5 + npm-watch: 0.11.0 open: 10.0.3 p-retry: 6.1.0 rimraf: 3.0.2 @@ -1113,10 +1194,11 @@ importers: '@trigger.dev/tsup': workspace:* '@types/humanize-duration': ^3.27.1 '@types/jest': ^29.5.3 - '@types/node': '16' + '@types/node': ^18 humanize-duration: ^3.27.3 jest: ^29.6.2 rimraf: ^3.0.2 + socket.io: ^4.7.4 ts-jest: ^29.1.1 tsup: ^8.0.1 typescript: ^5.3.0 @@ -1140,6 +1222,7 @@ importers: '@opentelemetry/sdk-trace-node': 1.21.0_@opentelemetry+api@1.7.0 '@opentelemetry/semantic-conventions': 1.21.0 humanize-duration: 3.27.3 + socket.io: 4.7.4 ulidx: 2.2.1 zod: 3.22.3 zod-error: 1.5.0 @@ -1148,13 +1231,29 @@ importers: '@trigger.dev/tsup': link:../../config-packages/tsup '@types/humanize-duration': 3.27.1 '@types/jest': 29.5.3 - '@types/node': 16.18.11 - jest: 29.6.2_@types+node@16.18.11 + '@types/node': 18.17.1 + jest: 29.6.2_@types+node@18.17.1 rimraf: 3.0.2 ts-jest: 29.1.1_3tyr5rkhps22cjyuxtwz6gzoky - tsup: 8.0.1_a5ztaafw5l4qfghy2hjjuynb34_typescript@5.3.2 + tsup: 8.0.2_typescript@5.3.2 typescript: 5.3.2 + packages/core-apps: + specifiers: + '@trigger.dev/tsconfig': workspace:* + '@trigger.dev/tsup': workspace:* + '@types/node': '18' + rimraf: ^3.0.2 + tsup: ^8.0.1 + typescript: ^5.3.0 + devDependencies: + '@trigger.dev/tsconfig': link:../../config-packages/tsconfig + '@trigger.dev/tsup': link:../../config-packages/tsup + '@types/node': 18.17.1 + rimraf: 3.0.2 + tsup: 8.0.2_typescript@5.3.3 + typescript: 5.3.3 + packages/core-backend: specifiers: '@opentelemetry/api': ^1.7.0 @@ -1206,7 +1305,7 @@ importers: '@react-email/section': ^0.0.1 '@react-email/text': ^0.0.2 '@trigger.dev/tsconfig': workspace:* - '@types/node': '16' + '@types/node': ^18 '@types/react': 18.2.17 react: ^18.2.0 react-email: ^1.6.1 @@ -1234,7 +1333,7 @@ importers: zod: 3.22.3 devDependencies: '@trigger.dev/tsconfig': link:../../config-packages/tsconfig - '@types/node': 16.18.11 + '@types/node': 18.17.1 '@types/react': 18.2.17 typescript: 4.9.5 @@ -1390,7 +1489,7 @@ importers: '@trigger.dev/tsconfig': workspace:* '@trigger.dev/tsup': workspace:* '@types/jest': ^29.5.3 - '@types/node': '16' + '@types/node': ^18 jest: ^29.6.2 long: ^5.2.3 protobufjs: ^7.2.6 @@ -1406,8 +1505,8 @@ importers: '@trigger.dev/tsconfig': link:../../config-packages/tsconfig '@trigger.dev/tsup': link:../../config-packages/tsup '@types/jest': 29.5.3 - '@types/node': 16.18.11 - jest: 29.6.2_@types+node@16.18.11 + '@types/node': 18.17.1 + jest: 29.6.2_@types+node@18.17.1 rimraf: 3.0.2 ts-jest: 29.1.1_5rpxlvaikdwu64qu6f72wnb6nm ts-proto: 1.167.3 @@ -5782,7 +5881,7 @@ packages: resolution: {integrity: sha512-lhc/XLith28QdW0HpHZvZKkorWgmCNT7sVelMHDj3HFdTfdqkwEKvT+aXVQtNAmCC39VJhunDkWhONWB7335mg==} dependencies: '@esbuild-kit/core-utils': 3.0.0 - get-tsconfig: 4.5.0 + get-tsconfig: 4.7.2 dev: true /@esbuild-kit/core-utils/3.0.0: @@ -5796,7 +5895,7 @@ packages: resolution: {integrity: sha512-afmtLf6uqxD5IgwCzomtqCYIgz/sjHzCWZFvfS5+FzeYxOURPUo4QcHtqJxbxWOMOogKriZanN/1bJQE/ZL93A==} dependencies: '@esbuild-kit/core-utils': 3.0.0 - get-tsconfig: 4.5.0 + get-tsconfig: 4.7.2 dev: true /@esbuild-plugins/node-globals-polyfill/0.2.3_esbuild@0.17.19: @@ -7390,7 +7489,7 @@ packages: dependencies: '@inquirer/type': 1.2.0 '@types/mute-stream': 0.0.4 - '@types/node': 20.11.20 + '@types/node': 20.11.22 '@types/wrap-ansi': 3.0.0 ansi-escapes: 4.3.2 chalk: 4.1.2 @@ -7524,7 +7623,7 @@ packages: dependencies: '@jest/fake-timers': 29.6.2 '@jest/types': 29.6.1 - '@types/node': 20.11.20 + '@types/node': 20.11.22 jest-mock: 29.6.2 dev: true @@ -7551,7 +7650,7 @@ packages: dependencies: '@jest/types': 29.6.1 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 20.11.20 + '@types/node': 20.11.22 jest-message-util: 29.6.2 jest-mock: 29.6.2 jest-util: 29.6.2 @@ -7764,6 +7863,30 @@ packages: resolution: {integrity: sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==} dev: true + /@kubernetes/client-node/0.20.0: + resolution: {integrity: sha512-xxlv5GLX4FVR/dDKEsmi4SPeuB49aRc35stndyxcC73XnUEEwF39vXbROpHOirmDse8WE9vxOjABnSVS+jb7EA==} + dependencies: + '@types/js-yaml': 4.0.9 + '@types/node': 20.6.0 + '@types/request': 2.48.12 + '@types/ws': 8.5.10 + byline: 5.0.0 + isomorphic-ws: 5.0.0_ws@8.16.0 + js-yaml: 4.1.0 + jsonpath-plus: 7.2.0 + request: 2.88.2 + rfc4648: 1.5.3 + stream-buffers: 3.0.2 + tar: 6.1.13 + tslib: 2.6.2 + ws: 8.16.0 + optionalDependencies: + openid-client: 5.6.4 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: false + /@kwsites/file-exists/1.1.1: resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} dependencies: @@ -14277,7 +14400,7 @@ packages: cacache: 15.3.0 chalk: 4.1.2 chokidar: 3.5.3 - dotenv: 16.3.1 + dotenv: 16.4.4 esbuild: 0.17.6 esbuild-plugins-node-modules-polyfill: 1.3.0_esbuild@0.17.6 execa: 5.1.1 @@ -14361,7 +14484,7 @@ packages: cacache: 17.1.4 chalk: 4.1.2 chokidar: 3.5.3 - dotenv: 16.3.1 + dotenv: 16.4.4 esbuild: 0.17.6 esbuild-plugins-node-modules-polyfill: 1.6.1_esbuild@0.17.6 execa: 5.1.1 @@ -15400,6 +15523,10 @@ packages: tslib: 2.6.2 dev: false + /@socket.io/component-emitter/3.1.0: + resolution: {integrity: sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==} + dev: false + /@storybook/addon-actions/7.0.9_biqbaboplfbrettd7655fr4n2y: resolution: {integrity: sha512-xEO/Ekj6m6p1l4CXGh75BNBD/sqF7TqnUA/f2T3UCtLf9xVMoGEczxNi28iZq2RHoIAaV2aKArqjM+qTwoLjZw==} peerDependencies: @@ -16225,7 +16352,7 @@ packages: ts-dedent: 2.2.0 util-deprecate: 1.0.2 watchpack: 2.4.0 - ws: 8.12.0 + ws: 8.16.0 transitivePeerDependencies: - bufferutil - encoding @@ -17287,7 +17414,7 @@ packages: chalk: 5.3.0 cronstrue: 2.21.0 debug: 4.3.4 - evt: 2.4.13 + evt: 2.5.7 get-caller-file: 2.0.5 git-remote-origin-url: 4.0.0 git-repo-info: 2.1.1 @@ -17296,7 +17423,7 @@ packages: terminal-link: 3.0.0 ulid: 2.3.0 uuid: 9.0.0 - ws: 8.12.0 + ws: 8.16.0 zod: 3.21.4 zod-error: 1.5.0 transitivePeerDependencies: @@ -17343,7 +17470,7 @@ packages: /@types/accepts/1.3.5: resolution: {integrity: sha512-jOdnI/3qTpHABjM5cx1Hc0sKsPoYCp+DP/GJRGtDlPd7fiV9oXGGIcjW/ZOxLIvjGz8MA+uMZI9metHlgqbgwQ==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/acorn/4.0.6: @@ -17420,7 +17547,7 @@ packages: /@types/bunyan/1.8.7: resolution: {integrity: sha512-jaNt6xX5poSmXuDAkQrSqx2zkR66OrdRDuVnU8ldvn3k/Ci/7Sf5nooKspQWimDnw337Bzt/yirqSThTjvrHkg==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/bunyan/1.8.9: @@ -17437,6 +17564,10 @@ packages: '@types/node': 20.6.0 '@types/responselike': 1.0.0 + /@types/caseless/0.12.5: + resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==} + dev: false + /@types/chai-subset/1.3.3: resolution: {integrity: sha512-frBecisrNGz+F4T6bcc+NLeolfiojh5FxW2klu669+8BARtyQv2C/GkNW6FUodVe4BroGMP/wER/YDGc7rEllw==} dependencies: @@ -17490,9 +17621,14 @@ packages: '@types/connect': 3.4.36 '@types/express': 4.17.18 '@types/keygrip': 1.0.2 - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false + /@types/cors/2.8.17: + resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} + dependencies: + '@types/node': 20.6.0 + /@types/d3-array/3.0.8: resolution: {integrity: sha512-2xAVyAUgaXHX9fubjcCbGAUOqYfRJN1em1EKR2HfzWBpObZhwfnZKvofTN4TplMqJdFQao61I+NVSai/vnBvDQ==} dev: false @@ -17654,19 +17790,19 @@ packages: resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==} dependencies: '@types/minimatch': 5.1.2 - '@types/node': 20.11.20 + '@types/node': 20.11.22 /@types/glob/8.1.0: resolution: {integrity: sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==} dependencies: '@types/minimatch': 5.1.2 - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: true /@types/graceful-fs/4.1.6: resolution: {integrity: sha512-Sig0SNORX9fdW+bQuTEovKj3uHcUL6LQKbCrrqb1X7J6/ReAbhCXRAhc+SMejhLELFj2QcyuxmUooZ4bt5ReSw==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: true /@types/gradient-string/1.1.2: @@ -17701,7 +17837,7 @@ packages: '@types/hapi__catbox': 10.2.4 '@types/hapi__mimos': 4.1.4 '@types/hapi__shot': 4.1.2 - '@types/node': 20.11.20 + '@types/node': 20.11.22 joi: 17.7.0 dev: false @@ -17714,7 +17850,7 @@ packages: /@types/hapi__shot/4.1.2: resolution: {integrity: sha512-8wWgLVP1TeGqgzZtCdt+F+k15DWQvLG1Yv6ZzPfb3D5WIo5/S+GGKtJBVo2uNEcqabP5Ifc71QnJTDnTmw1axA==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/hast/2.3.4: @@ -17751,7 +17887,7 @@ packages: /@types/ioredis/4.28.10: resolution: {integrity: sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/is-ci/3.0.0: @@ -17797,6 +17933,10 @@ packages: resolution: {integrity: sha512-qC4bCqYGy1y/NP7dDVr7KJarn+PbX1nSpwA7JXdu0HxT3QYjO8MJ+cntENtHFVy2dRAyBV23OZ6MxsW1AM1L8g==} dev: true + /@types/js-yaml/4.0.9: + resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} + dev: false + /@types/json-query/2.2.3: resolution: {integrity: sha512-ygE4p8lyKzTBo9LF2K/u6MHnxPxbHY6wGvwM7TdAKhbP3SvEf+Y9aeVWedDiP8SMIPowTl9R/6awQYjiUTHz2g==} dev: true @@ -17818,7 +17958,7 @@ packages: /@types/jsonwebtoken/9.0.1: resolution: {integrity: sha512-c5ltxazpWabia/4UzhIoaDcIza4KViOQhdbjRlfcIGVnsE3c3brkz9Z+F/EeJIECOQP7W7US2hNE930cWWkPiw==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/keygrip/1.0.2: @@ -17828,7 +17968,7 @@ packages: /@types/keyv/3.1.4: resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 /@types/koa-compose/3.2.5: resolution: {integrity: sha512-B8nG/OoE1ORZqCkBVsup/AKcvjdgoHnfi4pZMn5UwAPCbhk/96xyv284eBYW8JlQbQ7zDmnpFr68I/40mFoIBQ==} @@ -17846,7 +17986,7 @@ packages: '@types/http-errors': 2.0.1 '@types/keygrip': 1.0.2 '@types/koa-compose': 3.2.5 - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/koa/2.13.9: @@ -17941,7 +18081,7 @@ packages: /@types/morgan/1.9.4: resolution: {integrity: sha512-cXoc4k+6+YAllH3ZHmx4hf7La1dzUk6keTR4bF4b4Sc0mZxU/zK4wO7l+ZzezXm/jkYj/qC+uYGZrarZdIVvyQ==} dependencies: - '@types/node': 18.15.13 + '@types/node': 20.6.0 dev: true /@types/ms/0.7.31: @@ -17950,12 +18090,12 @@ packages: /@types/mute-stream/0.0.4: resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 /@types/mysql/2.15.19: resolution: {integrity: sha512-wSRg2QZv14CWcZXkgdvHbbV2ACufNy5EgI8mBBxnJIptchv7DBy/h53VMa2jDhyo0C9MO4iowE6z9vF8Ja1DkQ==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 dev: false /@types/mysql/2.15.22: @@ -17972,14 +18112,14 @@ packages: /@types/node-fetch/2.6.2: resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} dependencies: - '@types/node': 20.4.8 + '@types/node': 20.6.0 form-data: 3.0.1 dev: true /@types/node-fetch/2.6.4: resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.6.0 form-data: 3.0.1 /@types/node-forge/1.3.10: @@ -18011,9 +18151,15 @@ packages: /@types/node/18.17.1: resolution: {integrity: sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==} + dev: true - /@types/node/20.11.20: - resolution: {integrity: sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==} + /@types/node/18.19.20: + resolution: {integrity: sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==} + dependencies: + undici-types: 5.26.5 + + /@types/node/20.11.22: + resolution: {integrity: sha512-/G+IxWxma6V3E+pqK1tSl2Fo1kl41pK1yeCyDsgkF9WlVAme4j5ISYM2zR11bgLFJGLN5sVK40T4RJNuiZbEjA==} dependencies: undici-types: 5.26.5 @@ -18024,10 +18170,6 @@ packages: resolution: {integrity: sha512-rt40Nk13II9JwQBdeYqmbn2Q6IVTA5uPhvSO+JVqdXw/6/4glI6oR9ezty/A9Hg5u7JH4OmYmuQ+XvjKm0Datg==} dev: true - /@types/node/20.4.8: - resolution: {integrity: sha512-0mHckf6D2DiIAzh8fM8f3HQCvMKDpK94YQ0DSVkfWTG9BZleYIWudw9cJxX8oCk9bM+vAkDyujDV6dmKHbvQpg==} - dev: true - /@types/node/20.5.0: resolution: {integrity: sha512-Mgq7eCtoTjT89FqNoTzzXg2XvCi5VMhRV6+I2aYanc6kQCBImeNaAYRs/DyoVqk1YEUJK5gN9VO7HRIdz4Wo3Q==} dev: true @@ -18130,6 +18272,15 @@ packages: '@types/scheduler': 0.16.2 csstype: 3.1.1 + /@types/request/2.48.12: + resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==} + dependencies: + '@types/caseless': 0.12.5 + '@types/node': 20.6.0 + '@types/tough-cookie': 4.0.5 + form-data: 2.5.1 + dev: false + /@types/resolve/1.20.2: resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} @@ -18174,7 +18325,7 @@ packages: /@types/set-cookie-parser/2.4.2: resolution: {integrity: sha512-fBZgytwhYAUkj/jC/FAV4RQ5EerRup1YQsXQCh8rZfiHkc4UahC192oH0smGwsXol3cL3A5oETuAHeQHmhXM4w==} dependencies: - '@types/node': 20.6.0 + '@types/node': 20.11.22 dev: true /@types/shimmer/1.0.2: @@ -18212,7 +18363,7 @@ packages: /@types/tar/6.1.4: resolution: {integrity: sha512-Cp4oxpfIzWt7mr2pbhHT2OTXGMAL0szYCzuf8lRWyIMCgsx6/Hfc3ubztuhvzXHXgraTQxyOCmmg7TDGIMIJJQ==} dependencies: - '@types/node': 18.15.13 + '@types/node': 20.6.0 minipass: 4.0.0 dev: true @@ -18231,6 +18382,10 @@ packages: /@types/tinycolor2/1.4.3: resolution: {integrity: sha512-Kf1w9NE5HEgGxCRyIcRXR/ZYtDv0V8FVPtYHwLxl0O+maGX0erE77pQlD0gpP+/KByMZ87mOA79SjifhSB3PjQ==} + /@types/tough-cookie/4.0.5: + resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} + dev: false + /@types/trusted-types/2.0.3: resolution: {integrity: sha512-NfQ4gyz38SL8sDNrSixxU2Os1a5xcdFxipAFxYEuLUlvU2uDwS4NUpsImcf1//SlWItCVMMLiylsxbmNMToV/g==} dev: true @@ -18251,6 +18406,12 @@ packages: /@types/wrap-ansi/3.0.0: resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} + /@types/ws/8.5.10: + resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} + dependencies: + '@types/node': 20.6.0 + dev: false + /@types/ws/8.5.4: resolution: {integrity: sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==} dependencies: @@ -19377,6 +19538,14 @@ packages: clean-stack: 2.2.0 indent-string: 4.0.0 + /aggregate-error/4.0.1: + resolution: {integrity: sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==} + engines: {node: '>=12'} + dependencies: + clean-stack: 4.2.0 + indent-string: 5.0.0 + dev: true + /ahocorasick/1.0.2: resolution: {integrity: sha512-hCOfMzbFx5IDutmWLAt6MZwOUjIfSM9G9FyVxytmE4Rs/5YDPWQrD/+IR1w+FweD9H2oOZEnv36TmkjhNURBVA==} dev: true @@ -19724,6 +19893,11 @@ packages: engines: {node: '>=8'} dev: false + /arrify/3.0.0: + resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} + engines: {node: '>=12'} + dev: true + /as-table/1.0.55: resolution: {integrity: sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==} dependencies: @@ -19734,6 +19908,12 @@ packages: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} dev: true + /asn1/0.2.6: + resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} + dependencies: + safer-buffer: 2.1.2 + dev: false + /asn1js/3.0.5: resolution: {integrity: sha512-FVnvrKJwpt9LP2lAMl8qZswRNm3T4q9CON+bxldk2iwk3FFpuwhx2FfinyitizWHsVYyaY+y5JzDR0rCMV5yTQ==} engines: {node: '>=12.0.0'} @@ -19743,6 +19923,11 @@ packages: tslib: 2.6.2 dev: false + /assert-plus/1.0.0: + resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} + engines: {node: '>=0.8'} + dev: false + /assert/2.0.0: resolution: {integrity: sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==} dependencies: @@ -19940,6 +20125,14 @@ packages: - supports-color dev: true + /aws-sign2/0.7.0: + resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} + dev: false + + /aws4/1.12.0: + resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} + dev: false + /axe-core/4.6.2: resolution: {integrity: sha512-b1WlTV8+XKLj9gZy2DZXgQiyDp9xkkoe2a6U6UbYccScq2wgH/YwCeI2/Jq2mgo0HzQxqJOjWZBLeA/mqsk5Mg==} engines: {node: '>=4'} @@ -20214,6 +20407,10 @@ packages: /base64-js/1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + /base64id/2.0.0: + resolution: {integrity: sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==} + engines: {node: ^4.5.0 || >= 5.9} + /basic-auth/2.0.1: resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} engines: {node: '>= 0.8'} @@ -20224,6 +20421,12 @@ packages: resolution: {integrity: sha512-QHX8HLlncOLpy54mh+k/sWIFd0ThmRqwe9ZjELybGZK+tZ8rUb9VO0saKJUROTbE+KhzDUT7xziGpGrW8Kmd+g==} engines: {node: '>=10.0.0'} + /bcrypt-pbkdf/1.0.2: + resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} + dependencies: + tweetnacl: 0.14.5 + dev: false + /before-after-hook/2.2.3: resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==} dev: false @@ -20272,6 +20475,10 @@ packages: dev: true optional: true + /bintrees/1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + dev: false + /bl/4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} dependencies: @@ -20598,6 +20805,11 @@ packages: dependencies: streamsearch: 1.1.0 + /byline/5.0.0: + resolution: {integrity: sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==} + engines: {node: '>=0.10.0'} + dev: false + /bytes/3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} @@ -20813,6 +21025,10 @@ packages: engines: {node: '>=4'} dev: true + /caseless/0.12.0: + resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + dev: false + /ccount/2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -20971,6 +21187,13 @@ packages: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} + /clean-stack/4.2.0: + resolution: {integrity: sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==} + engines: {node: '>=12'} + dependencies: + escape-string-regexp: 5.0.0 + dev: true + /cli-boxes/2.2.1: resolution: {integrity: sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==} engines: {node: '>=6'} @@ -21401,6 +21624,10 @@ packages: requiresBuild: true dev: true + /core-util-is/1.0.2: + resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} + dev: false + /core-util-is/1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} @@ -21431,6 +21658,15 @@ packages: path-type: 4.0.0 dev: true + /cp-file/10.0.0: + resolution: {integrity: sha512-vy2Vi1r2epK5WqxOLnskeKeZkdZvTKfFZQCplE3XWsP+SUJyd5XAUFC9lFgTjjXJF2GMne/UML14iEmkAaDfFg==} + engines: {node: '>=14.16'} + dependencies: + graceful-fs: 4.2.10 + nested-error-stacks: 2.1.1 + p-event: 5.0.1 + dev: true + /cp-file/7.0.0: resolution: {integrity: sha512-0Cbj7gyvFVApzpK/uhCtQ/9kE9UnYpxMzaq5nQQC/Dh4iaj5fxp7iEFIullrYwzj8nf0qnsI1Qsx34hAeAebvw==} engines: {node: '>=8'} @@ -21441,6 +21677,29 @@ packages: p-event: 4.2.0 dev: false + /cpy-cli/5.0.0: + resolution: {integrity: sha512-fb+DZYbL9KHc0BC4NYqGRrDIJZPXUmjjtqdw4XRRg8iV8dIfghUX/WiL+q4/B/KFTy3sK6jsbUhBaz0/Hxg7IQ==} + engines: {node: '>=16'} + hasBin: true + dependencies: + cpy: 10.1.0 + meow: 12.1.1 + dev: true + + /cpy/10.1.0: + resolution: {integrity: sha512-VC2Gs20JcTyeQob6UViBLnyP0bYHkBh6EiKzot9vi2DmeGlFT9Wd7VG3NBrkNx/jYvFBeyDOMMHdHQhbtKLgHQ==} + engines: {node: '>=16'} + dependencies: + arrify: 3.0.0 + cp-file: 10.0.0 + globby: 13.2.2 + junk: 4.0.1 + micromatch: 4.0.5 + nested-error-stacks: 2.1.1 + p-filter: 3.0.0 + p-map: 6.0.0 + dev: true + /cpy/8.1.2: resolution: {integrity: sha512-dmC4mUesv0OYH2kNFEidtf/skUwv4zePmGeepjyyJ0qTo5+8KhA1o99oIAwVVLzQMAeDJml74d6wPPKb6EZUTg==} engines: {node: '>=8'} @@ -21732,6 +21991,13 @@ packages: /damerau-levenshtein/1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + /dashdash/1.14.1: + resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} + engines: {node: '>=0.10'} + dependencies: + assert-plus: 1.0.0 + dev: false + /data-uri-to-buffer/2.0.2: resolution: {integrity: sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==} dev: true @@ -22349,6 +22615,13 @@ packages: /eastasianwidth/0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + /ecc-jsbn/0.1.2: + resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} + dependencies: + jsbn: 0.1.1 + safer-buffer: 2.1.2 + dev: false + /ecdsa-sig-formatter/1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} dependencies: @@ -22425,6 +22698,57 @@ packages: objectorarray: 1.0.5 dev: true + /engine.io-client/6.5.3: + resolution: {integrity: sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==} + dependencies: + '@socket.io/component-emitter': 3.1.0 + debug: 4.3.4 + engine.io-parser: 5.2.2 + ws: 8.11.0 + xmlhttprequest-ssl: 2.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + dev: false + + /engine.io-client/6.5.3_supports-color@9.4.0: + resolution: {integrity: sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==} + dependencies: + '@socket.io/component-emitter': 3.1.0 + debug: 4.3.4_supports-color@9.4.0 + engine.io-parser: 5.2.2 + ws: 8.11.0 + xmlhttprequest-ssl: 2.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + dev: false + + /engine.io-parser/5.2.2: + resolution: {integrity: sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw==} + engines: {node: '>=10.0.0'} + + /engine.io/6.5.4: + resolution: {integrity: sha512-KdVSDKhVKyOi+r5uEabrDLZw2qXStVvCsEB/LN3mw4WFi6Gx50jTyuxYVCwAAC0U46FdnzP/ScKRBTXb/NiEOg==} + engines: {node: '>=10.2.0'} + dependencies: + '@types/cookie': 0.4.1 + '@types/cors': 2.8.17 + '@types/node': 20.6.0 + accepts: 1.3.8 + base64id: 2.0.0 + cookie: 0.4.2 + cors: 2.8.5 + debug: 4.3.4 + engine.io-parser: 5.2.2 + ws: 8.11.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + /enhanced-resolve/5.13.0: resolution: {integrity: sha512-eyV8f0y1+bzyfh8xAwW/WTSZpLbjhqc4ne9eGSH4Zo2ejdyiNG9pU6mf9DG8a7+Auk6MFTlNOT4Y2y/9k8GKVg==} engines: {node: '>=10.13.0'} @@ -23242,8 +23566,8 @@ packages: eslint: 8.42.0 eslint-module-utils: 2.7.4_pxhjym6joapyaysotvqp5q6kau eslint-plugin-import: 2.27.5_dclhgbuu3rymxzhsuuyv72p2ym - get-tsconfig: 4.5.0 - globby: 13.1.3 + get-tsconfig: 4.7.2 + globby: 13.2.2 is-core-module: 2.13.0 is-glob: 4.0.3 synckit: 0.8.5 @@ -23266,8 +23590,8 @@ packages: eslint: 8.45.0 eslint-module-utils: 2.7.4_mcdyrqs6peaairrn7md524c74a eslint-plugin-import: 2.27.5_xpt3ce3kmhzhmuk3dcuwe6u2pe - get-tsconfig: 4.5.0 - globby: 13.1.3 + get-tsconfig: 4.7.2 + globby: 13.2.2 is-core-module: 2.13.0 is-glob: 4.0.3 synckit: 0.8.5 @@ -23289,8 +23613,8 @@ packages: eslint: 8.31.0 eslint-module-utils: 2.7.4_hhwxwo5vg2mzr5fq3eei7kqaye eslint-plugin-import: 2.27.5_4nmwrqeucdooykidvg7oplsu44 - get-tsconfig: 4.5.0 - globby: 13.1.3 + get-tsconfig: 4.7.2 + globby: 13.2.2 is-core-module: 2.13.0 is-glob: 4.0.3 synckit: 0.8.5 @@ -24277,6 +24601,14 @@ packages: tsafe: 1.4.1 dev: false + /evt/2.5.7: + resolution: {integrity: sha512-dr7Wd16ry5F8WNU1xXLKpFpO3HsoAGg8zC48e08vDdzMzGWCP9/QFGt1PQptEEDh8SwYP3EL8M+d/Gb0kgUp6g==} + dependencies: + minimal-polyfills: 2.2.3 + run-exclusive: 2.2.19 + tsafe: 1.6.6 + dev: false + /execa/4.1.0: resolution: {integrity: sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==} engines: {node: '>=10'} @@ -24508,6 +24840,11 @@ packages: - supports-color dev: false + /extsprintf/1.3.0: + resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} + engines: {'0': node >=0.6.0} + dev: false + /fast-content-type-parse/1.1.0: resolution: {integrity: sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==} dev: true @@ -24922,6 +25259,10 @@ packages: cross-spawn: 7.0.3 signal-exit: 4.1.0 + /forever-agent/0.6.1: + resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} + dev: false + /fork-ts-checker-webpack-plugin/7.3.0_dtthwp2bsqb7yvb7hoeealjg4i: resolution: {integrity: sha512-IN+XTzusCjR5VgntYFgxbxVx3WraPRnKehBFrf00cMSrtUuW9MsG9dhL6MWpY6MkjC3wVwoujfCDgZZCQwbswA==} engines: {node: '>=12.13.0', yarn: '>=1.0.0'} @@ -24981,6 +25322,15 @@ packages: engines: {node: '>= 14.17'} dev: false + /form-data/2.3.3: + resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} + engines: {node: '>= 0.12'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + /form-data/2.5.1: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} @@ -25338,8 +25688,10 @@ packages: call-bind: 1.0.2 get-intrinsic: 1.1.3 - /get-tsconfig/4.5.0: - resolution: {integrity: sha512-MjhiaIWCJ1sAU4pIQ5i5OfOuHHxVo1oYeNsWTON7jxYkod8pHocXeh+SSbmu5OZZZK73B6cbJ2XADzXehLyovQ==} + /get-tsconfig/4.7.2: + resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} + dependencies: + resolve-pkg-maps: 1.0.0 /get-uri/6.0.1: resolution: {integrity: sha512-7ZqONUVqaabogsYNWlYj0t3YZaL6dhuEueZXGF+/YVmf6dHmaFg8/6psJKqhx9QykIDKzpGcy2cn4oV4YC7V/Q==} @@ -25369,6 +25721,12 @@ packages: engines: {node: '>=0.10.0'} dev: false + /getpass/0.1.7: + resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} + dependencies: + assert-plus: 1.0.0 + dev: false + /giget/1.1.2: resolution: {integrity: sha512-HsLoS07HiQ5oqvObOI+Qb2tyZH4Gj5nYGfF9qQcZNrPw+uEFhdXtgJr01aO2pWadGHucajYDLxxbtQkm97ON2A==} hasBin: true @@ -25587,8 +25945,8 @@ packages: merge2: 1.4.1 slash: 3.0.0 - /globby/13.1.3: - resolution: {integrity: sha512-8krCNHXvlCgHDpegPzleMq07yMYTO2sXKASmZmquEYWEmCx6J5UTRbp5RwMJkTJGtcQ44YpiUYUiN0b9mzy8Bw==} + /globby/13.2.2: + resolution: {integrity: sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: dir-glob: 3.0.1 @@ -25755,6 +26113,20 @@ packages: uglify-js: 3.17.4 dev: true + /har-schema/2.0.0: + resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} + engines: {node: '>=4'} + dev: false + + /har-validator/5.1.5: + resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} + engines: {node: '>=6'} + deprecated: this library is no longer supported + dependencies: + ajv: 6.12.6 + har-schema: 2.0.0 + dev: false + /hard-rejection/2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -26140,6 +26512,15 @@ packages: - supports-color dev: false + /http-signature/1.2.0: + resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} + engines: {node: '>=0.8', npm: '>=1.3.7'} + dependencies: + assert-plus: 1.0.0 + jsprim: 1.4.2 + sshpk: 1.18.0 + dev: false + /http2-wrapper/1.0.3: resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==} engines: {node: '>=10.19.0'} @@ -26350,7 +26731,6 @@ packages: /indent-string/5.0.0: resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} engines: {node: '>=12'} - dev: false /infer-owner/1.0.4: resolution: {integrity: sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==} @@ -26416,7 +26796,7 @@ packages: type-fest: 0.12.0 widest-line: 4.0.1 wrap-ansi: 8.1.0 - ws: 8.12.0 + ws: 8.16.0 yoga-wasm-web: 0.3.3 transitivePeerDependencies: - bufferutil @@ -27101,6 +27481,18 @@ packages: transitivePeerDependencies: - encoding + /isomorphic-ws/5.0.0_ws@8.16.0: + resolution: {integrity: sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==} + peerDependencies: + ws: '*' + dependencies: + ws: 8.16.0 + dev: false + + /isstream/0.1.2: + resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} + dev: false + /istanbul-lib-coverage/3.2.0: resolution: {integrity: sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==} engines: {node: '>=8'} @@ -27201,7 +27593,7 @@ packages: '@jest/expect': 29.6.2 '@jest/test-result': 29.6.2 '@jest/types': 29.6.1 - '@types/node': 20.11.20 + '@types/node': 20.11.22 chalk: 4.1.2 co: 4.6.0 dedent: 1.5.1 @@ -27222,35 +27614,6 @@ packages: - supports-color dev: true - /jest-cli/29.6.2_@types+node@16.18.11: - resolution: {integrity: sha512-TT6O247v6dCEX2UGHGyflMpxhnrL0DNqP2fRTKYm3nJJpCTfXX3GCMQPGFjXDoj0i5/Blp3jriKXFgdfmbYB6Q==} - engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - hasBin: true - peerDependencies: - node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 - peerDependenciesMeta: - node-notifier: - optional: true - dependencies: - '@jest/core': 29.6.2 - '@jest/test-result': 29.6.2 - '@jest/types': 29.6.1 - chalk: 4.1.2 - exit: 0.1.2 - graceful-fs: 4.2.10 - import-local: 3.1.0 - jest-config: 29.6.2_@types+node@16.18.11 - jest-util: 29.6.2 - jest-validate: 29.6.2 - prompts: 2.4.2 - yargs: 17.7.2 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - dev: true - /jest-cli/29.6.2_@types+node@18.15.13: resolution: {integrity: sha512-TT6O247v6dCEX2UGHGyflMpxhnrL0DNqP2fRTKYm3nJJpCTfXX3GCMQPGFjXDoj0i5/Blp3jriKXFgdfmbYB6Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -27309,46 +27672,6 @@ packages: - ts-node dev: true - /jest-config/29.6.2_@types+node@16.18.11: - resolution: {integrity: sha512-VxwFOC8gkiJbuodG9CPtMRjBUNZEHxwfQXmIudSTzFWxaci3Qub1ddTRbFNQlD/zUeaifLndh/eDccFX4wCMQw==} - engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - peerDependencies: - '@types/node': '*' - ts-node: '>=9.0.0' - peerDependenciesMeta: - '@types/node': - optional: true - ts-node: - optional: true - dependencies: - '@babel/core': 7.22.17 - '@jest/test-sequencer': 29.6.2 - '@jest/types': 29.6.1 - '@types/node': 16.18.11 - babel-jest: 29.6.2_@babel+core@7.22.17 - chalk: 4.1.2 - ci-info: 3.8.0 - deepmerge: 4.3.1 - glob: 7.2.3 - graceful-fs: 4.2.10 - jest-circus: 29.6.2 - jest-environment-node: 29.6.2 - jest-get-type: 29.4.3 - jest-regex-util: 29.4.3 - jest-resolve: 29.6.2 - jest-runner: 29.6.2 - jest-util: 29.6.2 - jest-validate: 29.6.2 - micromatch: 4.0.5 - parse-json: 5.2.0 - pretty-format: 29.6.2 - slash: 3.0.0 - strip-json-comments: 3.1.1 - transitivePeerDependencies: - - babel-plugin-macros - - supports-color - dev: true - /jest-config/29.6.2_@types+node@18.15.13: resolution: {integrity: sha512-VxwFOC8gkiJbuodG9CPtMRjBUNZEHxwfQXmIudSTzFWxaci3Qub1ddTRbFNQlD/zUeaifLndh/eDccFX4wCMQw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -27504,7 +27827,7 @@ packages: '@jest/environment': 29.6.2 '@jest/fake-timers': 29.6.2 '@jest/types': 29.6.1 - '@types/node': 20.11.20 + '@types/node': 20.11.22 jest-mock: 29.6.2 jest-util: 29.6.2 dev: true @@ -27579,7 +27902,7 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@jest/types': 29.6.1 - '@types/node': 20.11.20 + '@types/node': 20.11.22 jest-util: 29.6.2 dev: true @@ -27754,7 +28077,7 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 merge-stream: 2.0.0 supports-color: 8.1.1 dev: true @@ -27763,33 +28086,12 @@ packages: resolution: {integrity: sha512-l3ccBOabTdkng8I/ORCkADz4eSMKejTYv1vB/Z83UiubqhC1oQ5Li6dWCyqOIvSifGjUBxuvxvlm6KGK2DtuAQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.11.22 jest-util: 29.6.2 merge-stream: 2.0.0 supports-color: 8.1.1 dev: true - /jest/29.6.2_@types+node@16.18.11: - resolution: {integrity: sha512-8eQg2mqFbaP7CwfsTpCxQ+sHzw1WuNWL5UUvjnWP4hx2riGz9fPSzYOaU5q8/GqWn1TfgZIVTqYJygbGbWAANg==} - engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - hasBin: true - peerDependencies: - node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 - peerDependenciesMeta: - node-notifier: - optional: true - dependencies: - '@jest/core': 29.6.2 - '@jest/types': 29.6.1 - import-local: 3.1.0 - jest-cli: 29.6.2_@types+node@16.18.11 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - dev: true - /jest/29.6.2_@types+node@18.15.13: resolution: {integrity: sha512-8eQg2mqFbaP7CwfsTpCxQ+sHzw1WuNWL5UUvjnWP4hx2riGz9fPSzYOaU5q8/GqWn1TfgZIVTqYJygbGbWAANg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -27905,6 +28207,10 @@ packages: dependencies: argparse: 2.0.1 + /jsbn/0.1.1: + resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} + dev: false + /jscodeshift/0.14.0_@babel+preset-env@7.21.5: resolution: {integrity: sha512-7eCC1knD7bLUPuSCwXsMZUH51O8jIcoVyKtI6P0XM0IVzlGjckPy3FIwQlorzbN0Sg79oK+RlohN32Mqf/lrYA==} hasBin: true @@ -27987,9 +28293,17 @@ packages: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} dev: true + /json-schema/0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + dev: false + /json-stable-stringify-without-jsonify/1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + /json-stringify-safe/5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + dev: false + /json5/1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -28026,6 +28340,11 @@ packages: engines: {'0': node >= 0.2.0} dev: false + /jsonpath-plus/7.2.0: + resolution: {integrity: sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==} + engines: {node: '>=12.0.0'} + dev: false + /jsonpointer/5.0.1: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} @@ -28041,6 +28360,16 @@ packages: semver: 7.5.4 dev: false + /jsprim/1.4.2: + resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} + engines: {node: '>=0.6.0'} + dependencies: + assert-plus: 1.0.0 + extsprintf: 1.3.0 + json-schema: 0.4.0 + verror: 1.10.0 + dev: false + /jsx-ast-utils/3.3.3: resolution: {integrity: sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==} engines: {node: '>=4.0'} @@ -28053,6 +28382,11 @@ packages: engines: {node: '>=8'} dev: false + /junk/4.0.1: + resolution: {integrity: sha512-Qush0uP+G8ZScpGMZvHUiRfI0YBWuB3gVBYlI0v0vvOJt5FLicco+IkP0a50LqTTQhmts/m6tP5SWE+USyIvcQ==} + engines: {node: '>=12.20'} + dev: true + /jwa/1.4.1: resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} dependencies: @@ -28958,6 +29292,11 @@ packages: resolution: {integrity: sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==} engines: {node: '>= 0.10.0'} + /meow/12.1.1: + resolution: {integrity: sha512-BhXM0Au22RwUneMPwSCnyhTOizdWoIEPU9sp0Aqa1PnDMR5Wv2FGXYDjuzJEIX+Eo2Rb8xuYe5jrnm5QowQFkw==} + engines: {node: '>=16.10'} + dev: true + /meow/6.1.1: resolution: {integrity: sha512-3YffViIt2QWgTy6Pale5QpopX/IvU3LPL03jOTqp6pGj3VjesdO/U8CuHMKpnQr4shCNCM5fd5XFFvIIl6JBHg==} engines: {node: '>=8'} @@ -29402,7 +29741,7 @@ packages: stoppable: 1.1.0 undici: 5.25.4 workerd: 1.20231030.0 - ws: 8.12.0 + ws: 8.16.0 youch: 3.3.3 zod: 3.22.4 transitivePeerDependencies: @@ -29425,7 +29764,7 @@ packages: stoppable: 1.1.0 undici: 5.25.4 workerd: 1.20231030.0 - ws: 8.12.0 + ws: 8.16.0 youch: 3.3.3 zod: 3.22.4 transitivePeerDependencies: @@ -29448,7 +29787,7 @@ packages: stoppable: 1.1.0 undici: 5.25.4 workerd: 1.20231030.0 - ws: 8.12.0 + ws: 8.16.0 youch: 3.3.3 zod: 3.22.4 transitivePeerDependencies: @@ -29461,6 +29800,10 @@ packages: resolution: {integrity: sha512-eEOUq/LH/DbLWihrxUP050Wi7H/N/I2dQT98Ep6SqOpmIbk4sXOI4wqalve66QoZa+6oljbZWU6I6T4dehQGmw==} dev: false + /minimal-polyfills/2.2.3: + resolution: {integrity: sha512-oxdmJ9cL+xV72h0xYxp4tP2d5/fTBpP45H8DIOn9pASuF8a3IYTf+25fMGDYGiWW+MFsuog6KD6nfmhZJQ+uUw==} + dev: false + /minimatch/3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: @@ -29901,7 +30244,6 @@ packages: /nested-error-stacks/2.1.1: resolution: {integrity: sha512-9iN1ka/9zmX1ZvLV9ewJYEk9h7RyRRtqdK0woXcqohu8EWIerfPUjYJPg0ULy0UqP7cslmdGc8xKDJcojlKiaw==} - dev: false /netmask/2.0.2: resolution: {integrity: sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==} @@ -30215,6 +30557,23 @@ packages: /node-releases/2.0.13: resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==} + /nodemon/2.0.22: + resolution: {integrity: sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ==} + engines: {node: '>=8.10.0'} + hasBin: true + dependencies: + chokidar: 3.5.3 + debug: 3.2.7_supports-color@5.5.0 + ignore-by-default: 1.0.1 + minimatch: 3.1.2 + pstree.remy: 1.1.8 + semver: 5.7.1 + simple-update-notifier: 1.1.0 + supports-color: 5.5.0 + touch: 3.1.0 + undefsafe: 2.0.5 + dev: true + /nodemon/3.0.1: resolution: {integrity: sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==} engines: {node: '>=10'} @@ -30467,6 +30826,14 @@ packages: dependencies: path-key: 4.0.0 + /npm-watch/0.11.0: + resolution: {integrity: sha512-wAOd0moNX2kSA2FNvt8+7ORwYaJpQ1ZoWjUYdb1bBCxq4nkWuU0IiJa9VpVxrj5Ks+FGXQd62OC/Bjk0aSr+dg==} + hasBin: true + dependencies: + nodemon: 2.0.22 + through2: 4.0.2 + dev: true + /npmlog/5.0.1: resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==} dependencies: @@ -30492,6 +30859,10 @@ packages: boolbase: 1.0.0 dev: true + /oauth-sign/0.9.0: + resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} + dev: false + /object-assign/4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -30505,6 +30876,12 @@ packages: kind-of: 3.2.2 dev: false + /object-hash/2.2.0: + resolution: {integrity: sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==} + engines: {node: '>= 6'} + dev: false + optional: true + /object-hash/3.0.0: resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} engines: {node: '>= 6'} @@ -30638,6 +31015,12 @@ packages: resolution: {integrity: sha512-zuHHiGTYTA1sYJ/wZN+t5HKZaH23i4yI1HMwbuXm24Nid7Dv0KcuRlKoNKS9UNfAVSBlnGLcuQrnOKWOZoEGaw==} dev: false + /oidc-token-hash/5.0.3: + resolution: {integrity: sha512-IF4PcGgzAr6XXSff26Sk/+P4KZFJVuHAJZj3wgO3vX2bMdNVp/QXTP3P7CEm9V1IdG8lDLY3HhiqpsE/nOwpPw==} + engines: {node: ^10.13.0 || >=12.0.0} + dev: false + optional: true + /on-exit-leak-free/2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} @@ -30706,7 +31089,7 @@ packages: resolution: {integrity: sha512-VbAYerNZFfIIeESS+OL9vgDkK8Mnri55n+jN0UN/HZeuM0ghGh6nDN6UGRZxslNgyJ7XmY/Ca9DO4YYyvrszGA==} hasBin: true dependencies: - '@types/node': 18.17.1 + '@types/node': 18.19.20 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -30723,7 +31106,7 @@ packages: resolution: {integrity: sha512-JM8fhcpmpGN0vrUwGquYIzdcEQHtFuom6sRCbbCM6CfzZXNuRk33G7KfeRAIfnaCxSpzrP5iHtwJzIm6biUZ2Q==} hasBin: true dependencies: - '@types/node': 18.17.1 + '@types/node': 18.19.20 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -30740,7 +31123,7 @@ packages: resolution: {integrity: sha512-9cA5i6KpEAeLV8nZrVgOXqRmUWUIu4FV2oRKvSIwrA7Z5aMd4sXVOaqWAuRDhFFewirxhys8D6R5B2Drj1pKIQ==} hasBin: true dependencies: - '@types/node': 18.17.1 + '@types/node': 18.19.20 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -30756,6 +31139,17 @@ packages: resolution: {integrity: sha512-Faj29Kzh7oCbt1bz6vAGNKtRJlV/GolOQTx87eYUnfCK7eVXdN9jQVojroc7tcJ5OQgyhbeOqD7LS/8UtGBnMQ==} dev: false + /openid-client/5.6.4: + resolution: {integrity: sha512-T1h3B10BRPKfcObdBklX639tVz+xh34O7GjofqrqiAQdm7eHsQ00ih18x6wuJ/E6FxdtS2u3FmUGPDeEcMwzNA==} + requiresBuild: true + dependencies: + jose: 4.15.4 + lru-cache: 6.0.0 + object-hash: 2.2.0 + oidc-token-hash: 5.0.3 + dev: false + optional: true + /opentracing/0.14.7: resolution: {integrity: sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q==} engines: {node: '>=0.10'} @@ -30905,6 +31299,13 @@ packages: p-timeout: 3.2.0 dev: false + /p-event/5.0.1: + resolution: {integrity: sha512-dd589iCQ7m1L0bmC5NLlVYfy3TbBEsMUfWx9PyAgPeIcFZ/E2yaTZ4Rz4MiBmmJShviiftHVXOqfnfzJ6kyMrQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-timeout: 5.1.0 + dev: true + /p-filter/2.1.0: resolution: {integrity: sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw==} engines: {node: '>=8'} @@ -30912,6 +31313,13 @@ packages: p-map: 2.1.0 dev: false + /p-filter/3.0.0: + resolution: {integrity: sha512-QtoWLjXAW++uTX67HZQz1dbTpqBfiidsB6VtQUC9iR85S120+s0T5sO6s+B5MLzFcZkrEd/DGMmCjR+f2Qpxwg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-map: 5.5.0 + dev: true + /p-finally/1.0.0: resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} engines: {node: '>=4'} @@ -30979,6 +31387,18 @@ packages: dependencies: aggregate-error: 3.1.0 + /p-map/5.5.0: + resolution: {integrity: sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==} + engines: {node: '>=12'} + dependencies: + aggregate-error: 4.0.1 + dev: true + + /p-map/6.0.0: + resolution: {integrity: sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==} + engines: {node: '>=16'} + dev: true + /p-queue/6.6.2: resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} engines: {node: '>=8'} @@ -31015,6 +31435,11 @@ packages: p-finally: 1.0.0 dev: false + /p-timeout/5.1.0: + resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==} + engines: {node: '>=12'} + dev: true + /p-try/2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} @@ -31339,6 +31764,10 @@ packages: /pend/1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + /performance-now/2.1.0: + resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} + dev: false + /periscopic/3.1.0: resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} dependencies: @@ -32167,6 +32596,14 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + /prom-client/15.1.0: + resolution: {integrity: sha512-cCD7jLTqyPdjEPBo/Xk4Iu8jxjuZgZJ3e/oET3L+ZwOuap/7Cw3dH/TJSsZKs1TQLZ2IHpIlRAKw82ef06kmMw==} + engines: {node: ^16 || ^18 || >=20} + dependencies: + '@opentelemetry/api': 1.7.0 + tdigest: 0.1.2 + dev: false + /promise-inflight/1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: @@ -32273,6 +32710,10 @@ packages: /pseudomap/1.0.2: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} + /psl/1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + dev: false + /pstree.remy/1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true @@ -32354,6 +32795,11 @@ packages: dependencies: side-channel: 1.0.4 + /qs/6.5.3: + resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} + engines: {node: '>=0.6'} + dev: false + /queue-microtask/1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -33433,6 +33879,33 @@ packages: engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'} dev: false + /request/2.88.2: + resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} + engines: {node: '>= 6'} + deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 + dependencies: + aws-sign2: 0.7.0 + aws4: 1.12.0 + caseless: 0.12.0 + combined-stream: 1.0.8 + extend: 3.0.2 + forever-agent: 0.6.1 + form-data: 2.3.3 + har-validator: 5.1.5 + http-signature: 1.2.0 + is-typedarray: 1.0.0 + isstream: 0.1.2 + json-stringify-safe: 5.0.1 + mime-types: 2.1.35 + oauth-sign: 0.9.0 + performance-now: 2.1.0 + qs: 6.5.3 + safe-buffer: 5.2.1 + tough-cookie: 2.5.0 + tunnel-agent: 0.6.0 + uuid: 3.4.0 + dev: false + /require-directory/2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -33535,6 +34008,9 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} + /resolve-pkg-maps/1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + /resolve-url-loader/5.0.0: resolution: {integrity: sha512-uZtduh8/8srhBoMx//5bwqjQ+rfYOUq8zC9NrMUGtjBiGTtFJM42s58/36+hTqeqINcnYe08Nj3LkK9lW4N8Xg==} engines: {node: '>=12'} @@ -33674,6 +34150,10 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + /rfc4648/1.5.3: + resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==} + dev: false + /rfdc/1.3.0: resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} dev: true @@ -33797,7 +34277,13 @@ packages: /run-exclusive/2.2.18: resolution: {integrity: sha512-TXr1Gkl1iEAOCCpBTRm/2m0+1KGjORcWpZZ+VGGTe7dYX8E4y8/fMvrHk0zf+kclec2R//tpvdBxgG0bDgaJfw==} dependencies: - minimal-polyfills: 2.2.2 + minimal-polyfills: 2.2.3 + dev: false + + /run-exclusive/2.2.19: + resolution: {integrity: sha512-K3mdoAi7tjJ/qT7Flj90L7QyPozwUaAG+CVhkdDje4HLKXUYC3N/Jzkau3flHVDLQVhiHBtcimVodMjN9egYbA==} + dependencies: + minimal-polyfills: 2.2.3 dev: false /run-parallel/1.2.0: @@ -34401,6 +34887,80 @@ packages: - supports-color dev: false + /socket.io-adapter/2.5.2: + resolution: {integrity: sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA==} + dependencies: + ws: 8.11.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: false + + /socket.io-client/4.7.4: + resolution: {integrity: sha512-wh+OkeF0rAVCrABWQBaEjLfb7DVPotMbu0cgWgyR0v6eA4EoVnAwcIeIbcdTE3GT/H3kbdLl7OoH2+asoDRIIg==} + engines: {node: '>=10.0.0'} + dependencies: + '@socket.io/component-emitter': 3.1.0 + debug: 4.3.4 + engine.io-client: 6.5.3 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + dev: false + + /socket.io-client/4.7.4_supports-color@9.4.0: + resolution: {integrity: sha512-wh+OkeF0rAVCrABWQBaEjLfb7DVPotMbu0cgWgyR0v6eA4EoVnAwcIeIbcdTE3GT/H3kbdLl7OoH2+asoDRIIg==} + engines: {node: '>=10.0.0'} + dependencies: + '@socket.io/component-emitter': 3.1.0 + debug: 4.3.4_supports-color@9.4.0 + engine.io-client: 6.5.3_supports-color@9.4.0 + socket.io-parser: 4.2.4_supports-color@9.4.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + dev: false + + /socket.io-parser/4.2.4: + resolution: {integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==} + engines: {node: '>=10.0.0'} + dependencies: + '@socket.io/component-emitter': 3.1.0 + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + + /socket.io-parser/4.2.4_supports-color@9.4.0: + resolution: {integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==} + engines: {node: '>=10.0.0'} + dependencies: + '@socket.io/component-emitter': 3.1.0 + debug: 4.3.4_supports-color@9.4.0 + transitivePeerDependencies: + - supports-color + dev: false + + /socket.io/4.7.4: + resolution: {integrity: sha512-DcotgfP1Zg9iP/dH9zvAQcWrE0TtbMVwXmlV4T4mqsvY+gw+LqUGPfx2AoVyRk0FLME+GQhufDMyacFmw7ksqw==} + engines: {node: '>=10.2.0'} + dependencies: + accepts: 1.3.8 + base64id: 2.0.0 + cors: 2.8.5 + debug: 4.3.4 + engine.io: 6.5.4 + socket.io-adapter: 2.5.2 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + dev: false + /socks-proxy-agent/7.0.0: resolution: {integrity: sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==} engines: {node: '>= 10'} @@ -34622,6 +35182,22 @@ packages: - supports-color dev: false + /sshpk/1.18.0: + resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} + engines: {node: '>=0.10.0'} + hasBin: true + dependencies: + asn1: 0.2.6 + assert-plus: 1.0.0 + bcrypt-pbkdf: 1.0.2 + dashdash: 1.14.1 + ecc-jsbn: 0.1.2 + getpass: 0.1.7 + jsbn: 0.1.1 + safer-buffer: 2.1.2 + tweetnacl: 0.14.5 + dev: false + /ssri/10.0.5: resolution: {integrity: sha512-bSf16tAFkGeRlUNDjXu8FzaMQt6g2HZJrun7mtMbIPOddxt3GLMSz5VWUWcqTJUPfLEaDIepGxv+bYQW49596A==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -34768,6 +35344,11 @@ packages: - utf-8-validate dev: true + /stream-buffers/3.0.2: + resolution: {integrity: sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==} + engines: {node: '>= 0.10.0'} + dev: false + /stream-shift/1.0.1: resolution: {integrity: sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==} dev: true @@ -34966,7 +35547,7 @@ packages: resolution: {integrity: sha512-WrDlYH1p5jliY7uzSU5nLDY7OCIeRe6FkC0hhScpTGwMthP/Muk38WXGeggjDHKeXAGCs43jUheZ7Ud/NEAJdg==} engines: {node: '>=12.*'} dependencies: - '@types/node': 20.11.20 + '@types/node': 20.6.0 qs: 6.11.0 /striptags/2.2.1: @@ -35416,6 +35997,12 @@ packages: mkdirp: 1.0.4 yallist: 4.0.0 + /tdigest/0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + dependencies: + bintrees: 1.0.2 + dev: false + /telejson/7.1.0: resolution: {integrity: sha512-jFJO4P5gPebZAERPkJsqMAQ0IMA1Hi0AoSfxpnUaV6j6R2SZqlpkbS20U6dEUtA3RUYt2Ak/mTlkQzHH9Rv/hA==} dependencies: @@ -35573,6 +36160,12 @@ packages: xtend: 4.0.2 dev: true + /through2/4.0.2: + resolution: {integrity: sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==} + dependencies: + readable-stream: 3.6.0 + dev: true + /tiny-glob/0.2.9: resolution: {integrity: sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==} dependencies: @@ -35698,6 +36291,14 @@ packages: nopt: 1.0.10 dev: true + /tough-cookie/2.5.0: + resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} + engines: {node: '>=0.8'} + dependencies: + psl: 1.9.0 + punycode: 2.2.0 + dev: false + /tr46/0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -35811,7 +36412,7 @@ packages: dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 29.6.2_@types+node@16.18.11 + jest: 29.6.2_@types+node@18.17.1 jest-util: 29.6.2 json5: 2.2.3 lodash.memoize: 4.1.2 @@ -36000,6 +36601,10 @@ packages: resolution: {integrity: sha512-3IDBalvf6SyvHFS14UiwCWzqdSdo+Q0k2J7DZyJYaHW/iraW9DJpaBKDJpry3yQs3o/t/A+oGaRW3iVt2lKxzA==} dev: false + /tsafe/1.6.6: + resolution: {integrity: sha512-gzkapsdbMNwBnTIjgO758GujLCj031IgHK/PKr2mrmkCSJMhSOR5FeOuSxKLMUoYc0vAA4RGEYYbjt/v6afD3g==} + dev: false + /tsconfck/2.1.2: resolution: {integrity: sha512-ghqN1b0puy3MhhviwO2kGF8SeMDNhEbnKxjK7h6+fvY9JAxqvXi8y5NAHSQv687OVboS2uZIByzGd45/YxrRHg==} engines: {node: ^14.13.1 || ^16 || >=18} @@ -36264,6 +36869,84 @@ packages: dev: true patched: true + /tsup/8.0.2_typescript@5.3.2: + resolution: {integrity: sha512-NY8xtQXdH7hDUAZwcQdY/Vzlw9johQsaqf7iwZ6g1DOUlFYQ5/AtVAjTvihhEyeRlGo4dLRVHtrRaL35M1daqQ==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + dependencies: + bundle-require: 4.0.1_esbuild@0.19.11 + cac: 6.7.14 + chokidar: 3.5.3 + debug: 4.3.4 + esbuild: 0.19.11 + execa: 5.1.1 + globby: 11.1.0 + joycon: 3.1.1 + postcss-load-config: 4.0.1 + resolve-from: 5.0.0 + rollup: 4.6.1 + source-map: 0.8.0-beta.0 + sucrase: 3.32.0 + tree-kill: 1.2.2 + typescript: 5.3.2 + transitivePeerDependencies: + - supports-color + - ts-node + dev: true + + /tsup/8.0.2_typescript@5.3.3: + resolution: {integrity: sha512-NY8xtQXdH7hDUAZwcQdY/Vzlw9johQsaqf7iwZ6g1DOUlFYQ5/AtVAjTvihhEyeRlGo4dLRVHtrRaL35M1daqQ==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + dependencies: + bundle-require: 4.0.1_esbuild@0.19.11 + cac: 6.7.14 + chokidar: 3.5.3 + debug: 4.3.4 + esbuild: 0.19.11 + execa: 5.1.1 + globby: 11.1.0 + joycon: 3.1.1 + postcss-load-config: 4.0.1 + resolve-from: 5.0.0 + rollup: 4.6.1 + source-map: 0.8.0-beta.0 + sucrase: 3.32.0 + tree-kill: 1.2.2 + typescript: 5.3.3 + transitivePeerDependencies: + - supports-color + - ts-node + dev: true + /tsutils/3.21.0: resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} @@ -36323,6 +37006,17 @@ packages: fsevents: 2.3.3 dev: true + /tsx/4.7.1: + resolution: {integrity: sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==} + engines: {node: '>=18.0.0'} + hasBin: true + dependencies: + esbuild: 0.19.11 + get-tsconfig: 4.7.2 + optionalDependencies: + fsevents: 2.3.3 + dev: true + /tty-table/4.1.6: resolution: {integrity: sha512-kRj5CBzOrakV4VRRY5kUWbNYvo/FpOsz65DzI5op9P+cHov3+IqPbo1JE1ZnQGkHdZgNFDsrEjrfqqy/Ply9fw==} engines: {node: '>=8.0.0'} @@ -36363,7 +37057,6 @@ packages: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} dependencies: safe-buffer: 5.2.1 - optional: true /turbo-darwin-64/1.10.3: resolution: {integrity: sha512-IIB9IomJGyD3EdpSscm7Ip1xVWtYb7D0x7oH3vad3gjFcjHJzDz9xZ/iw/qItFEW+wGFcLSRPd+1BNnuLM8AsA==} @@ -36426,6 +37119,10 @@ packages: turbo-windows-arm64: 1.10.3 dev: true + /tweetnacl/0.14.5: + resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} + dev: false + /type-check/0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -37069,6 +37766,12 @@ packages: word-wrap: 1.2.3 dev: false + /uuid/3.4.0: + resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} + deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. + hasBin: true + dev: false + /uuid/8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -37122,6 +37825,15 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + /verror/1.10.0: + resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} + engines: {'0': node >=0.6.0} + dependencies: + assert-plus: 1.0.0 + core-util-is: 1.0.2 + extsprintf: 1.3.0 + dev: false + /vfile-location/4.0.1: resolution: {integrity: sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==} dependencies: @@ -37419,7 +38131,7 @@ packages: dependencies: '@types/node': 18.11.18 esbuild: 0.18.11 - postcss: 8.4.31 + postcss: 8.4.29 rollup: 3.29.1 optionalDependencies: fsevents: 2.3.3 @@ -38173,6 +38885,18 @@ packages: optional: true dev: true + /ws/8.11.0: + resolution: {integrity: sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + /ws/8.12.0: resolution: {integrity: sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==} engines: {node: '>=10.0.0'} @@ -38184,6 +38908,19 @@ packages: optional: true utf-8-validate: optional: true + dev: false + + /ws/8.16.0: + resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true /xdg-app-paths/8.3.0: resolution: {integrity: sha512-mgxlWVZw0TNWHoGmXq+NC3uhCIc55dDpAlDkMQUaIAcQzysb0kxctwv//fvuW61/nAAeUBJMQ8mnZjMmuYwOcQ==} @@ -38238,6 +38975,11 @@ packages: - supports-color dev: true + /xmlhttprequest-ssl/2.0.0: + resolution: {integrity: sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==} + engines: {node: '>=0.4.0'} + dev: false + /xorshift/1.2.0: resolution: {integrity: sha512-iYgNnGyeeJ4t6U11NpA/QiKy+PXn5Aa3Azg5qkwIFz1tBLllQrjjsk9yzD7IAK0naNU4JxdeDgqW9ov4u/hc4g==} dev: false diff --git a/references/v3-catalog/package.json b/references/v3-catalog/package.json index 0f06a78f5..14ae3e3c7 100644 --- a/references/v3-catalog/package.json +++ b/references/v3-catalog/package.json @@ -3,6 +3,7 @@ "version": "0.1.0", "private": true, "scripts": { + "build:trigger": "trigger.dev build", "dev:trigger": "trigger.dev dev", "start": "ts-node -r tsconfig-paths/register -r dotenv/config src/index.ts", "start:concurrency": "ts-node -r tsconfig-paths/register -r dotenv/config src/concurrencyUsage.ts", diff --git a/references/v3-catalog/src/index.ts b/references/v3-catalog/src/index.ts index 5ff6cab96..b6046ffd9 100644 --- a/references/v3-catalog/src/index.ts +++ b/references/v3-catalog/src/index.ts @@ -1,7 +1,7 @@ -import { parentTask, simpleParentTask } from "./trigger/simple"; +import { parentTask } from "./trigger/simple"; export async function main() { - const handle = await simpleParentTask.trigger({ + const handle = await parentTask.trigger({ payload: { message: "This is a message from the trigger-dev CLI" }, });