v3: edge runtime support (#1172)

* v3: remove node:stream and simplify env var upload API to better work with non-node runtimes * Remove file/Response envvars upload docs * Add changeset
2024-06-21 13:33:38 +01:00
parent 65f960e883
commit 098932ea96
11 changed files with 34 additions and 414 deletions
@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: vercel edge runtime support
@@ -811,17 +811,7 @@ paths:
                  description: Whether to override existing variables or not
                  default: false
              required: ["variables"]
-          multipart/form-data:
-            schema:
-              type: object
-              properties:
-                variables:
-                  type: string
-                  format: binary
-                override:
-                  type: boolean
-              required:
-                - variables
+
      responses:
        "200":
          description: Environment variables imported successfully
@@ -864,57 +854,11 @@ paths:
          source: |-
            import { envvars } from "@trigger.dev/sdk/v3";

-            // Import variables from an array
            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: [
-                {
-                  name: "SLACK_API_KEY",
-                  value: "slack_123456"
-                }
-              ],
+              variables: { SLACK_API_KEY: "slack_key_1234" },
              override: false
            });
-        - lang: typescript
-          label: Import variables from a read stream
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-            import { createReadStream } from "node:fs";

-            // Import variables in dotenv format from a file
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: createReadStream(".env"),
-              override: false
-            });
-        - lang: typescript
-          label: Import variables from a response
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-
-            // Import variables in dotenv format from a response
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: await fetch("https://example.com/.env"),
-              override: false
-            });
-        - lang: typescript
-          label: Import variables from a Buffer
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-
-            // Import variables in dotenv format from a buffer
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: Buffer.from("SLACK_API_KEY=slack_1234"),
-              override: false
-            });
-        - lang: typescript
-          label: Import variables from a File
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-
-            // Import variables in dotenv format from a file
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: new File(["SLACK_API_KEY=slack_1234"], ".env"),
-              override: false
-            });

  "/api/v1/projects/{projectRef}/envvars/{env}/{name}":
    parameters:
@@ -178,7 +178,7 @@ export async function readConfig(
    write: true,
    format: "cjs",
    platform: "node",
-    target: ["es2018", "node18"],
+    target: ["es2020", "node18"],
    outfile: builtConfigFilePath,
    logLevel: "silent",
    plugins: [
@@ -142,7 +142,6 @@
    "@opentelemetry/sdk-trace-base": "^1.22.0",
    "@opentelemetry/sdk-trace-node": "^1.22.0",
    "@opentelemetry/semantic-conventions": "^1.22.0",
-    "form-data-encoder": "^4.0.2",
    "humanize-duration": "^3.27.3",
    "socket.io-client": "4.7.4",
    "superjson": "^2.2.1",
@@ -157,6 +156,7 @@
    "@types/humanize-duration": "^3.27.1",
    "@types/jest": "^29.5.3",
    "@types/node": "20.12.7",
+    "@types/readable-stream": "^4.0.14",
    "jest": "^29.6.2",
    "rimraf": "^3.0.2",
    "socket.io": "4.7.4",
@@ -3,8 +3,7 @@ import { fromZodError } from "zod-validation-error";
 import { ApiConnectionError, ApiError } from "./errors";
 import { RetryOptions } from "../schemas";
 import { calculateNextRetryDelay } from "../utils/retries";
-import { FormDataEncoder } from "form-data-encoder";
-import { Readable } from "node:stream";
+
 import {
  CursorPage,
  CursorPageParams,
@@ -114,59 +113,6 @@ export function zodfetchOffsetLimitPage<TItemSchema extends z.ZodTypeAny>(
  return new OffsetLimitPagePromise(fetchResult, schema, url, params, requestInit, options);
 }

-export function zodupload<
-  TResponseBodySchema extends z.ZodTypeAny,
-  TBody = Record<string, unknown>,
->(
-  schema: TResponseBodySchema,
-  url: string,
-  body: TBody,
-  requestInit?: RequestInit,
-  options?: ZodFetchOptions
-): ApiPromise<z.output<TResponseBodySchema>> {
-  const finalRequestInit = createMultipartFormRequestInit(body, requestInit);
-
-  return new ApiPromise(_doZodFetch(schema, url, finalRequestInit, options));
-}
-
-async function createMultipartFormRequestInit<TBody = Record<string, unknown>>(
-  body: TBody,
-  requestInit?: RequestInit
-): Promise<RequestInit> {
-  const form = await createForm(body);
-  const encoder = new FormDataEncoder(form);
-
-  const finalHeaders: Record<string, string> = {};
-
-  for (const [key, value] of Object.entries(requestInit?.headers || {})) {
-    finalHeaders[key] = value as string;
-  }
-
-  for (const [key, value] of Object.entries(encoder.headers)) {
-    finalHeaders[key] = value;
-  }
-
-  finalHeaders["Content-Length"] = String(encoder.contentLength);
-
-  const finalRequestInit: RequestInit = {
-    ...requestInit,
-    headers: finalHeaders,
-    body: Readable.from(encoder) as any,
-    // @ts-expect-error
-    duplex: "half",
-  };
-
-  return finalRequestInit;
-}
-
-const createForm = async <T = Record<string, unknown>>(body: T | undefined): Promise<FormData> => {
-  const form = new FormData();
-  await Promise.all(
-    Object.entries(body || {}).map(([key, value]) => addFormValue(form, key, value))
-  );
-  return form;
-};
-
 type ZodFetchResult<T> = {
  data: T;
  response: Response;
@@ -324,214 +270,6 @@ function requestInitWithCache(requestInit?: RequestInit): RequestInit {
  }
 }

-const addFormValue = async (form: FormData, key: string, value: unknown): Promise<void> => {
-  if (value === undefined) return;
-  if (value == null) {
-    throw new TypeError(
-      `Received null for "${key}"; to pass null in FormData, you must use the string 'null'`
-    );
-  }
-
-  // TODO: make nested formats configurable
-  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
-    form.append(key, String(value));
-  } else if (
-    isUploadable(value) ||
-    isBlobLike(value) ||
-    value instanceof Buffer ||
-    value instanceof ArrayBuffer
-  ) {
-    const file = await toFile(value);
-    form.append(key, file as File);
-  } else if (Array.isArray(value)) {
-    await Promise.all(value.map((entry) => addFormValue(form, key + "[]", entry)));
-  } else if (typeof value === "object") {
-    await Promise.all(
-      Object.entries(value).map(([name, prop]) => addFormValue(form, `${key}[${name}]`, prop))
-    );
-  } else {
-    throw new TypeError(
-      `Invalid value given to form, expected a string, number, boolean, object, Array, File or Blob but got ${value} instead`
-    );
-  }
-};
-
-export type ToFileInput = Uploadable | Exclude<BlobLikePart, string> | AsyncIterable<BlobLikePart>;
-
-/**
- * Helper for creating a {@link File} to pass to an SDK upload method from a variety of different data formats
- * @param value the raw content of the file.  Can be an {@link Uploadable}, {@link BlobLikePart}, or {@link AsyncIterable} of {@link BlobLikePart}s
- * @param {string=} name the name of the file. If omitted, toFile will try to determine a file name from bits if possible
- * @param {Object=} options additional properties
- * @param {string=} options.type the MIME type of the content
- * @param {number=} options.lastModified the last modified timestamp
- * @returns a {@link File} with the given properties
- */
-export async function toFile(
-  value: ToFileInput | PromiseLike<ToFileInput>,
-  name?: string | null | undefined,
-  options?: FilePropertyBag | undefined
-): Promise<FileLike> {
-  // If it's a promise, resolve it.
-  value = await value;
-
-  // Use the file's options if there isn't one provided
-  options ??= isFileLike(value) ? { lastModified: value.lastModified, type: value.type } : {};
-
-  if (isResponseLike(value)) {
-    const blob = await value.blob();
-    name ||= new URL(value.url).pathname.split(/[\\/]/).pop() ?? "unknown_file";
-
-    return new File([blob as any], name, options);
-  }
-
-  const bits = await getBytes(value);
-
-  name ||= getName(value) ?? "unknown_file";
-
-  if (!options.type) {
-    const type = (bits[0] as any)?.type;
-    if (typeof type === "string") {
-      options = { ...options, type };
-    }
-  }
-
-  return new File(bits, name, options);
-}
-
-function getName(value: any): string | undefined {
-  return (
-    getStringFromMaybeBuffer(value.name) ||
-    getStringFromMaybeBuffer(value.filename) ||
-    // For fs.ReadStream
-    getStringFromMaybeBuffer(value.path)?.split(/[\\/]/).pop()
-  );
-}
-
-const getStringFromMaybeBuffer = (x: string | Buffer | unknown): string | undefined => {
-  if (typeof x === "string") return x;
-  if (typeof Buffer !== "undefined" && x instanceof Buffer) return String(x);
-  return undefined;
-};
-
-async function getBytes(value: ToFileInput): Promise<Array<BlobPart>> {
-  let parts: Array<BlobPart> = [];
-  if (
-    typeof value === "string" ||
-    ArrayBuffer.isView(value) || // includes Uint8Array, Buffer, etc.
-    value instanceof ArrayBuffer
-  ) {
-    parts.push(value);
-  } else if (isBlobLike(value)) {
-    parts.push(await value.arrayBuffer());
-  } else if (
-    isAsyncIterableIterator(value) // includes Readable, ReadableStream, etc.
-  ) {
-    for await (const chunk of value) {
-      parts.push(chunk as BlobPart); // TODO, consider validating?
-    }
-  } else {
-    throw new Error(
-      `Unexpected data type: ${typeof value}; constructor: ${value?.constructor
-        ?.name}; props: ${propsForError(value)}`
-    );
-  }
-
-  return parts;
-}
-
-function propsForError(value: any): string {
-  const props = Object.getOwnPropertyNames(value);
-  return `[${props.map((p) => `"${p}"`).join(", ")}]`;
-}
-
-const isAsyncIterableIterator = (value: any): value is AsyncIterableIterator<unknown> =>
-  value != null && typeof value === "object" && typeof value[Symbol.asyncIterator] === "function";
-
-/**
- * Intended to match web.Blob, node.Blob, node-fetch.Blob, etc.
- */
-export interface BlobLike {
-  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) */
-  readonly size: number;
-  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) */
-  readonly type: string;
-  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) */
-  text(): Promise<string>;
-  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) */
-  slice(start?: number, end?: number): BlobLike;
-  // unfortunately @types/node-fetch@^2.6.4 doesn't type the arrayBuffer method
-}
-
-/**
- * Intended to match web.File, node.File, node-fetch.File, etc.
- */
-export interface FileLike extends BlobLike {
-  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) */
-  readonly lastModified: number;
-  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) */
-  readonly name: string;
-}
-
-/**
- * Intended to match web.Response, node.Response, node-fetch.Response, etc.
- */
-export interface ResponseLike {
-  url: string;
-  blob(): Promise<BlobLike>;
-}
-
-export type Uploadable = FileLike | ResponseLike | Readable;
-
-export const isResponseLike = (value: any): value is ResponseLike =>
-  value != null &&
-  typeof value === "object" &&
-  typeof value.url === "string" &&
-  typeof value.blob === "function";
-
-export const isFileLike = (value: any): value is FileLike =>
-  value != null &&
-  typeof value === "object" &&
-  typeof value.name === "string" &&
-  typeof value.lastModified === "number" &&
-  isBlobLike(value);
-
-/**
- * The BlobLike type omits arrayBuffer() because @types/node-fetch@^2.6.4 lacks it; but this check
- * adds the arrayBuffer() method type because it is available and used at runtime
- */
-export const isBlobLike = (
-  value: any
-): value is BlobLike & { arrayBuffer(): Promise<ArrayBuffer> } =>
-  value != null &&
-  typeof value === "object" &&
-  typeof value.size === "number" &&
-  typeof value.type === "string" &&
-  typeof value.text === "function" &&
-  typeof value.slice === "function" &&
-  typeof value.arrayBuffer === "function";
-
-export const isFsReadStream = (value: any): value is Readable => value instanceof Readable;
-
-export const isUploadable = (value: any): value is Uploadable => {
-  return isFileLike(value) || isResponseLike(value) || isFsReadStream(value);
-};
-
-export type BlobLikePart =
-  | string
-  | ArrayBuffer
-  | ArrayBufferView
-  | BlobLike
-  | Uint8Array
-  | DataView;
-
-export const isRecordLike = (value: any): value is Record<string, string> =>
-  value != null &&
-  typeof value === "object" &&
-  !Array.isArray(value) &&
-  Object.keys(value).length > 0 &&
-  Object.keys(value).every((key) => typeof key === "string" && typeof value[key] === "string");
-
 /**
 * A subclass of `Promise` providing additional helper methods
 * for interacting with the SDK.
@@ -14,7 +14,6 @@ import {
  EnvironmentVariables,
  ListRunResponseItem,
  ListScheduleOptions,
-  ListSchedulesResult,
  ReplayRunResponse,
  RetrieveRunResponse,
  ScheduleObject,
@@ -28,11 +27,9 @@ import { taskContext } from "../task-context-api";
 import {
  CursorPagePromise,
  ZodFetchOptions,
-  isRecordLike,
  zodfetch,
  zodfetchCursorPage,
  zodfetchOffsetLimitPage,
-  zodupload,
 } from "./core";
 import { ApiError } from "./errors";
 import {
@@ -331,27 +328,15 @@ export class ApiClient {
  }

  importEnvVars(projectRef: string, slug: string, body: ImportEnvironmentVariablesParams) {
-    if (isRecordLike(body.variables)) {
-      return zodfetch(
-        EnvironmentVariableResponseBody,
-        `${this.baseUrl}/api/v1/projects/${projectRef}/envvars/${slug}/import`,
-        {
-          method: "POST",
-          headers: this.#getHeaders(false),
-          body: JSON.stringify(body),
-        }
-      );
-    } else {
-      return zodupload(
-        EnvironmentVariableResponseBody,
-        `${this.baseUrl}/api/v1/projects/${projectRef}/envvars/${slug}/import`,
-        body,
-        {
-          method: "POST",
-          headers: this.#getHeaders(false),
-        }
-      );
-    }
+    return zodfetch(
+      EnvironmentVariableResponseBody,
+      `${this.baseUrl}/api/v1/projects/${projectRef}/envvars/${slug}/import`,
+      {
+        method: "POST",
+        headers: this.#getHeaders(false),
+        body: JSON.stringify(body),
+      }
+    );
  }

  retrieveEnvVar(projectRef: string, slug: string, key: string) {
@@ -1,18 +1,13 @@
 import { RunStatus } from "../schemas";
-import { BlobLikePart, Uploadable } from "./core";
 import { CursorPageParams } from "./pagination";

 export interface ImportEnvironmentVariablesParams {
  /**
   * The variables to be imported. If a variable with the same key already exists, it will be overwritten when `override` is `true`.
   *
-   * There are two ways to specify the variables:
-   *
-   * 1. As a record of key-value pairs. e.g. `{ "key1": "value1", "key2": "value2" }`
-   * 2. As an "uploadable" object in dotenv format. An uploadable can be a Node readable stream, a string, or a Buffer. You can also pass the return value of a `fetch` call.
+   * To specify the variables, you can pass them in as a record of key-value pairs. e.g. `{ "key1": "value1", "key2": "value2" }`
   */
-  variables: Uploadable | BlobLikePart | Record<string, string>;
-
+  variables: Record<string, string>;
  override?: boolean;
 }

@@ -17,5 +17,4 @@ export default defineConfig({
    "./src/v3/workers/index.ts",
    "./src/v3/zodfetch.ts",
  ],
-  external: ["node:stream"],
 });
@@ -1754,9 +1754,6 @@ importers:
      '@opentelemetry/semantic-conventions':
        specifier: ^1.22.0
        version: 1.22.0
-      form-data-encoder:
-        specifier: ^4.0.2
-        version: 4.0.2
      humanize-duration:
        specifier: ^3.27.3
        version: 3.27.3
@@ -1794,6 +1791,9 @@ importers:
      '@types/node':
        specifier: 20.12.7
        version: 20.12.7
+      '@types/readable-stream':
+        specifier: ^4.0.14
+        version: 4.0.14
      jest:
        specifier: ^29.6.2
        version: 29.6.2(@types/node@20.12.7)
@@ -16125,6 +16125,13 @@ packages:
      '@types/prop-types': 15.7.5
      csstype: 3.1.1

+  /@types/readable-stream@4.0.14:
+    resolution: {integrity: sha512-xZn/AuUbCMShGsqH/ehZtGDwQtbx00M9rZ2ENLe4tOjFZ/JFeWMhEZkk2fEe1jAUqqEAURIkFJ7Az/go8mM1/w==}
+    dependencies:
+      '@types/node': 18.19.20
+      safe-buffer: 5.1.2
+    dev: true
+
  /@types/request@2.48.12:
    resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==}
    dependencies:
@@ -22965,11 +22972,6 @@ packages:
    engines: {node: '>= 14.17'}
    dev: false

-  /form-data-encoder@4.0.2:
-    resolution: {integrity: sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==}
-    engines: {node: '>= 18'}
-    dev: false
-
  /form-data@2.3.3:
    resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==}
    engines: {node: '>= 0.12'}
@@ -50,43 +50,6 @@ async function doEnvVars() {
  const deleteResponse = await envvars.del("yubjwjsfkxnylobaqvqz", "dev", "MY_ENV_VAR_CREATE");

  console.log("deleteResponse", deleteResponse);
-
-  const response2 = await envvars.upload("yubjwjsfkxnylobaqvqz", "dev", {
-    variables: createReadStream(".uploadable-env"),
-    override: true,
-  });
-
-  console.log("response2", response2);
-
-  const response3 = await envvars.upload("yubjwjsfkxnylobaqvqz", "prod", {
-    variables: createReadStream(".uploadable-env"),
-    override: true,
-  });
-
-  console.log("response3", response3);
-
-  const response4 = await envvars.upload("yubjwjsfkxnylobaqvqz", "prod", {
-    variables: await fetch(
-      "https://gist.githubusercontent.com/ericallam/7a1001c6b03986a74d0f8aad4fd890aa/raw/fe2bc4da82f3b17178d47f58ec1458af47af5035/.env"
-    ),
-    override: true,
-  });
-
-  console.log("response4", response4);
-
-  const response5 = await envvars.upload("yubjwjsfkxnylobaqvqz", "prod", {
-    variables: new File(["IM_A_FILE=GREAT_FOR_YOU"], ".env"),
-    override: true,
-  });
-
-  console.log("response5", response5);
-
-  const response6 = await envvars.upload("yubjwjsfkxnylobaqvqz", "prod", {
-    variables: Buffer.from("IN_BUFFER=TRUE"),
-    override: true,
-  });
-
-  console.log("response6", response6);
 }

 async function doRuns() {
@@ -273,8 +236,8 @@ async function doTriggerUnfriendlyTaskId() {
 }

 // doRuns().catch(console.error);
-doListRuns().catch(console.error);
+// doListRuns().catch(console.error);
 // doScheduleLists().catch(console.error);
 // doSchedules().catch(console.error);
-// doEnvVars().catch(console.error);
+doEnvVars().catch(console.error);
 // doTriggerUnfriendlyTaskId().catch(console.error);
@@ -31,18 +31,6 @@ export const taskWithSpecialCharacters = task({
  },
 });

-export const updateEnvVars = task({
-  id: "update-env-vars",
-  run: async () => {
-    return await envvars.upload({
-      variables: await fetch(
-        "https://gist.githubusercontent.com/ericallam/7a1001c6b03986a74d0f8aad4fd890aa/raw/fe2bc4da82f3b17178d47f58ec1458af47af5035/.env"
-      ),
-      override: true,
-    });
-  },
-});
-
 export const createJsonHeroDoc = task({
  id: "create-jsonhero-doc",
  run: async (payload: { title: string; content: any }, { ctx }) => {