Adding createImageEdit and createImageVariation tasks to openai

.changeset/good-donkeys-exist.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/integration-kit": patch
+"@trigger.dev/openai": patch
+---
+
+Adding createImageEdit and createImageVariation tasks to openai

docs/integrations/apis/openai.mdx

@@ -83,6 +83,8 @@ Tasks that are marked as "long-running" can last longer than your serverless tim
 | `listModels`                     | Lists the available models.                                               |
 | `createEdit`                     | Edits a given text prompt.                                                |
 | `createImage`                    | Generates images from textual descriptions.                               |
+| `createImageEdit`                | Creates an edited or extended image given an original image and a prompt  |
+| `createImageVariation`           | Creates a variation of a given image.                                     |
 | `createEmbedding`                | Generates embeddings for a given text.                                    |
 | `createFile`                     | Uploads a file to the OpenAI API.                                         |
 | `listFiles`                      | Lists the uploaded files.                                                 |

examples/job-catalog/src/supabase.ts

@@ -1,6 +1,7 @@
 import { TriggerClient } from "@trigger.dev/sdk";
 import { createExpressServer } from "@trigger.dev/express";
 import { Supabase, SupabaseManagement } from "@trigger.dev/supabase";
+import { OpenAI } from "@trigger.dev/openai";
 import { Database } from "./supabase-types";
 
 const supabaseManagement = new SupabaseManagement({
@@ -16,6 +17,11 @@ const supabase = new Supabase({
   supabaseUrl: process.env["SUPABASE_URL"]!,
 });
 
+const openai = new OpenAI({
+  id: "open-ai",
+  apiKey: process.env["OPENAI_API_KEY"]!,
+});
+
 export const client = new TriggerClient({
   id: "job-catalog",
   apiKey: process.env["TRIGGER_API_KEY"],
@@ -81,24 +87,34 @@ client.defineJob({
     },
   }),
   integrations: {
-    supabase,
+    openai,
   },
   run: async (payload, io, ctx) => {
-    const { signedUrl } = await io.supabase.runTask("create-signed-url", async (db) => {
+    if (!payload.record.name) {
-      if (!payload.record.name) {
+      return;
-        throw new Error("Missing record name");
+    }
-      }
 
-      const { data, error } = await db.storage
+    const {
-        .from("example_bucket")
+      data: { publicUrl },
-        .createSignedUrl(payload.record.name, 60);
+    } = supabase.native.storage.from("example_bucket").getPublicUrl(payload.record.name);
 
-      if (error) {
+    const imageVariation = await io.openai.createImageVariation("variation-image", {
-        throw error;
+      image: publicUrl,
-      }
+      n: 2,
-
+      response_format: "url",
-      return data;
+      size: "512x512",
     });
+
+    const imageEdit = await io.openai.createImageEdit("edit-image", {
+      image: publicUrl,
+      prompt:
+        "Fill in the background to make it seem like the cat is on the moon with a beautiful view of the earth.",
+      n: 2,
+      response_format: "url",
+      size: "512x512",
+    });
+
+    // return imageEdit;
   },
 });
 

integrations/openai/src/tasks.ts

@@ -10,7 +10,7 @@ import {
 } from "openai";
 import { OpenAIIntegrationAuth } from "./types";
 import { redactString } from "@trigger.dev/sdk";
-import { Prettify, fileFromString, truncate } from "@trigger.dev/integration-kit";
+import { Prettify, fileFromString, fileFromUrl, truncate } from "@trigger.dev/integration-kit";
 import { createTaskUsageProperties, onTaskError } from "./taskUtils";
 
 type OpenAIClientType = InstanceType<typeof OpenAIApi>;
@@ -326,6 +326,156 @@ export const createImage: AuthenticatedTask<
   },
 };
 
+export type CreateImageEditRequest = {
+  image: string | File;
+  prompt: string;
+  mask?: string | File;
+  n?: number;
+  size?: "256x256" | "512x512" | "1024x1024";
+  response_format?: "url" | "b64_json";
+  user?: string;
+};
+
+type CreateImageEditResponseData = Prettify<
+  Awaited<ReturnType<OpenAIClientType["createImageEdit"]>>["data"]
+>;
+
+export const createImageEdit: AuthenticatedTask<
+  OpenAIClientType,
+  Prettify<CreateImageEditRequest>,
+  CreateImageEditResponseData
+> = {
+  onError: onTaskError,
+  run: async (params, client, task) => {
+    const file = typeof params.image === "string" ? await fileFromUrl(params.image) : params.image;
+    const mask = typeof params.mask === "string" ? await fileFromUrl(params.mask) : params.mask;
+
+    const response = await client.createImageEdit(
+      file,
+      params.prompt,
+      mask,
+      params.n,
+      params.size,
+      params.response_format,
+      params.user
+    );
+
+    return response.data;
+  },
+  init: (params) => {
+    let properties = [];
+
+    properties.push({
+      label: "Prompt",
+      text: params.prompt,
+    });
+
+    if (params.n) {
+      properties.push({
+        label: "Number of images",
+        text: params.n.toString(),
+      });
+    }
+
+    if (params.size) {
+      properties.push({
+        label: "Size",
+        text: params.size,
+      });
+    }
+
+    if (params.response_format) {
+      properties.push({
+        label: "Response format",
+        text: params.response_format,
+      });
+    }
+
+    if (typeof params.image === "string") {
+      properties.push({
+        label: "Image URL",
+        text: params.image,
+        url: params.image,
+      });
+    }
+
+    return {
+      name: "Create image edit",
+      params,
+      icon: "openai",
+      properties,
+    };
+  },
+};
+
+type CreateImageVariationResponseData = Prettify<
+  Awaited<ReturnType<OpenAIClientType["createImageVariation"]>>["data"]
+>;
+
+export type CreateImageVariationRequest = {
+  image: string | File;
+  n?: number;
+  size?: "256x256" | "512x512" | "1024x1024";
+  response_format?: "url" | "b64_json";
+  user?: string;
+};
+
+export const createImageVariation: AuthenticatedTask<
+  OpenAIClientType,
+  Prettify<CreateImageVariationRequest>,
+  CreateImageVariationResponseData
+> = {
+  onError: onTaskError,
+  run: async (params, client, task) => {
+    const file = typeof params.image === "string" ? await fileFromUrl(params.image) : params.image;
+
+    const response = await client
+      .createImageVariation(file, params.n, params.size, params.response_format, params.user)
+      .then((res) => res.data);
+
+    return response;
+  },
+  init: (params) => {
+    let properties = [];
+
+    if (params.n) {
+      properties.push({
+        label: "Number of images",
+        text: params.n.toString(),
+      });
+    }
+
+    if (params.size) {
+      properties.push({
+        label: "Size",
+        text: params.size,
+      });
+    }
+
+    if (params.response_format) {
+      properties.push({
+        label: "Response format",
+        text: params.response_format,
+      });
+    }
+
+    if (typeof params.image === "string") {
+      properties.push({
+        label: "Image URL",
+        text: params.image,
+        url: params.image,
+      });
+    }
+
+    return {
+      name: "Create image variation",
+      params,
+      icon: "openai",
+      properties,
+    };
+  },
+};
+
 type CreateEmbeddingResponseData = Prettify<
   Awaited<ReturnType<OpenAIClientType["createEmbedding"]>>["data"]
 >;
@@ -383,7 +533,7 @@ export const createFile: AuthenticatedTask<
     let file: File;
 
     if (typeof params.file === "string") {
-      file = (await fileFromString(params.file, params.fileName ?? "file.txt")) as any;
+      file = await fileFromString(params.file, params.fileName ?? "file.txt");
     } else {
       file = params.file;
     }
@@ -441,10 +591,10 @@ export const createFineTuneFile: AuthenticatedTask<
 > = {
   onError: onTaskError,
   run: async (params, client) => {
-    const file = (await fileFromString(
+    const file = await fileFromString(
       params.examples.map((d) => JSON.stringify(d)).join("\n"),
       params.fileName
-    )) as any;
+    );
 
     return client.createFile(file, "fine-tune").then((res) => res.data);
   },

packages/integration-kit/package.json

@@ -20,6 +20,7 @@
   "devDependencies": {
     "@trigger.dev/tsconfig": "workspace:*",
     "@types/node": "18",
+    "@types/node-fetch": "2.6.x",
     "@types/uuid": "^9.0.0",
     "rimraf": "^3.0.2",
     "tsup": "^6.5.0",
@@ -32,6 +33,7 @@
     "typecheck": "tsup --dts-resolve --no-dts"
   },
   "dependencies": {
+    "node-fetch": "2.6.x",
     "uuid": "^9.0.0"
   },
   "engines": {

packages/integration-kit/src/file.ts

@@ -1,11 +1,20 @@
 import fs, { promises } from "fs";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
+import fetch from "node-fetch";
 
-export async function fileFromString(contents: string, fileName: string) {
+export async function fileFromString(contents: string | Buffer, fileName: string): Promise<File> {
   const directory = path.join("tmp", uuidv4());
   await promises.mkdir(directory, { recursive: true });
   const filePath = path.join(directory, fileName);
   await promises.writeFile(filePath, contents);
-  return fs.createReadStream(filePath);
+  return fs.createReadStream(filePath) as unknown as File;
+}
+
+export async function fileFromUrl(url: string) {
+  const response = await fetch(url);
+  const content = await response.buffer();
+  const fileName = path.basename(url);
+
+  return fileFromString(content, fileName);
 }

pnpm-lock.yaml

@@ -956,16 +956,20 @@ importers:
     specifiers:
       '@trigger.dev/tsconfig': workspace:*
       '@types/node': '18'
+      '@types/node-fetch': 2.6.x
       '@types/uuid': ^9.0.0
+      node-fetch: 2.6.x
       rimraf: ^3.0.2
       tsup: ^6.5.0
       tsx: ^3.12.1
       uuid: ^9.0.0
     dependencies:
+      node-fetch: 2.6.12
       uuid: 9.0.0
     devDependencies:
       '@trigger.dev/tsconfig': link:../../config-packages/tsconfig
       '@types/node': 18.15.13
+      '@types/node-fetch': 2.6.2
       '@types/uuid': 9.0.0
       rimraf: 3.0.2
       tsup: 6.6.3
@@ -11081,7 +11085,7 @@ packages:
   /@types/node-fetch/2.6.2:
     resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==}
     dependencies:
-      '@types/node': 20.3.2
+      '@types/node': 20.4.5
       form-data: 3.0.1
     dev: true
 
@@ -11105,10 +11109,6 @@ packages:
   /@types/node/18.17.1:
     resolution: {integrity: sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==}
 
-  /@types/node/20.3.2:
-    resolution: {integrity: sha512-vOBLVQeCQfIcF/2Y7eKFTqrMnizK5lRNQ7ykML/5RuwVXVWxYkgwS7xbt4B6fKCUPgbSL5FSsjHQpaGQP/dQmw==}
-    dev: true
-
   /@types/node/20.4.2:
     resolution: {integrity: sha512-Dd0BYtWgnWJKwO1jkmTrzofjK2QXXcai0dmtzvIBhcA+RsG5h8R3xlyta0kGOZRNfL9GuRtb1knmPEhQrePCEw==}