From 8412680863798e25161ed2f91c93de4874f1e16b Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 9 Aug 2023 10:39:14 +0100 Subject: [PATCH] Adding createImageEdit and createImageVariation tasks to openai --- .changeset/good-donkeys-exist.md | 6 + docs/integrations/apis/openai.mdx | 2 + examples/job-catalog/src/supabase.ts | 42 ++++--- integrations/openai/src/tasks.ts | 158 +++++++++++++++++++++++++- packages/integration-kit/package.json | 2 + packages/integration-kit/src/file.ts | 13 ++- pnpm-lock.yaml | 10 +- 7 files changed, 209 insertions(+), 24 deletions(-) create mode 100644 .changeset/good-donkeys-exist.md diff --git a/.changeset/good-donkeys-exist.md b/.changeset/good-donkeys-exist.md new file mode 100644 index 000000000..613667bad --- /dev/null +++ b/.changeset/good-donkeys-exist.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/integration-kit": patch +"@trigger.dev/openai": patch +--- + +Adding createImageEdit and createImageVariation tasks to openai diff --git a/docs/integrations/apis/openai.mdx b/docs/integrations/apis/openai.mdx index 66dc5b1e0..f37f35e6b 100644 --- a/docs/integrations/apis/openai.mdx +++ b/docs/integrations/apis/openai.mdx @@ -83,6 +83,8 @@ Tasks that are marked as "long-running" can last longer than your serverless tim | `listModels` | Lists the available models. | | `createEdit` | Edits a given text prompt. | | `createImage` | Generates images from textual descriptions. | +| `createImageEdit` | Creates an edited or extended image given an original image and a prompt | +| `createImageVariation` | Creates a variation of a given image. | | `createEmbedding` | Generates embeddings for a given text. | | `createFile` | Uploads a file to the OpenAI API. | | `listFiles` | Lists the uploaded files. | diff --git a/examples/job-catalog/src/supabase.ts b/examples/job-catalog/src/supabase.ts index 2d8205bed..1ceea2cc1 100644 --- a/examples/job-catalog/src/supabase.ts +++ b/examples/job-catalog/src/supabase.ts @@ -1,6 +1,7 @@ import { TriggerClient } from "@trigger.dev/sdk"; import { createExpressServer } from "@trigger.dev/express"; import { Supabase, SupabaseManagement } from "@trigger.dev/supabase"; +import { OpenAI } from "@trigger.dev/openai"; import { Database } from "./supabase-types"; const supabaseManagement = new SupabaseManagement({ @@ -16,6 +17,11 @@ const supabase = new Supabase({ supabaseUrl: process.env["SUPABASE_URL"]!, }); +const openai = new OpenAI({ + id: "open-ai", + apiKey: process.env["OPENAI_API_KEY"]!, +}); + export const client = new TriggerClient({ id: "job-catalog", apiKey: process.env["TRIGGER_API_KEY"], @@ -81,24 +87,34 @@ client.defineJob({ }, }), integrations: { - supabase, + openai, }, run: async (payload, io, ctx) => { - const { signedUrl } = await io.supabase.runTask("create-signed-url", async (db) => { - if (!payload.record.name) { - throw new Error("Missing record name"); - } + if (!payload.record.name) { + return; + } - const { data, error } = await db.storage - .from("example_bucket") - .createSignedUrl(payload.record.name, 60); + const { + data: { publicUrl }, + } = supabase.native.storage.from("example_bucket").getPublicUrl(payload.record.name); - if (error) { - throw error; - } - - return data; + const imageVariation = await io.openai.createImageVariation("variation-image", { + image: publicUrl, + n: 2, + response_format: "url", + size: "512x512", }); + + const imageEdit = await io.openai.createImageEdit("edit-image", { + image: publicUrl, + prompt: + "Fill in the background to make it seem like the cat is on the moon with a beautiful view of the earth.", + n: 2, + response_format: "url", + size: "512x512", + }); + + // return imageEdit; }, }); diff --git a/integrations/openai/src/tasks.ts b/integrations/openai/src/tasks.ts index 4bc7021ec..16373468b 100644 --- a/integrations/openai/src/tasks.ts +++ b/integrations/openai/src/tasks.ts @@ -10,7 +10,7 @@ import { } from "openai"; import { OpenAIIntegrationAuth } from "./types"; import { redactString } from "@trigger.dev/sdk"; -import { Prettify, fileFromString, truncate } from "@trigger.dev/integration-kit"; +import { Prettify, fileFromString, fileFromUrl, truncate } from "@trigger.dev/integration-kit"; import { createTaskUsageProperties, onTaskError } from "./taskUtils"; type OpenAIClientType = InstanceType; @@ -326,6 +326,156 @@ export const createImage: AuthenticatedTask< }, }; +export type CreateImageEditRequest = { + image: string | File; + prompt: string; + mask?: string | File; + n?: number; + size?: "256x256" | "512x512" | "1024x1024"; + response_format?: "url" | "b64_json"; + user?: string; +}; + +type CreateImageEditResponseData = Prettify< + Awaited>["data"] +>; + +export const createImageEdit: AuthenticatedTask< + OpenAIClientType, + Prettify, + CreateImageEditResponseData +> = { + onError: onTaskError, + run: async (params, client, task) => { + const file = typeof params.image === "string" ? await fileFromUrl(params.image) : params.image; + const mask = typeof params.mask === "string" ? await fileFromUrl(params.mask) : params.mask; + + const response = await client.createImageEdit( + file, + params.prompt, + mask, + params.n, + params.size, + params.response_format, + params.user + ); + + return response.data; + }, + init: (params) => { + let properties = []; + + properties.push({ + label: "Prompt", + text: params.prompt, + }); + + if (params.n) { + properties.push({ + label: "Number of images", + text: params.n.toString(), + }); + } + + if (params.size) { + properties.push({ + label: "Size", + text: params.size, + }); + } + + if (params.response_format) { + properties.push({ + label: "Response format", + text: params.response_format, + }); + } + + if (typeof params.image === "string") { + properties.push({ + label: "Image URL", + text: params.image, + url: params.image, + }); + } + + return { + name: "Create image edit", + params, + icon: "openai", + properties, + }; + }, +}; + +type CreateImageVariationResponseData = Prettify< + Awaited>["data"] +>; + +export type CreateImageVariationRequest = { + image: string | File; + n?: number; + size?: "256x256" | "512x512" | "1024x1024"; + response_format?: "url" | "b64_json"; + user?: string; +}; + +export const createImageVariation: AuthenticatedTask< + OpenAIClientType, + Prettify, + CreateImageVariationResponseData +> = { + onError: onTaskError, + run: async (params, client, task) => { + const file = typeof params.image === "string" ? await fileFromUrl(params.image) : params.image; + + const response = await client + .createImageVariation(file, params.n, params.size, params.response_format, params.user) + .then((res) => res.data); + + return response; + }, + init: (params) => { + let properties = []; + + if (params.n) { + properties.push({ + label: "Number of images", + text: params.n.toString(), + }); + } + + if (params.size) { + properties.push({ + label: "Size", + text: params.size, + }); + } + + if (params.response_format) { + properties.push({ + label: "Response format", + text: params.response_format, + }); + } + + if (typeof params.image === "string") { + properties.push({ + label: "Image URL", + text: params.image, + url: params.image, + }); + } + + return { + name: "Create image variation", + params, + icon: "openai", + properties, + }; + }, +}; + type CreateEmbeddingResponseData = Prettify< Awaited>["data"] >; @@ -383,7 +533,7 @@ export const createFile: AuthenticatedTask< let file: File; if (typeof params.file === "string") { - file = (await fileFromString(params.file, params.fileName ?? "file.txt")) as any; + file = await fileFromString(params.file, params.fileName ?? "file.txt"); } else { file = params.file; } @@ -441,10 +591,10 @@ export const createFineTuneFile: AuthenticatedTask< > = { onError: onTaskError, run: async (params, client) => { - const file = (await fileFromString( + const file = await fileFromString( params.examples.map((d) => JSON.stringify(d)).join("\n"), params.fileName - )) as any; + ); return client.createFile(file, "fine-tune").then((res) => res.data); }, diff --git a/packages/integration-kit/package.json b/packages/integration-kit/package.json index 57d31d176..99a173288 100644 --- a/packages/integration-kit/package.json +++ b/packages/integration-kit/package.json @@ -20,6 +20,7 @@ "devDependencies": { "@trigger.dev/tsconfig": "workspace:*", "@types/node": "18", + "@types/node-fetch": "2.6.x", "@types/uuid": "^9.0.0", "rimraf": "^3.0.2", "tsup": "^6.5.0", @@ -32,6 +33,7 @@ "typecheck": "tsup --dts-resolve --no-dts" }, "dependencies": { + "node-fetch": "2.6.x", "uuid": "^9.0.0" }, "engines": { diff --git a/packages/integration-kit/src/file.ts b/packages/integration-kit/src/file.ts index 2114cb4f1..f5ae9f6df 100644 --- a/packages/integration-kit/src/file.ts +++ b/packages/integration-kit/src/file.ts @@ -1,11 +1,20 @@ import fs, { promises } from "fs"; import path from "path"; import { v4 as uuidv4 } from "uuid"; +import fetch from "node-fetch"; -export async function fileFromString(contents: string, fileName: string) { +export async function fileFromString(contents: string | Buffer, fileName: string): Promise { const directory = path.join("tmp", uuidv4()); await promises.mkdir(directory, { recursive: true }); const filePath = path.join(directory, fileName); await promises.writeFile(filePath, contents); - return fs.createReadStream(filePath); + return fs.createReadStream(filePath) as unknown as File; +} + +export async function fileFromUrl(url: string) { + const response = await fetch(url); + const content = await response.buffer(); + const fileName = path.basename(url); + + return fileFromString(content, fileName); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index afe0e3bc0..fc4aab40c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -956,16 +956,20 @@ importers: specifiers: '@trigger.dev/tsconfig': workspace:* '@types/node': '18' + '@types/node-fetch': 2.6.x '@types/uuid': ^9.0.0 + node-fetch: 2.6.x rimraf: ^3.0.2 tsup: ^6.5.0 tsx: ^3.12.1 uuid: ^9.0.0 dependencies: + node-fetch: 2.6.12 uuid: 9.0.0 devDependencies: '@trigger.dev/tsconfig': link:../../config-packages/tsconfig '@types/node': 18.15.13 + '@types/node-fetch': 2.6.2 '@types/uuid': 9.0.0 rimraf: 3.0.2 tsup: 6.6.3 @@ -11081,7 +11085,7 @@ packages: /@types/node-fetch/2.6.2: resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} dependencies: - '@types/node': 20.3.2 + '@types/node': 20.4.5 form-data: 3.0.1 dev: true @@ -11105,10 +11109,6 @@ packages: /@types/node/18.17.1: resolution: {integrity: sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==} - /@types/node/20.3.2: - resolution: {integrity: sha512-vOBLVQeCQfIcF/2Y7eKFTqrMnizK5lRNQ7ykML/5RuwVXVWxYkgwS7xbt4B6fKCUPgbSL5FSsjHQpaGQP/dQmw==} - dev: true - /@types/node/20.4.2: resolution: {integrity: sha512-Dd0BYtWgnWJKwO1jkmTrzofjK2QXXcai0dmtzvIBhcA+RsG5h8R3xlyta0kGOZRNfL9GuRtb1knmPEhQrePCEw==}