From a5b49cb61ffda19bc2169616a39e9d5b2888e7de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:53:06 +0000 Subject: [PATCH 1/8] chore: Update version for release (#753) Co-authored-by: github-actions[bot] --- .changeset/gentle-brooms-sing.md | 12 ------------ .changeset/hip-bulldogs-serve.md | 6 ------ apps/proxy/CHANGELOG.md | 7 +++++++ apps/proxy/package.json | 2 +- integrations/airtable/CHANGELOG.md | 12 ++++++++++++ integrations/airtable/package.json | 6 +++--- integrations/github/CHANGELOG.md | 9 +++++++++ integrations/github/package.json | 6 +++--- integrations/linear/CHANGELOG.md | 9 +++++++++ integrations/linear/package.json | 6 +++--- integrations/openai/CHANGELOG.md | 9 +++++++++ integrations/openai/package.json | 6 +++--- integrations/plain/CHANGELOG.md | 9 +++++++++ integrations/plain/package.json | 6 +++--- integrations/replicate/CHANGELOG.md | 9 +++++++++ integrations/replicate/package.json | 6 +++--- integrations/resend/CHANGELOG.md | 9 +++++++++ integrations/resend/package.json | 6 +++--- integrations/sendgrid/CHANGELOG.md | 9 +++++++++ integrations/sendgrid/package.json | 6 +++--- integrations/shopify/CHANGELOG.md | 14 ++++++++++++++ integrations/shopify/package.json | 6 +++--- integrations/slack/CHANGELOG.md | 8 ++++++++ integrations/slack/package.json | 4 ++-- integrations/stripe/CHANGELOG.md | 9 +++++++++ integrations/stripe/package.json | 6 +++--- integrations/supabase/CHANGELOG.md | 9 +++++++++ integrations/supabase/package.json | 6 +++--- integrations/typeform/CHANGELOG.md | 9 +++++++++ integrations/typeform/package.json | 6 +++--- packages/astro/CHANGELOG.md | 8 ++++++++ packages/astro/package.json | 4 ++-- packages/cli/CHANGELOG.md | 10 ++++++++++ packages/cli/package.json | 2 +- packages/core-backend/CHANGELOG.md | 2 ++ packages/core-backend/package.json | 2 +- packages/core/CHANGELOG.md | 8 ++++++++ packages/core/package.json | 2 +- packages/eslint-plugin/CHANGELOG.md | 2 ++ packages/eslint-plugin/package.json | 2 +- packages/express/CHANGELOG.md | 8 ++++++++ packages/express/package.json | 6 +++--- packages/integration-kit/CHANGELOG.md | 10 ++++++++++ packages/integration-kit/package.json | 2 +- packages/nestjs/CHANGELOG.md | 8 ++++++++ packages/nestjs/package.json | 6 +++--- packages/nextjs/CHANGELOG.md | 8 ++++++++ packages/nextjs/package.json | 4 ++-- packages/react/CHANGELOG.md | 7 +++++++ packages/react/package.json | 4 ++-- packages/remix/CHANGELOG.md | 8 ++++++++ packages/remix/package.json | 4 ++-- packages/sveltekit/CHANGELOG.md | 8 ++++++++ packages/sveltekit/package.json | 4 ++-- packages/testing/CHANGELOG.md | 9 +++++++++ packages/testing/package.json | 2 +- packages/trigger-sdk/CHANGELOG.md | 12 ++++++++++++ packages/trigger-sdk/package.json | 6 +++--- 58 files changed, 303 insertions(+), 82 deletions(-) delete mode 100644 .changeset/gentle-brooms-sing.md delete mode 100644 .changeset/hip-bulldogs-serve.md create mode 100644 integrations/shopify/CHANGELOG.md diff --git a/.changeset/gentle-brooms-sing.md b/.changeset/gentle-brooms-sing.md deleted file mode 100644 index 1dbcf4a5b..000000000 --- a/.changeset/gentle-brooms-sing.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -"@trigger.dev/integration-kit": patch -"@trigger.dev/airtable": patch -"@trigger.dev/shopify": patch -"@trigger.dev/sdk": patch -"@trigger.dev/core": patch -"@trigger.dev/cli": patch ---- - -- Simplify `Webhook Triggers` and use the new HTTP Endpoints -- Add a `Key-Value Store` for use in and outside of Jobs -- Add a `@trigger.dev/shopify` package diff --git a/.changeset/hip-bulldogs-serve.md b/.changeset/hip-bulldogs-serve.md deleted file mode 100644 index f9a38c638..000000000 --- a/.changeset/hip-bulldogs-serve.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@trigger.dev/shopify": patch -"@trigger.dev/sdk": patch ---- - -Fix `@trigger.dev/shopify` imports, enhance docs, and suppress HTTP Endpoint warnings diff --git a/apps/proxy/CHANGELOG.md b/apps/proxy/CHANGELOG.md index fbaa92bc0..145c910f4 100644 --- a/apps/proxy/CHANGELOG.md +++ b/apps/proxy/CHANGELOG.md @@ -1,5 +1,12 @@ # proxy +## 0.0.2 + +### Patch Changes + +- Updated dependencies [067e19fe] + - @trigger.dev/core@2.2.8 + ## 0.0.1 ### Patch Changes diff --git a/apps/proxy/package.json b/apps/proxy/package.json index 7327db854..0014b60e0 100644 --- a/apps/proxy/package.json +++ b/apps/proxy/package.json @@ -1,6 +1,6 @@ { "name": "proxy", - "version": "0.0.1", + "version": "0.0.2", "private": true, "scripts": { "deploy": "wrangler deploy", diff --git a/integrations/airtable/CHANGELOG.md b/integrations/airtable/CHANGELOG.md index b72051075..882b50c5f 100644 --- a/integrations/airtable/CHANGELOG.md +++ b/integrations/airtable/CHANGELOG.md @@ -1,5 +1,17 @@ # @trigger.dev/airtable +## 2.2.8 + +### Patch Changes + +- 067e19fe: - Simplify `Webhook Triggers` and use the new HTTP Endpoints + - Add a `Key-Value Store` for use in and outside of Jobs + - Add a `@trigger.dev/shopify` package +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/airtable/package.json b/integrations/airtable/package.json index c79a70940..be3aed142 100644 --- a/integrations/airtable/package.json +++ b/integrations/airtable/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/airtable", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for airtable", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -26,8 +26,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "airtable": "^0.12.1", "zod": "3.22.3" }, diff --git a/integrations/github/CHANGELOG.md b/integrations/github/CHANGELOG.md index 3263671d8..d7ec1cc8f 100644 --- a/integrations/github/CHANGELOG.md +++ b/integrations/github/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/github +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/github/package.json b/integrations/github/package.json index 771359a3c..aa42716b8 100644 --- a/integrations/github/package.json +++ b/integrations/github/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/github", - "version": "2.2.7", + "version": "2.2.8", "description": "The official GitHub integration for Trigger.dev", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -29,8 +29,8 @@ "@octokit/request": "^6.2.5", "@octokit/request-error": "^4.0.1", "@octokit/webhooks": "^10.4.0", - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "octokit": "^2.0.14", "zod": "3.22.3" }, diff --git a/integrations/linear/CHANGELOG.md b/integrations/linear/CHANGELOG.md index 367e7000f..7ddd94a16 100644 --- a/integrations/linear/CHANGELOG.md +++ b/integrations/linear/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/linear +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/linear/package.json b/integrations/linear/package.json index 4a91f4fa7..6290e818c 100644 --- a/integrations/linear/package.json +++ b/integrations/linear/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/linear", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for @linear/sdk", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -27,8 +27,8 @@ }, "dependencies": { "@linear/sdk": "^8.0.0", - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "zod": "3.22.3" }, "engines": { diff --git a/integrations/openai/CHANGELOG.md b/integrations/openai/CHANGELOG.md index c2dcd1e9f..86206fced 100644 --- a/integrations/openai/CHANGELOG.md +++ b/integrations/openai/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/slack +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/openai/package.json b/integrations/openai/package.json index 382bbfd32..00a75a70c 100644 --- a/integrations/openai/package.json +++ b/integrations/openai/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/openai", - "version": "2.2.7", + "version": "2.2.8", "description": "The official OpenAI integration for Trigger.dev", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -31,8 +31,8 @@ }, "dependencies": { "openai": "^4.16.1", - "@trigger.dev/sdk": "workspace:^2.2.7", - "@trigger.dev/integration-kit": "workspace:^2.2.7" + "@trigger.dev/sdk": "workspace:^2.2.8", + "@trigger.dev/integration-kit": "workspace:^2.2.8" }, "engines": { "node": ">=18.0.0" diff --git a/integrations/plain/CHANGELOG.md b/integrations/plain/CHANGELOG.md index 62bad1104..45f9fdfef 100644 --- a/integrations/plain/CHANGELOG.md +++ b/integrations/plain/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/plain +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/plain/package.json b/integrations/plain/package.json index a915fec61..8ba089644 100644 --- a/integrations/plain/package.json +++ b/integrations/plain/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/plain", - "version": "2.2.7", + "version": "2.2.8", "description": "The official Plain.com integration for Trigger.dev", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -24,8 +24,8 @@ "build:tsup": "tsup" }, "dependencies": { - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "@team-plain/typescript-sdk": "^2.7.0" }, "engines": { diff --git a/integrations/replicate/CHANGELOG.md b/integrations/replicate/CHANGELOG.md index 1d59330e8..c1afc8210 100644 --- a/integrations/replicate/CHANGELOG.md +++ b/integrations/replicate/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/replicate +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/replicate/package.json b/integrations/replicate/package.json index 483e4eeb7..e54d27423 100644 --- a/integrations/replicate/package.json +++ b/integrations/replicate/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/replicate", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for replicate", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -26,8 +26,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "replicate": "^0.18.1", "zod": "3.22.3" }, diff --git a/integrations/resend/CHANGELOG.md b/integrations/resend/CHANGELOG.md index 19c6159e1..d577936cc 100644 --- a/integrations/resend/CHANGELOG.md +++ b/integrations/resend/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/resend +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/resend/package.json b/integrations/resend/package.json index 3ad2f20fe..61e378ac8 100644 --- a/integrations/resend/package.json +++ b/integrations/resend/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/resend", - "version": "2.2.7", + "version": "2.2.8", "description": "The official Resend.com integration for Trigger.dev", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -24,8 +24,8 @@ "build:tsup": "tsup" }, "dependencies": { - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "resend": "^2.0.0" }, "engines": { diff --git a/integrations/sendgrid/CHANGELOG.md b/integrations/sendgrid/CHANGELOG.md index b2d45e4d9..2f8cc7c35 100644 --- a/integrations/sendgrid/CHANGELOG.md +++ b/integrations/sendgrid/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/sendgrid +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/sendgrid/package.json b/integrations/sendgrid/package.json index 7b9336160..677f7f5d5 100644 --- a/integrations/sendgrid/package.json +++ b/integrations/sendgrid/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sendgrid", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for @sendgrid/mail", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -27,8 +27,8 @@ }, "dependencies": { "@sendgrid/mail": "^7.7.0", - "@trigger.dev/sdk": "workspace:^2.2.7", - "@trigger.dev/integration-kit": "workspace:^2.2.7" + "@trigger.dev/sdk": "workspace:^2.2.8", + "@trigger.dev/integration-kit": "workspace:^2.2.8" }, "engines": { "node": ">=16.8.0" diff --git a/integrations/shopify/CHANGELOG.md b/integrations/shopify/CHANGELOG.md new file mode 100644 index 000000000..690c2066b --- /dev/null +++ b/integrations/shopify/CHANGELOG.md @@ -0,0 +1,14 @@ +# @trigger.dev/shopify + +## 2.2.8 + +### Patch Changes + +- 067e19fe: - Simplify `Webhook Triggers` and use the new HTTP Endpoints + - Add a `Key-Value Store` for use in and outside of Jobs + - Add a `@trigger.dev/shopify` package +- 096151c0: Fix `@trigger.dev/shopify` imports, enhance docs, and suppress HTTP Endpoint warnings +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 diff --git a/integrations/shopify/package.json b/integrations/shopify/package.json index 0fef619e3..041f47d60 100644 --- a/integrations/shopify/package.json +++ b/integrations/shopify/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/shopify", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for @shopify/shopify-api", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -27,8 +27,8 @@ }, "dependencies": { "@shopify/shopify-api": "^8.0.2", - "@trigger.dev/sdk": "workspace:^2.2.6", - "@trigger.dev/integration-kit": "workspace:^2.2.6", + "@trigger.dev/sdk": "workspace:^2.2.8", + "@trigger.dev/integration-kit": "workspace:^2.2.8", "zod": "3.22.3" }, "engines": { diff --git a/integrations/slack/CHANGELOG.md b/integrations/slack/CHANGELOG.md index 3d58dbd69..93f75ddc3 100644 --- a/integrations/slack/CHANGELOG.md +++ b/integrations/slack/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/slack +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/slack/package.json b/integrations/slack/package.json index af438dfe1..5f112b200 100644 --- a/integrations/slack/package.json +++ b/integrations/slack/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/slack", - "version": "2.2.7", + "version": "2.2.8", "description": "The official Slack integration for Trigger.dev", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -25,7 +25,7 @@ }, "dependencies": { "@slack/web-api": "^6.8.1", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/sdk": "workspace:^2.2.8", "zod": "3.22.3" }, "engines": { diff --git a/integrations/stripe/CHANGELOG.md b/integrations/stripe/CHANGELOG.md index 58bdda01e..bf7b46f00 100644 --- a/integrations/stripe/CHANGELOG.md +++ b/integrations/stripe/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/stripe +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/stripe/package.json b/integrations/stripe/package.json index 1cb15f956..f86c28d4f 100644 --- a/integrations/stripe/package.json +++ b/integrations/stripe/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/stripe", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for stripe", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -26,8 +26,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "stripe": "^12.14.0", "zod": "3.22.3" }, diff --git a/integrations/supabase/CHANGELOG.md b/integrations/supabase/CHANGELOG.md index c428bc952..bf95c5a78 100644 --- a/integrations/supabase/CHANGELOG.md +++ b/integrations/supabase/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/supabase +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/supabase/package.json b/integrations/supabase/package.json index 3a7d9e07a..19d75d31b 100644 --- a/integrations/supabase/package.json +++ b/integrations/supabase/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/supabase", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev integration for @supabase/supabase-js", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -27,8 +27,8 @@ }, "dependencies": { "@supabase/supabase-js": "^2.26.0", - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "supabase-management-js": "^0.1.4", "zod": "3.22.3" }, diff --git a/integrations/typeform/CHANGELOG.md b/integrations/typeform/CHANGELOG.md index c4e798cdd..72b1346d3 100644 --- a/integrations/typeform/CHANGELOG.md +++ b/integrations/typeform/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/typeform +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/integration-kit@2.2.8 + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/integrations/typeform/package.json b/integrations/typeform/package.json index 709d7ffd4..44dfee1b5 100644 --- a/integrations/typeform/package.json +++ b/integrations/typeform/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/typeform", - "version": "2.2.7", + "version": "2.2.8", "description": "The official Typeform integration for Trigger.dev", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -25,8 +25,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@trigger.dev/integration-kit": "workspace:^2.2.7", - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/integration-kit": "workspace:^2.2.8", + "@trigger.dev/sdk": "workspace:^2.2.8", "@typeform/api-client": "^1.8.0", "zod": "3.22.3" }, diff --git a/packages/astro/CHANGELOG.md b/packages/astro/CHANGELOG.md index 69ca726d7..8bf6c2617 100644 --- a/packages/astro/CHANGELOG.md +++ b/packages/astro/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/astro +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/astro/package.json b/packages/astro/package.json index fae98a1c0..6e9bfb641 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -1,7 +1,7 @@ { "name": "@trigger.dev/astro", "description": "An Astro-native integration for Trigger.dev background jobs platform", - "version": "2.2.7", + "version": "2.2.8", "main": "./dist/index.js", "types": "./dist/index.d.ts", "files": [ @@ -20,7 +20,7 @@ "build:tsup": "tsup" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7" + "@trigger.dev/sdk": "workspace:^2.2.8" }, "devDependencies": { "astro": "^3.0.12", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 9134424ae..5847f1176 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,15 @@ # create-trigger +## 2.2.8 + +### Patch Changes + +- 067e19fe: - Simplify `Webhook Triggers` and use the new HTTP Endpoints + - Add a `Key-Value Store` for use in and outside of Jobs + - Add a `@trigger.dev/shopify` package +- Updated dependencies [067e19fe] + - @trigger.dev/core@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 7ab9bd26d..b73d9ec4d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/cli", - "version": "2.2.7", + "version": "2.2.8", "description": "The Trigger.dev CLI", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/core-backend/CHANGELOG.md b/packages/core-backend/CHANGELOG.md index 4c78822c6..af0d39420 100644 --- a/packages/core-backend/CHANGELOG.md +++ b/packages/core-backend/CHANGELOG.md @@ -1,5 +1,7 @@ # @trigger.dev/core-backend +## 2.2.8 + ## 2.2.7 ## 2.2.6 diff --git a/packages/core-backend/package.json b/packages/core-backend/package.json index 8f9302787..fe3322b32 100644 --- a/packages/core-backend/package.json +++ b/packages/core-backend/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/core-backend", - "version": "2.2.7", + "version": "2.2.8", "description": "Core code used across `@trigger.dev/sdk` and Trigger.dev server", "license": "MIT", "main": "./dist/index.js", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 41846a5bb..90a957b23 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,13 @@ # internal-platform +## 2.2.8 + +### Patch Changes + +- 067e19fe: - Simplify `Webhook Triggers` and use the new HTTP Endpoints + - Add a `Key-Value Store` for use in and outside of Jobs + - Add a `@trigger.dev/shopify` package + ## 2.2.7 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index e20676305..1a166b678 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/core", - "version": "2.2.7", + "version": "2.2.8", "description": "Core code used across the Trigger.dev SDK and platform", "license": "MIT", "main": "./dist/index.js", diff --git a/packages/eslint-plugin/CHANGELOG.md b/packages/eslint-plugin/CHANGELOG.md index 5b883a679..8be7b4876 100644 --- a/packages/eslint-plugin/CHANGELOG.md +++ b/packages/eslint-plugin/CHANGELOG.md @@ -1,5 +1,7 @@ # @trigger.dev/eslint-plugin +## 2.2.8 + ## 2.2.7 ## 2.2.6 diff --git a/packages/eslint-plugin/package.json b/packages/eslint-plugin/package.json index 460d363d6..138a68371 100644 --- a/packages/eslint-plugin/package.json +++ b/packages/eslint-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/eslint-plugin", - "version": "2.2.7", + "version": "2.2.8", "description": "ESLint plugin with trigger.dev best practices", "keywords": [ "eslint", diff --git a/packages/express/CHANGELOG.md b/packages/express/CHANGELOG.md index 626e620c5..13453e2a2 100644 --- a/packages/express/CHANGELOG.md +++ b/packages/express/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/express +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/express/package.json b/packages/express/package.json index 58d32327d..059f64460 100644 --- a/packages/express/package.json +++ b/packages/express/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/express", - "version": "2.2.7", + "version": "2.2.8", "description": "Official Express adapter for Trigger.dev", "license": "MIT", "main": "./dist/index.js", @@ -19,7 +19,7 @@ "./package.json": "./package.json" }, "devDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/sdk": "workspace:^2.2.8", "@trigger.dev/tsconfig": "workspace:*", "@types/debug": "^4.1.7", "@types/express": "^4.17.13", @@ -33,7 +33,7 @@ "build:tsup": "tsup" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7" + "@trigger.dev/sdk": "workspace:^2.2.8" }, "dependencies": { "debug": "^4.3.4", diff --git a/packages/integration-kit/CHANGELOG.md b/packages/integration-kit/CHANGELOG.md index ef33298ef..9558a0b25 100644 --- a/packages/integration-kit/CHANGELOG.md +++ b/packages/integration-kit/CHANGELOG.md @@ -1,5 +1,15 @@ # @trigger.dev/integration-kit +## 2.2.8 + +### Patch Changes + +- 067e19fe: - Simplify `Webhook Triggers` and use the new HTTP Endpoints + - Add a `Key-Value Store` for use in and outside of Jobs + - Add a `@trigger.dev/shopify` package +- Updated dependencies [067e19fe] + - @trigger.dev/core@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/integration-kit/package.json b/packages/integration-kit/package.json index 396f97673..fc8c16e4e 100644 --- a/packages/integration-kit/package.json +++ b/packages/integration-kit/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/integration-kit", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev Integration Kit has helpers to make creating integrations easier", "license": "MIT", "main": "./dist/index.js", diff --git a/packages/nestjs/CHANGELOG.md b/packages/nestjs/CHANGELOG.md index 11d28ae05..e718b87e0 100644 --- a/packages/nestjs/CHANGELOG.md +++ b/packages/nestjs/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/nestjs +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/nestjs/package.json b/packages/nestjs/package.json index 137a0f169..793921e99 100644 --- a/packages/nestjs/package.json +++ b/packages/nestjs/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/nestjs", - "version": "2.2.7", + "version": "2.2.8", "description": "Official NestJS adapter for Trigger.dev", "license": "MIT", "main": "./dist/index.js", @@ -19,7 +19,7 @@ "./package.json": "./package.json" }, "devDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/sdk": "workspace:^2.2.8", "@trigger.dev/tsconfig": "workspace:*", "@types/debug": "^4.1.7", "@types/express": "^4.17.13", @@ -35,7 +35,7 @@ }, "peerDependencies": { "@nestjs/common": ">=10.0.0", - "@trigger.dev/sdk": "workspace:^2.2.7" + "@trigger.dev/sdk": "workspace:^2.2.8" }, "dependencies": { "@nestjs/common": "^10.2.4", diff --git a/packages/nextjs/CHANGELOG.md b/packages/nextjs/CHANGELOG.md index 4866fafbc..c52e67e6a 100644 --- a/packages/nextjs/CHANGELOG.md +++ b/packages/nextjs/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/nextjs +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/nextjs/package.json b/packages/nextjs/package.json index 31441a38a..371130a16 100644 --- a/packages/nextjs/package.json +++ b/packages/nextjs/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/nextjs", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev Next.js integration", "license": "MIT", "main": "./dist/index.js", @@ -36,7 +36,7 @@ "build:tsup": "tsup" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/sdk": "workspace:^2.2.8", "next": ">=12.0.0" }, "dependencies": { diff --git a/packages/react/CHANGELOG.md b/packages/react/CHANGELOG.md index 54bd4fa3f..3700742a3 100644 --- a/packages/react/CHANGELOG.md +++ b/packages/react/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/react +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] + - @trigger.dev/core@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/react/package.json b/packages/react/package.json index b037874da..0b3303721 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/react", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev React SDK", "license": "MIT", "types": "dist/index.d.ts", @@ -27,7 +27,7 @@ }, "dependencies": { "@tanstack/react-query": "5.0.0-beta.2", - "@trigger.dev/core": "workspace:^2.2.7", + "@trigger.dev/core": "workspace:^2.2.8", "debug": "^4.3.4", "zod": "3.22.3" }, diff --git a/packages/remix/CHANGELOG.md b/packages/remix/CHANGELOG.md index ff92fb82d..bc7d903d3 100644 --- a/packages/remix/CHANGELOG.md +++ b/packages/remix/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/remix +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/remix/package.json b/packages/remix/package.json index ff1a03185..b60390b93 100644 --- a/packages/remix/package.json +++ b/packages/remix/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/remix", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev Remix integration", "license": "MIT", "main": "./dist/index.js", @@ -34,7 +34,7 @@ "build:tsup": "tsup" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7", + "@trigger.dev/sdk": "workspace:^2.2.8", "@remix-run/server-runtime": ">1.19.0" }, "dependencies": { diff --git a/packages/sveltekit/CHANGELOG.md b/packages/sveltekit/CHANGELOG.md index def9dd7d4..de2f7abc7 100644 --- a/packages/sveltekit/CHANGELOG.md +++ b/packages/sveltekit/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/sveltekit +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/sveltekit/package.json b/packages/sveltekit/package.json index 0600e2a52..b61a4b664 100644 --- a/packages/sveltekit/package.json +++ b/packages/sveltekit/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sveltekit", - "version": "2.2.7", + "version": "2.2.8", "description": "Trigger.dev svelteKit integration", "license": "MIT", "main": "./dist/index.js", @@ -33,7 +33,7 @@ "build:tsup": "tsup" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^2.2.7" + "@trigger.dev/sdk": "workspace:^2.2.8" }, "dependencies": { "debug": "^4.3.4" diff --git a/packages/testing/CHANGELOG.md b/packages/testing/CHANGELOG.md index 9a22b6bb4..b5b8ebbaf 100644 --- a/packages/testing/CHANGELOG.md +++ b/packages/testing/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/testing +## 2.2.8 + +### Patch Changes + +- Updated dependencies [067e19fe] +- Updated dependencies [096151c0] + - @trigger.dev/sdk@2.2.8 + - @trigger.dev/core@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/testing/package.json b/packages/testing/package.json index 8f5653f25..92c7b9f1d 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -1,7 +1,7 @@ { "name": "@trigger.dev/testing", "description": "A collection of useful tools to write tests for Trigger.dev.", - "version": "2.2.7", + "version": "2.2.8", "license": "MIT", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/trigger-sdk/CHANGELOG.md b/packages/trigger-sdk/CHANGELOG.md index 6f198ae82..cd3c7f296 100644 --- a/packages/trigger-sdk/CHANGELOG.md +++ b/packages/trigger-sdk/CHANGELOG.md @@ -1,5 +1,17 @@ # @trigger.dev/sdk +## 2.2.8 + +### Patch Changes + +- 067e19fe: - Simplify `Webhook Triggers` and use the new HTTP Endpoints + - Add a `Key-Value Store` for use in and outside of Jobs + - Add a `@trigger.dev/shopify` package +- 096151c0: Fix `@trigger.dev/shopify` imports, enhance docs, and suppress HTTP Endpoint warnings +- Updated dependencies [067e19fe] + - @trigger.dev/core@2.2.8 + - @trigger.dev/core-backend@2.2.8 + ## 2.2.7 ### Patch Changes diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index 652f29e50..45a0a1010 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sdk", - "version": "2.2.7", + "version": "2.2.8", "description": "trigger.dev Node.JS SDK", "license": "MIT", "main": "./dist/index.js", @@ -25,8 +25,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@trigger.dev/core": "workspace:^2.2.7", - "@trigger.dev/core-backend": "workspace:^2.2.7", + "@trigger.dev/core": "workspace:^2.2.8", + "@trigger.dev/core-backend": "workspace:^2.2.8", "chalk": "^5.2.0", "colorette": "^2.0.20", "cronstrue": "^2.21.0", From dbc2e3f713f9ca3149fe4225679abdf3194b79dd Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 28 Nov 2023 14:53:37 +0000 Subject: [PATCH 2/8] Refreshed pnpm lock --- pnpm-lock.yaml | 70 +++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca73d2418..a736f021a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -413,8 +413,8 @@ importers: integrations/airtable: specifiers: - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': 16.x airtable: ^0.12.1 @@ -441,8 +441,8 @@ importers: '@octokit/types': ^9.2.3 '@octokit/webhooks': ^10.4.0 '@octokit/webhooks-types': ^6.10.0 - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': '18' octokit: ^2.0.14 @@ -468,8 +468,8 @@ importers: integrations/linear: specifiers: '@linear/sdk': ^8.0.0 - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': 16.x rimraf: ^3.0.2 @@ -490,8 +490,8 @@ importers: integrations/openai: specifiers: - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/jest': ^29.5.3 '@types/node': '18' @@ -518,8 +518,8 @@ importers: integrations/plain: specifiers: '@team-plain/typescript-sdk': ^2.7.0 - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': '18' rimraf: ^3.0.2 @@ -536,8 +536,8 @@ importers: integrations/replicate: specifiers: - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': 16.x replicate: ^0.18.1 @@ -559,8 +559,8 @@ importers: integrations/resend: specifiers: - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': '18' resend: ^2.0.0 @@ -579,8 +579,8 @@ importers: integrations/sendgrid: specifiers: '@sendgrid/mail': ^7.7.0 - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': 16.x rimraf: ^3.0.2 @@ -600,8 +600,8 @@ importers: integrations/shopify: specifiers: '@shopify/shopify-api': ^8.0.2 - '@trigger.dev/integration-kit': workspace:^2.2.6 - '@trigger.dev/sdk': workspace:^2.2.6 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': 16.x rimraf: ^3.0.2 @@ -623,7 +623,7 @@ importers: integrations/slack: specifiers: '@slack/web-api': ^6.8.1 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': '18' rimraf: ^3.0.2 @@ -641,8 +641,8 @@ importers: integrations/stripe: specifiers: - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@types/node': 16.x rimraf: ^3.0.2 stripe: ^12.14.0 @@ -665,8 +665,8 @@ importers: integrations/supabase: specifiers: '@supabase/supabase-js': ^2.26.0 - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/node': 18.x rimraf: ^3.0.2 @@ -689,8 +689,8 @@ importers: integrations/typeform: specifiers: - '@trigger.dev/integration-kit': workspace:^2.2.7 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/integration-kit': workspace:^2.2.8 + '@trigger.dev/sdk': workspace:^2.2.8 '@typeform/api-client': ^1.8.0 '@types/node': 16.x rimraf: ^3.0.2 @@ -944,7 +944,7 @@ importers: packages/express: specifiers: - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/debug': ^4.1.7 '@types/express': ^4.17.13 @@ -992,7 +992,7 @@ importers: specifiers: '@nestjs/common': ^10.2.4 '@remix-run/web-fetch': ^4.3.5 - '@trigger.dev/sdk': workspace:^2.2.7 + '@trigger.dev/sdk': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/debug': ^4.1.7 '@types/express': ^4.17.13 @@ -1045,7 +1045,7 @@ importers: packages/react: specifiers: '@tanstack/react-query': 5.0.0-beta.2 - '@trigger.dev/core': workspace:^2.2.7 + '@trigger.dev/core': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/debug': ^4.1.7 '@types/react': 18.2.17 @@ -1140,8 +1140,8 @@ importers: packages/trigger-sdk: specifiers: - '@trigger.dev/core': workspace:^2.2.7 - '@trigger.dev/core-backend': workspace:^2.2.7 + '@trigger.dev/core': workspace:^2.2.8 + '@trigger.dev/core-backend': workspace:^2.2.8 '@trigger.dev/tsconfig': workspace:* '@types/debug': ^4.1.7 '@types/node': '18' @@ -15413,7 +15413,6 @@ packages: typescript: 5.1.6 transitivePeerDependencies: - supports-color - dev: false /@typescript-eslint/parser/5.59.6_mhvizudgavq6jzvqzwcqcv4nnm: resolution: {integrity: sha512-7pCa6al03Pv1yf/dUg/s1pXz/yGMUBAw5EeWqNTFiSueKvRNonze3hma3lhdsOrQcaOXhbk5gKu2Fludiho9VA==} @@ -15433,6 +15432,7 @@ packages: typescript: 4.9.5 transitivePeerDependencies: - supports-color + dev: true /@typescript-eslint/parser/5.59.6_ox3na7ge7wjdarbyztnclevxam: resolution: {integrity: sha512-7pCa6al03Pv1yf/dUg/s1pXz/yGMUBAw5EeWqNTFiSueKvRNonze3hma3lhdsOrQcaOXhbk5gKu2Fludiho9VA==} @@ -15618,6 +15618,7 @@ packages: typescript: 4.9.5 transitivePeerDependencies: - supports-color + dev: true /@typescript-eslint/typescript-estree/5.59.6_typescript@5.0.4: resolution: {integrity: sha512-vW6JP3lMAs/Tq4KjdI/RiHaaJSO7IUsbkz17it/Rl9Q+WkQ77EOuOnlbaU8kKfVIOJxMhnRiBG+olE7f3M16DA==} @@ -15659,7 +15660,6 @@ packages: typescript: 5.1.6 transitivePeerDependencies: - supports-color - dev: false /@typescript-eslint/typescript-estree/5.59.6_typescript@5.2.2: resolution: {integrity: sha512-vW6JP3lMAs/Tq4KjdI/RiHaaJSO7IUsbkz17it/Rl9Q+WkQ77EOuOnlbaU8kKfVIOJxMhnRiBG+olE7f3M16DA==} @@ -20255,7 +20255,7 @@ packages: eslint-import-resolver-webpack: optional: true dependencies: - '@typescript-eslint/parser': 5.59.6_mhvizudgavq6jzvqzwcqcv4nnm + '@typescript-eslint/parser': 5.59.6_ko3fmmbeyij36muomfgt2u76xu debug: 3.2.7 eslint: 8.45.0 eslint-import-resolver-node: 0.3.7 @@ -20402,7 +20402,7 @@ packages: '@typescript-eslint/parser': optional: true dependencies: - '@typescript-eslint/parser': 5.59.6_mhvizudgavq6jzvqzwcqcv4nnm + '@typescript-eslint/parser': 5.59.6_ko3fmmbeyij36muomfgt2u76xu array-includes: 3.1.6 array.prototype.flat: 1.3.1 array.prototype.flatmap: 1.3.1 @@ -32420,6 +32420,7 @@ packages: dependencies: tslib: 1.14.1 typescript: 4.9.5 + dev: true /tsutils/3.21.0_typescript@5.0.4: resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} @@ -32439,7 +32440,6 @@ packages: dependencies: tslib: 1.14.1 typescript: 5.1.6 - dev: false /tsutils/3.21.0_typescript@5.2.2: resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} From caf203c084bc6dd22038890aad73de4b931467a5 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 28 Nov 2023 15:04:24 +0000 Subject: [PATCH 3/8] Made webhook index stats optional --- .../app/services/endpoints/performEndpointIndexService.ts | 2 +- packages/core/src/schemas/api.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/services/endpoints/performEndpointIndexService.ts b/apps/webapp/app/services/endpoints/performEndpointIndexService.ts index 6c31280d7..c3f91a1fe 100644 --- a/apps/webapp/app/services/endpoints/performEndpointIndexService.ts +++ b/apps/webapp/app/services/endpoints/performEndpointIndexService.ts @@ -324,7 +324,7 @@ export class PerformEndpointIndexService { for (const webhook of webhooks) { try { await this.#registerWebhookService.call(endpoint, webhook); - indexStats.webhooks++; + indexStats.webhooks = indexStats.webhooks ?? 0 + 1; } catch (error) { logger.error("Failed to register webhook", { endpointId: endpoint.id, diff --git a/packages/core/src/schemas/api.ts b/packages/core/src/schemas/api.ts index fddbb67a2..c09410b33 100644 --- a/packages/core/src/schemas/api.ts +++ b/packages/core/src/schemas/api.ts @@ -398,7 +398,7 @@ export type EndpointIndexError = z.infer; const IndexEndpointStatsSchema = z.object({ jobs: z.number(), sources: z.number(), - webhooks: z.number(), + webhooks: z.number().optional(), dynamicTriggers: z.number(), dynamicSchedules: z.number(), disabledJobs: z.number().default(0), From 6ebd435e811f0911e4b014e7447f82acce016cad Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 28 Nov 2023 16:21:06 +0000 Subject: [PATCH 4/8] Feature: Run execution concurrency limits (#750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP execution concurrency controls implemented via Redis - Split up resuming a run and executing a run - Added some new statuses to better show what is going on in a run - Removed preprocessing runs * WIP * Convert to using ZSETs and adding env vars * Removed unused import * Improve run number generation using advistory locks, and only on start * More execution concurrency stuff * Add support for job concurrency limits and concurrency limit groups * Create wild-swans-battle.md * Increase slots refresh timeout to 10s * Try to fix Redis connection issues * Donโ€™t be so strict about the APP_ENV * Add the blank tls option to the normal redis client as well * Add docs --- .changeset/wild-swans-battle.md | 6 + .../webapp/app/components/JobsStatusTable.tsx | 23 +- .../app/components/navigation/SideMenu.tsx | 7 + .../app/components/primitives/Buttons.tsx | 7 +- .../webapp/app/components/run/RunOverview.tsx | 52 +-- .../app/components/runs/RunStatuses.tsx | 64 +-- apps/webapp/app/components/runs/RunsTable.tsx | 31 +- apps/webapp/app/env.server.ts | 20 +- apps/webapp/app/models/jobRun.server.ts | 45 ++ .../app/models/jobRunExecution.server.ts | 47 -- apps/webapp/app/platform/zodWorker.server.ts | 28 +- .../app/presenters/JobPresenter.server.ts | 9 + .../app/presenters/RunListPresenter.server.ts | 37 +- .../app/presenters/RunPresenter.server.ts | 7 + .../route.tsx | 9 + .../route.tsx | 14 +- .../ListPagination.tsx | 24 +- .../route.tsx | 2 +- .../route.tsx | 4 +- .../route.tsx | 4 +- .../route.tsx | 88 ++++ .../webapp/app/routes/_app.orgs.new/route.tsx | 1 + .../executions/createExecutionEvent.server.ts | 7 +- .../app/services/jobs/registerJob.server.ts | 116 +++-- .../runExecutionRateLimiter.server.ts | 406 ++++++++++++++++ .../app/services/runs/cancelRun.server.ts | 7 +- .../app/services/runs/continueRun.server.ts | 7 +- .../app/services/runs/createRun.server.ts | 21 - .../runs/performRunExecutionV3.server.ts | 437 ++++++------------ .../app/services/runs/resumeRun.server.ts | 158 +++++++ .../app/services/runs/startRun.server.ts | 49 +- .../app/services/tasks/resumeTask.server.ts | 9 +- .../app/services/tasks/runTask.server.ts | 4 +- apps/webapp/app/services/worker.server.ts | 18 +- apps/webapp/app/utils.ts | 15 +- apps/webapp/app/utils/pathBuilder.ts | 4 + apps/webapp/package.json | 1 + docker/dev-compose.yml | 19 + docker/docker-compose.yml | 19 + docs/_snippets/jobs/options.mdx | 3 + docs/documentation/concepts/limits.mdx | 47 +- docs/documentation/concepts/runs.mdx | 48 +- docs/mint.json | 50 +- .../instancemethods/concurrency-limit.mdx | 47 ++ packages/core/src/schemas/api.ts | 7 +- packages/core/src/schemas/runs.ts | 3 + .../migration.sql | 11 + .../migration.sql | 2 + .../migration.sql | 7 + .../migration.sql | 10 + .../migration.sql | 30 ++ .../migration.sql | 17 + .../migration.sql | 11 + .../migration.sql | 4 + packages/database/prisma/schema.prisma | 40 +- packages/trigger-sdk/src/concurrencyLimit.ts | 16 + packages/trigger-sdk/src/job.ts | 20 +- packages/trigger-sdk/src/triggerClient.ts | 11 + perf/src/index.ts | 33 +- perf/src/trigger.ts | 116 ++++- perf/tsconfig.json | 2 + pnpm-lock.yaml | 57 +++ 62 files changed, 1786 insertions(+), 632 deletions(-) create mode 100644 .changeset/wild-swans-battle.md create mode 100644 apps/webapp/app/models/jobRun.server.ts delete mode 100644 apps/webapp/app/models/jobRunExecution.server.ts create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.runs/route.tsx create mode 100644 apps/webapp/app/services/runExecutionRateLimiter.server.ts create mode 100644 apps/webapp/app/services/runs/resumeRun.server.ts create mode 100644 docs/sdk/triggerclient/instancemethods/concurrency-limit.mdx create mode 100644 packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql create mode 100644 packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql create mode 100644 packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql create mode 100644 packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql create mode 100644 packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql create mode 100644 packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql create mode 100644 packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql create mode 100644 packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql create mode 100644 packages/trigger-sdk/src/concurrencyLimit.ts diff --git a/.changeset/wild-swans-battle.md b/.changeset/wild-swans-battle.md new file mode 100644 index 000000000..b1bf3ccda --- /dev/null +++ b/.changeset/wild-swans-battle.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/sdk": patch +"@trigger.dev/core": patch +--- + +Feature: Run execution concurrency limits diff --git a/apps/webapp/app/components/JobsStatusTable.tsx b/apps/webapp/app/components/JobsStatusTable.tsx index dbf66e2ab..88c0a8ff1 100644 --- a/apps/webapp/app/components/JobsStatusTable.tsx +++ b/apps/webapp/app/components/JobsStatusTable.tsx @@ -16,19 +16,23 @@ export type JobEnvironment = { lastRun?: Date; version: string; enabled: boolean; + concurrencyLimit?: number | null; + concurrencyLimitGroup?: { name: string; concurrencyLimit: number } | null; }; type JobStatusTableProps = { environments: JobEnvironment[]; + displayStyle?: "short" | "long"; }; -export function JobStatusTable({ environments }: JobStatusTableProps) { +export function JobStatusTable({ environments, displayStyle = "short" }: JobStatusTableProps) { return ( Env Last Run + {displayStyle === "long" && Concurrency} Version Status @@ -42,6 +46,23 @@ export function JobStatusTable({ environments }: JobStatusTableProps) { {environment.lastRun ? : "Never Run"} + {displayStyle === "long" && ( + + {environment.concurrencyLimitGroup ? ( + + {environment.concurrencyLimitGroup.name} + + ({environment.concurrencyLimitGroup.concurrencyLimit}) + + + ) : typeof environment.concurrencyLimit === "number" ? ( + {environment.concurrencyLimit} + ) : ( + Not specified + )} + + )} + {environment.version} diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 75b7a2ada..3591f4c34 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -26,6 +26,7 @@ import { projectEnvironmentsPath, projectHttpEndpointsPath, projectPath, + projectRunsPath, projectSetupPath, projectTriggersPath, } from "~/utils/pathBuilder"; @@ -120,6 +121,12 @@ export function SideMenu({ user, project, organization, organizations }: SideMen to={projectPath(organization, project)} data-action="jobs" /> + ( } ); -type LinkPropsType = Pick & React.ComponentProps; -export const LinkButton = ({ to, ...props }: LinkPropsType) => { +type LinkPropsType = Pick & + React.ComponentProps; +export const LinkButton = ({ to, onClick, ...props }: LinkPropsType) => { const innerRef = useRef(null); if (props.shortcut) { useShortcutKeys({ @@ -324,6 +325,7 @@ export const LinkButton = ({ to, ...props }: LinkPropsType) => { href={to.toString()} ref={innerRef} className={cn("group outline-none", props.fullWidth ? "w-full" : "")} + onClick={onClick} > @@ -334,6 +336,7 @@ export const LinkButton = ({ to, ...props }: LinkPropsType) => { to={to} ref={innerRef} className={cn("group outline-none", props.fullWidth ? "w-full" : "")} + onClick={onClick} > diff --git a/apps/webapp/app/components/run/RunOverview.tsx b/apps/webapp/app/components/run/RunOverview.tsx index 96f15f53a..bfb4ab4dc 100644 --- a/apps/webapp/app/components/run/RunOverview.tsx +++ b/apps/webapp/app/components/run/RunOverview.tsx @@ -10,13 +10,14 @@ import { useNavigate, useNavigation, } from "@remix-run/react"; -import { JobRunStatus, RuntimeEnvironmentType } from "@trigger.dev/database"; +import { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useMemo } from "react"; import { usePathName } from "~/hooks/usePathName"; +import type { RunBasicStatus } from "~/models/jobRun.server"; import { ViewRun } from "~/presenters/RunPresenter.server"; import { cancelSchema } from "~/routes/resources.runs.$runId.cancel"; import { schema } from "~/routes/resources.runs.$runId.rerun"; -import { formatDuration } from "~/utils"; +import { formatDuration, formatDurationMilliseconds } from "~/utils"; import { cn } from "~/utils/cn"; import { runCompletedPath, runTaskPath, runTriggerPath } from "~/utils/pathBuilder"; import { CodeBlock } from "../code/CodeBlock"; @@ -38,14 +39,7 @@ import { } from "../primitives/PageHeader"; import { Paragraph } from "../primitives/Paragraph"; import { Popover, PopoverContent, PopoverTrigger } from "../primitives/Popover"; -import { - RunBasicStatus, - RunStatusIcon, - RunStatusLabel, - hasFinished, - runBasicStatus, - runStatusTitle, -} from "../runs/RunStatuses"; +import { RunStatusIcon, RunStatusLabel, runStatusTitle } from "../runs/RunStatuses"; import { RunPanel, RunPanelBody, @@ -95,8 +89,6 @@ export function RunOverview({ run, trigger, showRerun, paths }: RunOverviewProps } }, [pathName]); - const basicStatus = runBasicStatus(run.status); - return ( @@ -106,7 +98,9 @@ export function RunOverview({ run, trigger, showRerun, paths }: RunOverviewProps to: paths.back, text: "Runs", }} - title={`Run #${run.number}`} + title={ + typeof run.number === "number" ? `Run #${run.number}` : `Run ${run.id.slice(0, 8)}` + } /> {run.isTest && ( @@ -115,15 +109,15 @@ export function RunOverview({ run, trigger, showRerun, paths }: RunOverviewProps Test run )} - {showRerun && hasFinished(run.status) && ( + {showRerun && run.isFinished && ( )} - {!hasFinished(run.status) && } + {!run.isFinished && } @@ -146,7 +140,17 @@ export function RunOverview({ run, trigger, showRerun, paths }: RunOverviewProps + + @@ -211,10 +215,10 @@ export function RunOverview({ run, trigger, showRerun, paths }: RunOverviewProps ); }) ) : ( - + )} - {(basicStatus === "COMPLETED" || basicStatus === "FAILED") && ( + {(run.basicStatus === "COMPLETED" || run.basicStatus === "FAILED") && (
Run Summary There were no tasks for this run.; diff --git a/apps/webapp/app/components/runs/RunStatuses.tsx b/apps/webapp/app/components/runs/RunStatuses.tsx index 828fe50b4..f993e963b 100644 --- a/apps/webapp/app/components/runs/RunStatuses.tsx +++ b/apps/webapp/app/components/runs/RunStatuses.tsx @@ -3,6 +3,7 @@ import { CheckCircleIcon, ClockIcon, ExclamationTriangleIcon, + PauseCircleIcon, WrenchIcon, XCircleIcon, } from "@heroicons/react/24/solid"; @@ -10,18 +11,6 @@ import type { JobRunStatus } from "@trigger.dev/database"; import { cn } from "~/utils/cn"; import { Spinner } from "../primitives/Spinner"; -export function hasFinished(status: JobRunStatus): boolean { - return ( - status === "SUCCESS" || - status === "FAILURE" || - status === "ABORTED" || - status === "TIMED_OUT" || - status === "CANCELED" || - status === "UNRESOLVED_AUTH" || - status === "INVALID_PAYLOAD" - ); -} - export function RunStatus({ status }: { status: JobRunStatus }) { return ( @@ -40,49 +29,26 @@ export function RunStatusIcon({ status, className }: { status: JobRunStatus; cla case "SUCCESS": return ; case "PENDING": + case "WAITING_TO_CONTINUE": return ; case "QUEUED": - return ; + case "WAITING_TO_EXECUTE": + return ; + case "PREPROCESSING": case "STARTED": + case "EXECUTING": return ; - case "FAILURE": - return ; case "TIMED_OUT": return ; case "UNRESOLVED_AUTH": + case "FAILURE": + case "ABORTED": case "INVALID_PAYLOAD": return ; case "WAITING_ON_CONNECTIONS": return ; - case "ABORTED": - return ; - case "PREPROCESSING": - return ; case "CANCELED": return ; - } -} - -export type RunBasicStatus = "WAITING" | "PENDING" | "RUNNING" | "COMPLETED" | "FAILED"; - -export function runBasicStatus(status: JobRunStatus): RunBasicStatus { - switch (status) { - case "WAITING_ON_CONNECTIONS": - case "QUEUED": - case "PREPROCESSING": - case "PENDING": - return "PENDING"; - case "STARTED": - return "RUNNING"; - case "FAILURE": - case "TIMED_OUT": - case "UNRESOLVED_AUTH": - case "CANCELED": - case "ABORTED": - case "INVALID_PAYLOAD": - return "FAILED"; - case "SUCCESS": - return "COMPLETED"; default: { const _exhaustiveCheck: never = status; throw new Error(`Non-exhaustive match for value: ${status}`); @@ -99,7 +65,12 @@ export function runStatusTitle(status: JobRunStatus): string { case "STARTED": return "In progress"; case "QUEUED": + case "WAITING_TO_EXECUTE": return "Queued"; + case "EXECUTING": + return "Executing"; + case "WAITING_TO_CONTINUE": + return "Waiting"; case "FAILURE": return "Failed"; case "TIMED_OUT": @@ -130,9 +101,12 @@ export function runStatusClassNameColor(status: JobRunStatus): string { case "PENDING": return "text-slate-500"; case "STARTED": + case "EXECUTING": + case "WAITING_TO_CONTINUE": + case "WAITING_TO_EXECUTE": return "text-blue-500"; case "QUEUED": - return "text-amber-300"; + return "text-slate-500"; case "FAILURE": case "UNRESOLVED_AUTH": case "INVALID_PAYLOAD": @@ -147,5 +121,9 @@ export function runStatusClassNameColor(status: JobRunStatus): string { return "text-blue-500"; case "CANCELED": return "text-slate-500"; + default: { + const _exhaustiveCheck: never = status; + throw new Error(`Non-exhaustive match for value: ${status}`); + } } } diff --git a/apps/webapp/app/components/runs/RunsTable.tsx b/apps/webapp/app/components/runs/RunsTable.tsx index 879bf3505..0cb184c89 100644 --- a/apps/webapp/app/components/runs/RunsTable.tsx +++ b/apps/webapp/app/components/runs/RunsTable.tsx @@ -1,7 +1,7 @@ import { StopIcon } from "@heroicons/react/24/outline"; import { CheckIcon } from "@heroicons/react/24/solid"; import { JobRunStatus, RuntimeEnvironmentType } from "@trigger.dev/database"; -import { formatDuration } from "~/utils"; +import { formatDuration, formatDurationMilliseconds } from "~/utils"; import { EnvironmentLabel } from "../environments/EnvironmentLabel"; import { DateTime } from "../primitives/DateTime"; import { Paragraph } from "../primitives/Paragraph"; @@ -20,14 +20,16 @@ import { RunStatus } from "./RunStatuses"; type RunTableItem = { id: string; - number: number; + number: number | null; environment: { type: RuntimeEnvironmentType; }; + job: { title: string; slug: string }; status: JobRunStatus; startedAt: Date | null; completedAt: Date | null; createdAt: Date | null; + executionDuration: number; version: string; isTest: boolean; }; @@ -35,6 +37,7 @@ type RunTableItem = { type RunsTableProps = { total: number; hasFilters: boolean; + showJob?: boolean; runs: RunTableItem[]; isLoading?: boolean; runsParentPath: string; @@ -45,6 +48,7 @@ export function RunsTable({ hasFilters, runs, isLoading = false, + showJob = false, runsParentPath, }: RunsTableProps) { return ( @@ -52,10 +56,12 @@ export function RunsTable({ Run + {showJob && Job} Env Status Started Duration + Exec Time Test Version Created at @@ -66,19 +72,24 @@ export function RunsTable({ {total === 0 && !hasFilters ? ( - - + + ) : runs.length === 0 ? ( - + ) : ( runs.map((run) => { - const path = `${runsParentPath}/${run.id}/trigger`; + const path = showJob + ? `${runsParentPath}/jobs/${run.job.slug}/runs/${run.id}/trigger` + : `${runsParentPath}/${run.id}/trigger`; return ( - #{run.number} + + {typeof run.number === "number" ? `#${run.number}` : "-"} + + {showJob && {run.job.slug}} @@ -93,6 +104,11 @@ export function RunsTable({ style: "short", })} + + {formatDurationMilliseconds(run.executionDuration, { + style: "short", + })} + {run.isTest ? ( @@ -121,6 +137,7 @@ export function RunsTable({
); } + function NoRuns({ title }: { title: string }) { return (
diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index e3f6d62ec..3e3b7ca89 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -18,14 +18,7 @@ const EnvironmentSchema = z.object({ REMIX_APP_PORT: z.string().optional(), LOGIN_ORIGIN: z.string().default("http://localhost:3030"), APP_ORIGIN: z.string().default("http://localhost:3030"), - APP_ENV: z - .union([ - z.literal("development"), - z.literal("production"), - z.literal("test"), - z.literal("staging"), - ]) - .default(process.env.NODE_ENV), + APP_ENV: z.string().default(process.env.NODE_ENV), SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"), POSTHOG_PROJECT_KEY: z.string().optional(), TELEMETRY_TRIGGER_API_KEY: z.string().optional(), @@ -59,6 +52,17 @@ const EnvironmentSchema = z.object({ AWS_SQS_QUEUE_URL: z.string().optional(), AWS_SQS_BATCH_SIZE: z.coerce.number().int().optional().default(10), DISABLE_SSE: z.string().optional(), + + // Redis options + REDIS_HOST: z.string().optional(), + REDIS_READER_HOST: z.string().optional(), + REDIS_READER_PORT: z.coerce.number().optional(), + REDIS_PORT: z.coerce.number().optional(), + REDIS_USERNAME: z.string().optional(), + REDIS_PASSWORD: z.string().optional(), + + DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10), + DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1), }); export type Environment = z.infer; diff --git a/apps/webapp/app/models/jobRun.server.ts b/apps/webapp/app/models/jobRun.server.ts new file mode 100644 index 000000000..537879ce2 --- /dev/null +++ b/apps/webapp/app/models/jobRun.server.ts @@ -0,0 +1,45 @@ +import type { JobRun, JobRunStatus } from "@trigger.dev/database"; + +const COMPLETED_STATUSES: Array = [ + "CANCELED", + "ABORTED", + "SUCCESS", + "TIMED_OUT", + "INVALID_PAYLOAD", + "FAILURE", + "UNRESOLVED_AUTH", +]; + +export function isRunCompleted(status: JobRunStatus) { + return COMPLETED_STATUSES.includes(status); +} + +export type RunBasicStatus = "WAITING" | "PENDING" | "RUNNING" | "COMPLETED" | "FAILED"; + +export function runBasicStatus(status: JobRunStatus): RunBasicStatus { + switch (status) { + case "WAITING_ON_CONNECTIONS": + case "QUEUED": + case "PREPROCESSING": + case "PENDING": + return "PENDING"; + case "STARTED": + case "EXECUTING": + case "WAITING_TO_CONTINUE": + case "WAITING_TO_EXECUTE": + return "RUNNING"; + case "FAILURE": + case "TIMED_OUT": + case "UNRESOLVED_AUTH": + case "CANCELED": + case "ABORTED": + case "INVALID_PAYLOAD": + return "FAILED"; + case "SUCCESS": + return "COMPLETED"; + default: { + const _exhaustiveCheck: never = status; + throw new Error(`Non-exhaustive match for value: ${status}`); + } + } +} diff --git a/apps/webapp/app/models/jobRunExecution.server.ts b/apps/webapp/app/models/jobRunExecution.server.ts deleted file mode 100644 index c22cc5846..000000000 --- a/apps/webapp/app/models/jobRunExecution.server.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { JobRun } from "@trigger.dev/database"; -import { PrismaClientOrTransaction } from "~/db.server"; -import { executionWorker } from "~/services/worker.server"; - -export async function dequeueRunExecutionV2(run: JobRun, tx: PrismaClientOrTransaction) { - return await executionWorker.dequeue(`job_run:${run.id}`, { - tx, - }); -} - -export type EnqueueRunExecutionV3Options = { - runAt?: Date; - skipRetrying?: boolean; -}; - -export async function enqueueRunExecutionV3( - run: JobRun, - tx: PrismaClientOrTransaction, - options: EnqueueRunExecutionV3Options = {} -) { - const reason = run.status === "PREPROCESSING" ? "PREPROCESS" : "EXECUTE_JOB"; - - return await executionWorker.enqueue( - "performRunExecutionV3", - { - id: run.id, - reason: reason, - }, - { - tx, - runAt: options.runAt, - queueName: `job_run:${run.id}`, - jobKey: `job_run:${reason}:${run.id}`, - maxAttempts: options.skipRetrying ? 1 : undefined, - } - ); -} - -export async function dequeueRunExecutionV3(run: JobRun, tx: PrismaClientOrTransaction) { - await executionWorker.dequeue(`job_run:EXECUTE_JOB:${run.id}`, { - tx, - }); - - await executionWorker.dequeue(`job_run:PREPROCESS:${run.id}`, { - tx, - }); -} diff --git a/apps/webapp/app/platform/zodWorker.server.ts b/apps/webapp/app/platform/zodWorker.server.ts index 347dd5981..b88520a58 100644 --- a/apps/webapp/app/platform/zodWorker.server.ts +++ b/apps/webapp/app/platform/zodWorker.server.ts @@ -94,6 +94,11 @@ export type ZodWorkerCleanupOptions = { type ZodWorkerReporter = (event: string, properties: Record) => Promise; +export interface ZodWorkerRateLimiter { + forbiddenFlags(): Promise; + wrapTask(t: Task, rescheduler: Task): Task; +} + export type ZodWorkerOptions = { name: string; runnerOptions: RunnerOptions; @@ -104,6 +109,7 @@ export type ZodWorkerOptions = { cleanup?: ZodWorkerCleanupOptions; reporter?: ZodWorkerReporter; shutdownTimeoutInMs?: number; + rateLimiter?: ZodWorkerRateLimiter; }; export class ZodWorker { @@ -116,6 +122,7 @@ export class ZodWorker { #runner?: GraphileRunner; #cleanup: ZodWorkerCleanupOptions | undefined; #reporter?: ZodWorkerReporter; + #rateLimiter?: ZodWorkerRateLimiter; #shutdownTimeoutInMs?: number; #shuttingDown = false; @@ -128,6 +135,7 @@ export class ZodWorker { this.#recurringTasks = options.recurringTasks; this.#cleanup = options.cleanup; this.#reporter = options.reporter; + this.#rateLimiter = options.rateLimiter; this.#shutdownTimeoutInMs = options.shutdownTimeoutInMs ?? 60000; // default to 60 seconds } @@ -151,6 +159,7 @@ export class ZodWorker { noHandleSignals: true, taskList: this.#createTaskListFromTasks(), parsedCronItems, + forbiddenFlags: this.#rateLimiter?.forbiddenFlags.bind(this.#rateLimiter), }); if (!this.#runner) { @@ -395,7 +404,11 @@ export class ZodWorker { return this.#handleMessage(key, payload, helpers); }; - taskList[key] = task; + if (this.#rateLimiter) { + taskList[key] = this.#rateLimiter.wrapTask(task, this.#rescheduleTask.bind(this)); + } else { + taskList[key] = task; + } } for (const [key] of Object.entries(this.#recurringTasks ?? {})) { @@ -425,6 +438,19 @@ export class ZodWorker { return taskList; } + async #rescheduleTask(payload: unknown, helpers: JobHelpers) { + this.#logDebug("Rescheduling task", { payload, job: helpers.job }); + + await this.enqueue(helpers.job.task_identifier, payload, { + runAt: helpers.job.run_at, + queueName: helpers.job.queue_name ?? undefined, + priority: helpers.job.priority, + jobKey: helpers.job.key ?? undefined, + flags: Object.keys(helpers.job.flags ?? []), + maxAttempts: helpers.job.max_attempts, + }); + } + #createCronItemsFromRecurringTasks() { const cronItems: CronItem[] = []; diff --git a/apps/webapp/app/presenters/JobPresenter.server.ts b/apps/webapp/app/presenters/JobPresenter.server.ts index 957ed7200..023ee89aa 100644 --- a/apps/webapp/app/presenters/JobPresenter.server.ts +++ b/apps/webapp/app/presenters/JobPresenter.server.ts @@ -43,6 +43,13 @@ export class JobPresenter { eventSpecification: true, properties: true, status: true, + concurrencyLimit: true, + concurrencyLimitGroup: { + select: { + name: true, + concurrencyLimit: true, + }, + }, runs: { select: { createdAt: true, @@ -186,6 +193,8 @@ export class JobPresenter { enabled: alias.version.status === "ACTIVE", lastRun: alias.version.runs.at(0)?.createdAt, version: alias.version.version, + concurrencyLimit: alias.version.concurrencyLimit, + concurrencyLimitGroup: alias.version.concurrencyLimitGroup, })); const projectRootPath = projectPath({ slug: organizationSlug }, { slug: projectSlug }); diff --git a/apps/webapp/app/presenters/RunListPresenter.server.ts b/apps/webapp/app/presenters/RunListPresenter.server.ts index 63a1df2b5..fabfcd896 100644 --- a/apps/webapp/app/presenters/RunListPresenter.server.ts +++ b/apps/webapp/app/presenters/RunListPresenter.server.ts @@ -6,14 +6,15 @@ export type Direction = z.infer; type RunListOptions = { userId: string; - jobSlug: string; + jobSlug?: string; organizationSlug: string; projectSlug: string; direction?: Direction; cursor?: string; + pageSize?: number; }; -const PAGE_SIZE = 20; +const DEFAULT_PAGE_SIZE = 20; export type RunList = Awaited>; @@ -31,6 +32,7 @@ export class RunListPresenter { projectSlug, direction = "forward", cursor, + pageSize = DEFAULT_PAGE_SIZE, }: RunListOptions) { const directionMultiplier = direction === "forward" ? 1 : -1; @@ -41,6 +43,7 @@ export class RunListPresenter { startedAt: true, completedAt: true, createdAt: true, + executionDuration: true, isTest: true, status: true, environment: { @@ -59,11 +62,19 @@ export class RunListPresenter { version: true, }, }, + job: { + select: { + slug: true, + title: true, + }, + }, }, where: { - job: { - slug: jobSlug, - }, + job: jobSlug + ? { + slug: jobSlug, + } + : undefined, project: { slug: projectSlug, }, @@ -82,8 +93,8 @@ export class RunListPresenter { }, }, orderBy: [{ id: "desc" }], - //take an extra page to tell if there are more - take: directionMultiplier * (PAGE_SIZE + 1), + //take an extra record to tell if there are more + take: directionMultiplier * (pageSize + 1), //skip the cursor if there is one skip: cursor ? 1 : 0, cursor: cursor @@ -93,7 +104,7 @@ export class RunListPresenter { : undefined, }); - const hasMore = runs.length > PAGE_SIZE; + const hasMore = runs.length > pageSize; //get cursors for next and previous pages let next: string | undefined; @@ -102,19 +113,21 @@ export class RunListPresenter { case "forward": previous = cursor ? runs.at(0)?.id : undefined; if (hasMore) { - next = runs[PAGE_SIZE - 1]?.id; + next = runs[pageSize - 1]?.id; } break; case "backward": if (hasMore) { previous = runs[1]?.id; + next = runs[pageSize]?.id; + } else { + next = runs[pageSize - 1]?.id; } - next = runs[PAGE_SIZE - 1]?.id; break; } const runsToReturn = - direction === "backward" && hasMore ? runs.slice(1, PAGE_SIZE + 1) : runs.slice(0, PAGE_SIZE); + direction === "backward" && hasMore ? runs.slice(1, pageSize + 1) : runs.slice(0, pageSize); return { runs: runsToReturn.map((run) => ({ @@ -123,6 +136,7 @@ export class RunListPresenter { startedAt: run.startedAt, completedAt: run.completedAt, createdAt: run.createdAt, + executionDuration: run.executionDuration, isTest: run.isTest, status: run.status, version: run.version?.version ?? "unknown", @@ -131,6 +145,7 @@ export class RunListPresenter { slug: run.environment.slug, userId: run.environment.orgMember?.userId, }, + job: run.job, })), pagination: { next, diff --git a/apps/webapp/app/presenters/RunPresenter.server.ts b/apps/webapp/app/presenters/RunPresenter.server.ts index 8b14e0e49..c658d5aa6 100644 --- a/apps/webapp/app/presenters/RunPresenter.server.ts +++ b/apps/webapp/app/presenters/RunPresenter.server.ts @@ -5,6 +5,7 @@ import { StyleSchema, } from "@trigger.dev/core"; import { PrismaClient, prisma } from "~/db.server"; +import { isRunCompleted, runBasicStatus } from "~/models/jobRun.server"; import { mergeProperties } from "~/utils/mergeProperties.server"; import { taskListToTree } from "~/utils/taskListToTree"; @@ -67,6 +68,8 @@ export class RunPresenter { id: run.id, number: run.number, status: run.status, + basicStatus: runBasicStatus(run.status), + isFinished: isRunCompleted(run.status), startedAt: run.startedAt, completedAt: run.completedAt, isTest: run.isTest, @@ -82,6 +85,8 @@ export class RunPresenter { runConnections: run.runConnections, missingConnections: run.missingConnections, error: runError, + executionDuration: run.executionDuration, + executionCount: run.executionCount, }; } @@ -112,6 +117,8 @@ export class RunPresenter { isTest: true, properties: true, output: true, + executionCount: true, + executionDuration: true, version: { select: { version: true, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug._index/route.tsx index 4282a4dc8..f7aaed590 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug._index/route.tsx @@ -10,6 +10,8 @@ import { PageTitleRow, PageTitle, PageButtons, + PageInfoRow, + PageInfoGroup, } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; import { useOrganization } from "~/hooks/useOrganizations"; @@ -38,6 +40,13 @@ export default function Page() { + + + + UID: {organization.id} + + +
    diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx index 412b9619b..d0a3ad7f9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx @@ -105,12 +105,14 @@ export default function Page() { }; }, [selected, clients]); - const isAnyClientFullyConfigured = useMemo(() => { - return clients.some((client) => { - const { DEVELOPMENT, PRODUCTION } = client.endpoints; - return PRODUCTION.state === "configured" && DEVELOPMENT.state === PRODUCTION.state; - }); - }, [clients]); + const isAnyClientFullyConfigured = clients.some((client) => { + const { DEVELOPMENT, PRODUCTION, STAGING } = client.endpoints; + return ( + PRODUCTION.state === "configured" || + DEVELOPMENT.state === "configured" || + (STAGING && STAGING.state === "configured") + ); + }); const organization = useOrganization(); const project = useProject(); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination.tsx index 0992803bc..514ba4db6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination.tsx @@ -22,31 +22,39 @@ export function ListPagination({ function NextButton({ cursor }: { cursor?: string }) { const path = useCursorPath(cursor, "forward"); - return path ? ( + return ( !path && e.preventDefault()} > Next - ) : null; + ); } function PreviousButton({ cursor }: { cursor?: string }) { const path = useCursorPath(cursor, "backward"); - return path ? ( + return ( !path && e.preventDefault()} > Prev - ) : null; + ); } function useCursorPath(cursor: string | undefined, direction: Direction) { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/route.tsx index ceeb43196..4561f60b4 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/route.tsx @@ -72,8 +72,8 @@ export default function Page() {
    - +
    + {(open) => (
    @@ -32,7 +32,7 @@ export default function Page() { Environments
    - +
    {job.status === "ACTIVE" && ( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx index 1a76ae80a..e7638fb17 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx @@ -297,7 +297,9 @@ export default function Page() { label={} description={ <> - Run #{run.number}{" "} + {typeof run.number === "number" + ? `Run #${run.number}` + : `Run ${run.id.slice(0, 8)}`} {runStatusTitle(run.status).toLocaleLowerCase()} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.runs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.runs/route.tsx new file mode 100644 index 000000000..978263524 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.runs/route.tsx @@ -0,0 +1,88 @@ +import { useNavigation } from "@remix-run/react"; +import { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { + PageButtons, + PageDescription, + PageHeader, + PageTitle, + PageTitleRow, +} from "~/components/primitives/PageHeader"; +import { RunsTable } from "~/components/runs/RunsTable"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; +import { RunListPresenter } from "~/presenters/RunListPresenter.server"; +import { requireUserId } from "~/services/session.server"; +import { ProjectParamSchema, docsPath, projectPath } from "~/utils/pathBuilder"; +import { ListPagination } from "../_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination"; +import { RunListSearchSchema } from "../_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/route"; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { projectParam, organizationSlug } = ProjectParamSchema.parse(params); + + const url = new URL(request.url); + const s = Object.fromEntries(url.searchParams.entries()); + const searchParams = RunListSearchSchema.parse(s); + + const presenter = new RunListPresenter(); + const list = await presenter.call({ + userId, + projectSlug: projectParam, + organizationSlug, + direction: searchParams.direction, + cursor: searchParams.cursor, + pageSize: 25, + }); + + return typedjson({ + list, + }); +}; + +export default function Page() { + const { list } = useTypedLoaderData(); + const navigation = useNavigation(); + const isLoading = navigation.state !== "idle"; + const organization = useOrganization(); + const project = useProject(); + + return ( + + + + + + + Run documentation + + + + All job runs in this project + + + +
    +
    + +
    + + +
    +     +     + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.new/route.tsx b/apps/webapp/app/routes/_app.orgs.new/route.tsx index f6ba49474..fbb2f55a9 100644 --- a/apps/webapp/app/routes/_app.orgs.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.new/route.tsx @@ -97,6 +97,7 @@ export default function NewOrganizationPage() { {...conform.input(orgName, { type: "text" })} placeholder="Your Organization name" icon="organization" + autoFocus /> E.g. your company name or your workspace name. {orgName.error} diff --git a/apps/webapp/app/services/executions/createExecutionEvent.server.ts b/apps/webapp/app/services/executions/createExecutionEvent.server.ts index 2d5f83791..971ac1088 100644 --- a/apps/webapp/app/services/executions/createExecutionEvent.server.ts +++ b/apps/webapp/app/services/executions/createExecutionEvent.server.ts @@ -10,6 +10,7 @@ export type CreateExecutionEventInput = { eventTime: Date; eventType: "start" | "finish"; drift?: number; + concurrencyLimitGroupId?: string | null; }; export class CreateExecutionEventService { @@ -25,7 +26,8 @@ export class CreateExecutionEventService { "run_id", "event_time", "event_type", - "drift_amount_in_ms" + "drift_amount_in_ms", + "concurrency_limit_group_id" ) VALUES ( ${input.organizationId}, ${input.projectId}, @@ -34,7 +36,8 @@ export class CreateExecutionEventService { ${input.runId}, ${input.eventTime}, ${input.eventType === "start" ? 1 : -1}, - ${input.drift} + ${input.drift}, + ${input.concurrencyLimitGroupId} ) `; } diff --git a/apps/webapp/app/services/jobs/registerJob.server.ts b/apps/webapp/app/services/jobs/registerJob.server.ts index 63aeb5cb5..82d6367c8 100644 --- a/apps/webapp/app/services/jobs/registerJob.server.ts +++ b/apps/webapp/app/services/jobs/registerJob.server.ts @@ -14,6 +14,7 @@ import type { RuntimeEnvironment } from "~/models/runtimeEnvironment.server"; import type { AuthenticatedEnvironment } from "../apiAuth.server"; import { logger } from "../logger.server"; import { RegisterScheduleSourceService } from "../schedules/registerScheduleSource.server"; +import { executionRateLimiter } from "../runExecutionRateLimiter.server"; export class RegisterJobService { #prismaClient: PrismaClient; @@ -105,32 +106,28 @@ export class RegisterJobService { }, }); - // Upsert the JobQueue - const queueName = "default"; + const { examples, ...eventSpecification } = metadata.event; // Job Queues are going to be deprecated or used for something else, we're just doing this for now - const jobQueue = await this.#prismaClient.jobQueue.upsert({ - where: { - environmentId_name: { - environmentId: environment.id, - name: queueName, - }, - }, - create: { - environment: { - connect: { - id: environment.id, - }, - }, - name: queueName, - maxJobs: DEFAULT_MAX_CONCURRENT_RUNS, - }, - update: { - maxJobs: DEFAULT_MAX_CONCURRENT_RUNS, - }, - }); - - const { examples, ...eventSpecification } = metadata.event; + const concurrencyLimitGroup = + typeof metadata.concurrencyLimit === "object" + ? await this.#prismaClient.concurrencyLimitGroup.upsert({ + where: { + environmentId_name: { + environmentId: environment.id, + name: metadata.concurrencyLimit.id, + }, + }, + create: { + environmentId: environment.id, + name: metadata.concurrencyLimit.id, + concurrencyLimit: metadata.concurrencyLimit.limit, + }, + update: { + concurrencyLimit: metadata.concurrencyLimit.limit, + }, + }) + : null; // Upsert the JobVersion const jobVersion = await this.#prismaClient.jobVersion.upsert({ @@ -142,57 +139,29 @@ export class RegisterJobService { }, }, create: { - job: { - connect: { - id: job.id, - }, - }, - endpoint: { - connect: { - id: endpoint.id, - }, - }, - environment: { - connect: { - id: environment.id, - }, - }, - organization: { - connect: { - id: environment.organizationId, - }, - }, - project: { - connect: { - id: environment.projectId, - }, - }, - queue: { - connect: { - id: jobQueue.id, - }, - }, + jobId: job.id, + endpointId: endpoint.id, + environmentId: environment.id, + organizationId: environment.organizationId, + projectId: environment.projectId, version: metadata.version, eventSpecification, preprocessRuns: metadata.preprocessRuns, startPosition: "LATEST", status: "ACTIVE", + concurrencyLimitGroupId: concurrencyLimitGroup?.id ?? null, + concurrencyLimit: + typeof metadata.concurrencyLimit === "number" ? metadata.concurrencyLimit : null, }, update: { status: "ACTIVE", startPosition: "LATEST", eventSpecification, preprocessRuns: metadata.preprocessRuns, - queue: { - connect: { - id: jobQueue.id, - }, - }, - endpoint: { - connect: { - id: endpoint.id, - }, - }, + endpointId: endpoint.id, + concurrencyLimitGroupId: concurrencyLimitGroup?.id ?? null, + concurrencyLimit: + typeof metadata.concurrencyLimit === "number" ? metadata.concurrencyLimit : null, }, include: { integrations: { @@ -200,9 +169,28 @@ export class RegisterJobService { integration: true, }, }, + concurrencyLimitGroup: true, }, }); + try { + if (jobVersion.concurrencyLimitGroup) { + // Upsert the maxSize for the concurrency limit group + await executionRateLimiter?.putConcurrencyLimitGroup( + jobVersion.concurrencyLimitGroup, + environment + ); + } + + await executionRateLimiter?.putJobVersionConcurrencyLimit(jobVersion, environment); + } catch (error) { + logger.error("Error setting concurrency limit", { + error, + jobVersionId: jobVersion.id, + environmentId: environment.id, + }); + } + // Upsert the examples and delete any that are no longer in the metadata const upsertedExamples = new Set(); if (examples) { diff --git a/apps/webapp/app/services/runExecutionRateLimiter.server.ts b/apps/webapp/app/services/runExecutionRateLimiter.server.ts new file mode 100644 index 000000000..b482c5c50 --- /dev/null +++ b/apps/webapp/app/services/runExecutionRateLimiter.server.ts @@ -0,0 +1,406 @@ +import { env } from "~/env.server"; +import { + Callback, + Cluster, + ClusterNode, + ClusterOptions, + Redis, + RedisOptions, + Result, +} from "ioredis"; +import { JobHelpers, Task } from "graphile-worker"; +import { singleton } from "~/utils/singleton"; +import { logger } from "./logger.server"; +import { ZodWorkerRateLimiter } from "~/platform/zodWorker.server"; +import { + ConcurrencyLimitGroup, + JobRun, + JobVersion, + RuntimeEnvironment, +} from "@trigger.dev/database"; + +export interface RunExecutionRateLimiter { + putConcurrencyLimitGroup( + concurrencyLimitGroup: ConcurrencyLimitGroup, + env: RuntimeEnvironment + ): Promise; + putJobVersionConcurrencyLimit(jobVersion: JobVersion, env: RuntimeEnvironment): Promise; + setMaxSizeForFlag(flag: string, maxSize: number): Promise; + delMaxSizeForFlag(flag: string): Promise; + flagsForRun( + run: JobRun, + version: JobVersion & { + environment: RuntimeEnvironment; + concurrencyLimitGroup?: ConcurrencyLimitGroup; + } + ): string[]; +} + +declare module "ioredis" { + interface RedisCommander { + beforeTask( + setKey: string, + maxSizeKey: string, + forbiddenFlagsKey: string, + jobId: string, + timestamp: string, + windowSize: string, + forbiddenFlag: string, + maxSize: string, + callback?: Callback + ): Result; + rollbackBeforeTask(keys: number, ...args: string[]): Result; + + afterTask( + setKey: string, + maxSizeKey: string, + forbiddenFlagsKey: string, + jobId: string, + timestamp: string, + windowSize: string, + forbiddenFlag: string, + maxSize: string, + callback?: Callback + ): Result; + } +} + +type RedisRunExecutionRateLimiterOptions = { + redis?: RedisOptions; + cluster?: { + startupNodes: ClusterNode[]; + options?: ClusterOptions; + }; + defaultConcurrency?: number; + windowSize?: number; + prefix?: string; +}; + +const FORBIDDEN_FLAG_KEY = "forbiddenFlags"; +const KEY_PREFIX = "tr:exec:"; + +class RedisRunExecutionRateLimiter implements RunExecutionRateLimiter, ZodWorkerRateLimiter { + private redis: Redis | Cluster; + private defaultMaxSize: number; + private windowSize: number; + + constructor(options?: RedisRunExecutionRateLimiterOptions) { + this.redis = options?.cluster + ? new Redis.Cluster(options.cluster.startupNodes, options.cluster.options) + : new Redis(options?.redis ?? {}); + this.defaultMaxSize = options?.defaultConcurrency ?? 10; + this.windowSize = options?.windowSize ?? 1000 * 15 * 60; // 2 minutes + + this.redis.defineCommand("beforeTask", { + numberOfKeys: 3, + lua: ` +local setKey = KEYS[1] +local maxSizeKey = KEYS[2] +local forbiddenFlagsKey = KEYS[3] +local jobId = ARGV[1] +local timestamp = ARGV[2] +local windowSize = ARGV[3] +local forbiddenFlag = ARGV[4] +local defaultMaxSize = ARGV[5] + +local maxSize = tonumber(redis.call('GET', maxSizeKey) or defaultMaxSize) +local currentSize = redis.call('ZCOUNT', setKey, timestamp - windowSize, timestamp) + +if currentSize < maxSize then + redis.call('ZADD', setKey, timestamp, jobId) + + return true +else + redis.call('SADD', forbiddenFlagsKey, forbiddenFlag) + + return false +end + `, + }); + + // This will remove the job ID from the ZSET + this.redis.defineCommand("rollbackBeforeTask", { + lua: ` +for i, key in ipairs(KEYS) do + redis.call('ZREM', key, ARGV[1]) +end + `, + }); + + this.redis.defineCommand("afterTask", { + numberOfKeys: 3, + lua: ` +local setKey = KEYS[1] +local maxSizeKey = KEYS[2] +local forbiddenFlagsKey = KEYS[3] +local jobId = ARGV[1] +local timestamp = ARGV[2] +local windowSize = ARGV[3] +local forbiddenFlag = ARGV[4] +local defaultMaxSize = ARGV[5] + +local maxSize = tonumber(redis.call('GET', maxSizeKey) or defaultMaxSize) + +-- Remove the job ID from the ZSET +redis.call('ZREM', setKey, jobId) + +-- Count the current number of jobs in the window +local currentSize = redis.call('ZCOUNT', setKey, timestamp - windowSize, timestamp) + +-- The cleanup of old job IDs is now an essential part of maintaining the ZSET's size +redis.call('ZREMRANGEBYSCORE', setKey, '-inf', timestamp - windowSize) + +-- Update the forbidden flags based on the current size +if currentSize < maxSize then + -- Only remove the forbidden flag if it's no longer needed + redis.call('SREM', forbiddenFlagsKey, forbiddenFlag) + return true +else + -- No need to add the forbidden flag here as it should be handled in beforeTask + return false +end + + `, + }); + + if (this.redis instanceof Redis) { + logger.debug("โšก RedisGraphileRateLimiter connected to Redis", { + host: this.redis.options.host, + port: this.redis.options.port, + }); + } else { + logger.debug("โšก RedisGraphileRateLimiter connected to Redis Cluster", { + nodes: this.redis.nodes, + }); + } + } + + async forbiddenFlags(): Promise { + return this.redis.smembers(FORBIDDEN_FLAG_KEY); + } + + async putConcurrencyLimitGroup( + concurrencyLimitGroup: ConcurrencyLimitGroup, + env: RuntimeEnvironment + ): Promise { + await this.setMaxSizeForFlag( + this.flagForConcurrencyLimitGroup(concurrencyLimitGroup, env), + concurrencyLimitGroup.concurrencyLimit + ); + } + + async putJobVersionConcurrencyLimit( + jobVersion: JobVersion, + env: RuntimeEnvironment + ): Promise { + const flag = this.flagForJobVersion(jobVersion, env); + + if (typeof jobVersion.concurrencyLimit === "number" && jobVersion.concurrencyLimit > 0) { + await this.setMaxSizeForFlag(flag, jobVersion.concurrencyLimit); + } else { + await this.delMaxSizeForFlag(flag); + } + } + + flagsForRun( + run: JobRun, + version: JobVersion & { + environment: RuntimeEnvironment; + concurrencyLimitGroup?: ConcurrencyLimitGroup | null; + } + ): string[] { + const flags = [this.flagForOrganization(run)]; + + if (version.concurrencyLimitGroup) { + flags.push( + this.flagForConcurrencyLimitGroup(version.concurrencyLimitGroup, version.environment) + ); + } else if (typeof version.concurrencyLimit === "number" && version.concurrencyLimit > 0) { + flags.push(this.flagForJobVersion(version, version.environment)); + } + + return flags; + } + + flagForConcurrencyLimitGroup( + concurrencyLimitGroup: ConcurrencyLimitGroup, + env: RuntimeEnvironment + ): string { + return `rl:group:${env.id}:${env.slug}:${concurrencyLimitGroup.name}`; + } + + flagForOrganization(run: JobRun): string { + return `rl:org:${run.organizationId}`; + } + + flagForJobVersion(version: JobVersion, env: RuntimeEnvironment): string { + return `rl:job:${env.slug}:${version.id}`; + } + + async setMaxSizeForFlag(flag: string, maxSize: number): Promise { + await this.redis.set(`${flag}:maxSize`, String(maxSize)); + } + + async delMaxSizeForFlag(flag: string): Promise { + await this.redis.del(`${flag}:maxSize`); + } + + wrapTask(t: Task, rescheduler: Task): Task { + return async (payload: unknown, helpers: JobHelpers) => { + const flags = Object.keys(helpers.job.flags ?? {}).filter((flag) => flag.startsWith("rl:")); + + if (flags.length === 0) { + return t(payload, helpers); + } + + let passedFlags = []; + + for (const flag of flags) { + const result = await this.#callBeforeTask(flag, String(helpers.job.id)); + + if ( + (result.status === "fulfilled" && result.value === null) || + result.status === "rejected" + ) { + logger.debug("Rolling back passed flags", { + flag, + passedFlags, + jobId: String(helpers.job.id), + result, + }); + // If there are any passed flags, we need to roll them back + await this.#rollbackPassedFlags(passedFlags, String(helpers.job.id)); + + return await rescheduler(payload, helpers); + } + + passedFlags.push(flag); + } + + try { + await t(payload, helpers); + } finally { + const afterResults = await Promise.allSettled( + flags.map(async (flag) => this.#callAfterTask(flag, String(helpers.job.id))) + ); + } + }; + } + + async #callBeforeTask( + flag: string, + jobId: string + ): Promise< + | { status: "fulfilled"; value: number | null; durationInMs: number } + | { status: "rejected"; error: any } + > { + try { + const now = performance.now(); + const value = await this.redis.beforeTask( + flag, + `${flag}:maxSize`, + FORBIDDEN_FLAG_KEY, + jobId, + String(Date.now()), + String(this.windowSize), + flag, + String(this.defaultMaxSize) + ); + + const durationInMs = performance.now() - now; + + return { + status: "fulfilled", + value, + durationInMs, + }; + } catch (error) { + logger.error("Failed to call beforeTask", { error, flag, jobId }); + + return { + status: "rejected", + error, + }; + } + } + + // Method for rolling back passed flags using a single Lua script + async #rollbackPassedFlags(passedFlags: string[], jobId: string) { + if (passedFlags.length > 0) { + await this.redis.rollbackBeforeTask(passedFlags.length, ...passedFlags, jobId); + } + } + + async #callAfterTask(flag: string, jobId: string) { + try { + const now = performance.now(); + + const results = await this.redis.afterTask( + flag, + `${flag}:maxSize`, + FORBIDDEN_FLAG_KEY, + jobId, + String(Date.now()), + String(this.windowSize), + flag, + String(this.defaultMaxSize) + ); + + const durationInMs = performance.now() - now; + + return { + results, + durationInMs, + }; + } catch (error) { + logger.error("Failed to call afterTask", { error, flag, jobId }); + } + } +} + +export const executionRateLimiter = singleton("execution-rate-limiter", getRateLimiter); + +function getRateLimiter() { + if (env.REDIS_HOST && env.REDIS_PORT) { + if (env.REDIS_READER_HOST) { + return new RedisRunExecutionRateLimiter({ + cluster: { + startupNodes: [ + { host: env.REDIS_HOST, port: env.REDIS_PORT }, + { host: env.REDIS_READER_HOST, port: env.REDIS_READER_PORT ?? env.REDIS_PORT }, + ], + options: { + keyPrefix: KEY_PREFIX, + scaleReads: "slave", + redisOptions: { + password: env.REDIS_PASSWORD, + tls: { + checkServerIdentity: () => { + // disable TLS verification + return undefined + } + }, + enableAutoPipelining: true, + }, + dnsLookup: (address, callback) => callback(null, address), + slotsRefreshTimeout: 10000, + }, + }, + defaultConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT, + }); + } else { + return new RedisRunExecutionRateLimiter({ + redis: { + keyPrefix: KEY_PREFIX, + port: env.REDIS_PORT, + host: env.REDIS_HOST, + username: env.REDIS_USERNAME, + password: env.REDIS_PASSWORD, + enableAutoPipelining: true, + tls: {} + }, + defaultConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT, + }); + } + } +} diff --git a/apps/webapp/app/services/runs/cancelRun.server.ts b/apps/webapp/app/services/runs/cancelRun.server.ts index f33457a81..4b4d5926a 100644 --- a/apps/webapp/app/services/runs/cancelRun.server.ts +++ b/apps/webapp/app/services/runs/cancelRun.server.ts @@ -1,6 +1,6 @@ import { PrismaClient, prisma } from "~/db.server"; -import { executionWorker } from "../worker.server"; -import { dequeueRunExecutionV3 } from "~/models/jobRunExecution.server"; +import { PerformRunExecutionV3Service } from "./performRunExecutionV3.server"; +import { ResumeRunService } from "./resumeRun.server"; export class CancelRunService { #prismaClient: PrismaClient; @@ -39,7 +39,8 @@ export class CancelRunService { }, }); - await dequeueRunExecutionV3(run, tx); + await PerformRunExecutionV3Service.dequeue(run, tx); + await ResumeRunService.dequeue(run, tx); }); } catch (error) { throw error; diff --git a/apps/webapp/app/services/runs/continueRun.server.ts b/apps/webapp/app/services/runs/continueRun.server.ts index c001b8a4f..982f0daac 100644 --- a/apps/webapp/app/services/runs/continueRun.server.ts +++ b/apps/webapp/app/services/runs/continueRun.server.ts @@ -1,6 +1,5 @@ -import { RuntimeEnvironmentType } from "@trigger.dev/database"; import { $transaction, Prisma, PrismaClient, prisma } from "~/db.server"; -import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server"; +import { ResumeRunService } from "./resumeRun.server"; const RESUMABLE_STATUSES = ["FAILURE", "TIMED_OUT", "UNRESOLVED_AUTH", "ABORTED", "CANCELED"]; @@ -39,9 +38,7 @@ export class ContinueRunService { }, }); - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(run, tx); }, { timeout: 10000 } ); diff --git a/apps/webapp/app/services/runs/createRun.server.ts b/apps/webapp/app/services/runs/createRun.server.ts index 131835875..aab48b746 100644 --- a/apps/webapp/app/services/runs/createRun.server.ts +++ b/apps/webapp/app/services/runs/createRun.server.ts @@ -31,12 +31,6 @@ export class CreateRunService { }, }); - const jobQueue = await this.#prismaClient.jobQueue.findUniqueOrThrow({ - where: { - id: version.queueId, - }, - }); - const eventRecord = await this.#prismaClient.eventRecord.findUniqueOrThrow({ where: { id: eventId, @@ -44,22 +38,8 @@ export class CreateRunService { }); return await $transaction(this.#prismaClient, async (tx) => { - // Get the current max number for the given jobId - const latestJob = await tx.jobRun.findFirst({ - where: { jobId: job.id }, - orderBy: { id: "desc" }, - select: { - number: true, - }, - }); - - // Increment the number for the new execution - const newNumber = (latestJob?.number ?? 0) + 1; - - // Create the new execution with the incremented number const run = await tx.jobRun.create({ data: { - number: newNumber, preprocess: version.preprocessRuns, jobId: job.id, versionId: version.id, @@ -68,7 +48,6 @@ export class CreateRunService { organizationId: environment.organizationId, projectId: environment.projectId, endpointId: endpoint.id, - queueId: jobQueue.id, externalAccountId: eventRecord.externalAccountId ? eventRecord.externalAccountId : undefined, diff --git a/apps/webapp/app/services/runs/performRunExecutionV3.server.ts b/apps/webapp/app/services/runs/performRunExecutionV3.server.ts index a9f836058..52fc0ed78 100644 --- a/apps/webapp/app/services/runs/performRunExecutionV3.server.ts +++ b/apps/webapp/app/services/runs/performRunExecutionV3.server.ts @@ -16,7 +16,12 @@ import { supportsFeature, } from "@trigger.dev/core"; import { BloomFilter } from "@trigger.dev/core-backend"; -import { RuntimeEnvironmentType, type Task } from "@trigger.dev/database"; +import { + ConcurrencyLimitGroup, + JobRun, + JobVersion, + RuntimeEnvironment, +} from "@trigger.dev/database"; import { generateErrorMessage } from "zod-error"; import { eventRecordToApiJson } from "~/api.server"; import { @@ -26,7 +31,7 @@ import { } from "~/consts"; import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server"; import { detectResponseIsTimeout } from "~/models/endpoint.server"; -import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server"; +import { isRunCompleted } from "~/models/jobRun.server"; import { resolveRunConnections } from "~/models/runConnection.server"; import { prepareTasksForCaching, prepareTasksForCachingLegacy } from "~/models/task.server"; import { CompleteRunTaskService } from "~/routes/api.v1.runs.$runId.tasks.$id.complete"; @@ -36,8 +41,11 @@ import { EndpointApi } from "../endpointApi.server"; import { createExecutionEvent } from "../executions/createExecutionEvent.server"; import { logger } from "../logger.server"; import { ResumeTaskService } from "../tasks/resumeTask.server"; -import { workerQueue } from "../worker.server"; +import { executionWorker, workerQueue } from "../worker.server"; import { forceYieldCoordinator } from "./forceYieldCoordinator.server"; +import { ResumeRunService } from "./resumeRun.server"; +import { executionRateLimiter } from "../runExecutionRateLimiter.server"; +import { env } from "~/env.server"; type FoundRun = NonNullable>>; type FoundTask = FoundRun["tasks"][number]; @@ -58,8 +66,15 @@ export type PerformRunExecutionV3Input = { * @deprecated Resuming tasks now goes through ResumeTaskService, this is included here for backwards compatibility */ resumeTaskId?: string; + + /** + * Specifies whether this should be the last attempt to execute the run. If so, we can't retry the run in case of a failure. + */ + lastAttempt: boolean; }; +export type RunExecutionPriority = "initial" | "resume"; + export class PerformRunExecutionV3Service { #prismaClient: PrismaClient; @@ -74,206 +89,85 @@ export class PerformRunExecutionV3Service { return; } - switch (input.reason) { - case "PREPROCESS": { - await this.#executePreprocessing(run); - break; - } - case "EXECUTE_JOB": { - await this.#executeJob(run, input, driftInMs); - break; - } - } + await this.#executeJob(run, input, driftInMs); } - // Execute the preprocessing step of a run, which will send the payload to the endpoint and give the job - // an opportunity to generate run properties based on the payload. - // If the endpoint is not available, or the response is not ok, - // the run execution will be marked as failed and the run will start - async #executePreprocessing(run: FoundRun) { - const client = new EndpointApi(run.environment.apiKey, run.endpoint.url); - const event = eventRecordToApiJson(run.event); - - const { response, parser } = await client.preprocessRunRequest({ - event, - job: { - id: run.version.job.slug, - version: run.version.version, - }, - run: { + static async enqueue( + run: JobRun & { + version: JobVersion & { + environment: RuntimeEnvironment; + concurrencyLimitGroup?: ConcurrencyLimitGroup | null; + }; + }, + priority: RunExecutionPriority, + tx: PrismaClientOrTransaction, + options: { + runAt?: Date; + skipRetrying?: boolean; + } = {} + ) { + return await executionWorker.enqueue( + "performRunExecutionV3", + { id: run.id, - isTest: run.isTest, + reason: "EXECUTE_JOB", }, - environment: { - id: run.environment.id, - slug: run.environment.slug, - type: run.environment.type, - }, - organization: { - id: run.organization.id, - slug: run.organization.slug, - title: run.organization.title, - }, - account: run.externalAccount - ? { - id: run.externalAccount.identifier, - metadata: run.externalAccount.metadata, - } - : undefined, - }); - - if (!response) { - return await this.#failRunExecution(this.#prismaClient, "PREPROCESS", run, { - message: "Could not connect to the endpoint", - }); - } - - if (!response.ok) { - return await this.#failRunExecution(this.#prismaClient, "PREPROCESS", run, { - message: `Endpoint responded with ${response.status} status code`, - }); - } - - const rawBody = await response.text(); - const safeBody = safeJsonZodParse(parser, rawBody); - - if (!safeBody) { - return await this.#failRunExecution(this.#prismaClient, "PREPROCESS", run, { - message: "Endpoint responded with invalid JSON", - }); - } - - if (!safeBody.success) { - return await this.#failRunExecution(this.#prismaClient, "PREPROCESS", run, { - message: generateErrorMessage(safeBody.error.issues), - }); - } - - if (safeBody.data.abort) { - return this.#failRunExecution( - this.#prismaClient, - "PREPROCESS", - run, - { message: "Endpoint aborted the run" }, - "ABORTED" - ); - } else { - await $transaction(this.#prismaClient, async (tx) => { - await tx.jobRun.update({ - where: { - id: run.id, - }, - data: { - status: "STARTED", - startedAt: new Date(), - properties: safeBody.data.properties, - forceYieldImmediately: false, - }, - }); - - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); - }); - } + { + tx, + runAt: options.runAt, + jobKey: `job_run:EXECUTE_JOB:${run.id}`, + maxAttempts: options.skipRetrying ? env.DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS : undefined, + flags: executionRateLimiter?.flagsForRun(run, run.version) ?? [], + priority: priority === "initial" ? 0 : -1, + } + ); } + + static async dequeue(run: JobRun, tx: PrismaClientOrTransaction) { + await executionWorker.dequeue(`job_run:EXECUTE_JOB:${run.id}`, { + tx, + }); + } + async #executeJob(run: FoundRun, input: PerformRunExecutionV3Input, driftInMs: number = 0) { try { - const { isRetry, resumeTaskId } = input; - - if (run.status === "CANCELED") { - await this.#cancelExecution(run); + if (isRunCompleted(run.status)) { return; } - try { - if ( - typeof process.env.BLOCKED_ORGS === "string" && - process.env.BLOCKED_ORGS.includes(run.organizationId) - ) { - logger.debug("Skipping execution for blocked org", { - orgId: run.organizationId, - }); - - await this.#prismaClient.jobRun.update({ - where: { - id: run.id, - }, - data: { - status: "CANCELED", - completedAt: new Date(), - }, - }); - - return; - } - } catch (e) {} - const client = new EndpointApi(run.environment.apiKey, run.endpoint.url); const event = eventRecordToApiJson(run.event); const startedAt = new Date(); - const { executionCount } = await this.#prismaClient.jobRun.update({ - where: { - id: run.id, - }, - data: { - status: run.status === "QUEUED" ? "STARTED" : run.status, - startedAt: run.startedAt ?? new Date(), - executionCount: { - increment: 1, - }, - }, - select: { - executionCount: true, - }, - }); - const connections = await resolveRunConnections(run.runConnections); if (!connections.success) { - return this.#failRunExecution(this.#prismaClient, "EXECUTE_JOB", run, { + return this.#failRunExecution(this.#prismaClient, run, { message: `Could not resolve all connections for run ${run.id}. This should not happen`, }); } - let resumedTask: Task | undefined; - - if (resumeTaskId) { - resumedTask = - (await this.#prismaClient.task.findUnique({ - where: { - id: resumeTaskId, - }, - })) ?? undefined; - - if (resumedTask) { - resumedTask = await this.#prismaClient.task.update({ - where: { - id: resumeTaskId, - }, - data: { - status: resumedTask.noop ? "COMPLETED" : "RUNNING", - completedAt: resumedTask.noop ? new Date() : undefined, - }, - }); - } - } - const sourceContext = RunSourceContextSchema.safeParse(run.event.sourceContext); const executionBody = await this.#createExecutionBody( run, - [run.tasks, resumedTask].flat().filter(Boolean), + run.tasks, startedAt, - isRetry, + false, connections.auth, event, sourceContext.success ? sourceContext.data : undefined ); - forceYieldCoordinator.registerRun(run.id); + await this.#prismaClient.jobRun.update({ + where: { + id: run.id, + }, + data: { + status: "EXECUTING", + }, + }); await createExecutionEvent({ eventType: "start", @@ -284,8 +178,12 @@ export class PerformRunExecutionV3Service { projectId: run.projectId, jobId: run.jobId, runId: run.id, + concurrencyLimitGroupId: run.version.concurrencyLimitGroupId, }); + forceYieldCoordinator.registerRun(run.id); + + // TODO: add the ability to abort the execution from any server using Redis pub/sub const { response, parser, errorParser, headersParser, durationInMs } = await client.executeJobRequest(executionBody); @@ -298,12 +196,13 @@ export class PerformRunExecutionV3Service { projectId: run.projectId, jobId: run.jobId, runId: run.id, + concurrencyLimitGroupId: run.version.concurrencyLimitGroupId, }); forceYieldCoordinator.deregisterRun(run.id); if (!response) { - return await this.#failRunExecutionWithRetry({ + return await this.#failRunExecutionWithRetry(run, input.lastAttempt, { message: `Connection could not be established to the endpoint (${run.endpoint.url})`, }); } @@ -393,14 +292,9 @@ export class PerformRunExecutionV3Service { if (errorBody && errorBody.success) { // Only retry if the error isn't a 4xx if (response.status >= 400 && response.status <= 499) { - return await this.#failRunExecution( - this.#prismaClient, - "EXECUTE_JOB", - run, - errorBody.data - ); + return await this.#failRunExecution(this.#prismaClient, run, errorBody.data); } else { - return await this.#failRunExecutionWithRetry(errorBody.data); + return await this.#failRunExecutionWithRetry(run, input.lastAttempt, errorBody.data); } } @@ -408,7 +302,6 @@ export class PerformRunExecutionV3Service { if (response.status >= 400 && response.status <= 499 && response.status !== 408) { return await this.#failRunExecution( this.#prismaClient, - "EXECUTE_JOB", run, { message: `Endpoint responded with ${response.status} status code`, @@ -423,11 +316,10 @@ export class PerformRunExecutionV3Service { this.#prismaClient, run, input, - durationInMs, - executionCount + durationInMs ); } else { - return await this.#failRunExecutionWithRetry({ + return await this.#failRunExecutionWithRetry(run, input.lastAttempt, { message: `Endpoint responded with ${response.status} status code`, }); } @@ -439,7 +331,6 @@ export class PerformRunExecutionV3Service { if (!safeBody) { return await this.#failRunExecution( this.#prismaClient, - "EXECUTE_JOB", run, { message: "Endpoint responded with invalid JSON", @@ -452,7 +343,6 @@ export class PerformRunExecutionV3Service { if (!safeBody.success) { return await this.#failRunExecution( this.#prismaClient, - "EXECUTE_JOB", run, { message: generateErrorMessage(safeBody.error.issues), @@ -491,7 +381,6 @@ export class PerformRunExecutionV3Service { break; } case "CANCELED": { - await this.#cancelExecution(run); break; } case "UNRESOLVED_AUTH_ERROR": { @@ -644,6 +533,9 @@ export class PerformRunExecutionV3Service { executionDuration: { increment: durationInMs, }, + executionCount: { + increment: 1, + }, }, }); @@ -661,17 +553,18 @@ export class PerformRunExecutionV3Service { run: FoundRun, data: RunJobResumeWithTask, durationInMs: number, - executionCount: number = 1 + executionCountIncrement: number = 1 ) { return await $transaction(this.#prismaClient, async (tx) => { await tx.jobRun.update({ where: { id: run.id }, data: { + status: "WAITING_TO_CONTINUE", executionDuration: { increment: durationInMs, }, executionCount: { - increment: executionCount, + increment: executionCountIncrement, }, }, }); @@ -744,7 +637,6 @@ export class PerformRunExecutionV3Service { case "ERROR": { return await this.#failRunExecution( this.#prismaClient, - "EXECUTE_JOB", run, childError.error ?? undefined, "FAILURE", @@ -754,7 +646,6 @@ export class PerformRunExecutionV3Service { case "INVALID_PAYLOAD": { return await this.#failRunExecution( this.#prismaClient, - "EXECUTE_JOB", run, childError.errors, "INVALID_PAYLOAD", @@ -774,7 +665,6 @@ export class PerformRunExecutionV3Service { case "UNRESOLVED_AUTH_ERROR": { return await this.#failRunExecution( this.#prismaClient, - "EXECUTE_JOB", run, childError.issues, "UNRESOLVED_AUTH", @@ -805,14 +695,7 @@ export class PerformRunExecutionV3Service { }); } - await this.#failRunExecution( - tx, - "EXECUTE_JOB", - execution, - data.error ?? undefined, - "FAILURE", - durationInMs - ); + await this.#failRunExecution(tx, execution, data.error ?? undefined, "FAILURE", durationInMs); }); } @@ -822,14 +705,7 @@ export class PerformRunExecutionV3Service { durationInMs: number ) { return await $transaction(this.#prismaClient, async (tx) => { - await this.#failRunExecution( - tx, - "EXECUTE_JOB", - execution, - data.issues, - "UNRESOLVED_AUTH", - durationInMs - ); + await this.#failRunExecution(tx, execution, data.issues, "UNRESOLVED_AUTH", durationInMs); }); } @@ -839,14 +715,7 @@ export class PerformRunExecutionV3Service { durationInMs: number ) { return await $transaction(this.#prismaClient, async (tx) => { - await this.#failRunExecution( - tx, - "EXECUTE_JOB", - execution, - data.errors, - "INVALID_PAYLOAD", - durationInMs - ); + await this.#failRunExecution(tx, execution, data.errors, "INVALID_PAYLOAD", durationInMs); }); } @@ -860,7 +729,6 @@ export class PerformRunExecutionV3Service { if (run.yieldedExecutions.length + 1 > MAX_RUN_YIELDED_EXECUTIONS) { return await this.#failRunExecution( tx, - "EXECUTE_JOB", run, { message: `Run has yielded too many times, the maximum is ${MAX_RUN_YIELDED_EXECUTIONS}`, @@ -875,6 +743,7 @@ export class PerformRunExecutionV3Service { id: run.id, }, data: { + status: "WAITING_TO_EXECUTE", executionDuration: { increment: durationInMs, }, @@ -892,9 +761,7 @@ export class PerformRunExecutionV3Service { }, }); - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(run, tx); }); } @@ -910,6 +777,7 @@ export class PerformRunExecutionV3Service { id: run.id, }, data: { + status: "WAITING_TO_EXECUTE", executionDuration: { increment: durationInMs, }, @@ -933,9 +801,7 @@ export class PerformRunExecutionV3Service { }, }); - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(run, tx); }); } @@ -981,9 +847,7 @@ export class PerformRunExecutionV3Service { output: data.output ? (JSON.parse(data.output) as any) : undefined, }); - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(run, tx); }); } @@ -1035,6 +899,7 @@ export class PerformRunExecutionV3Service { status: "WAITING", run: { update: { + status: "WAITING_TO_CONTINUE", executionDuration: { increment: durationInMs, }, @@ -1054,8 +919,7 @@ export class PerformRunExecutionV3Service { prisma: PrismaClientOrTransaction, run: FoundRun, input: PerformRunExecutionV3Input, - durationInMs: number, - executionCount: number + durationInMs: number ) { await $transaction(prisma, async (tx) => { const executionDuration = run.executionDuration + durationInMs; @@ -1064,7 +928,6 @@ export class PerformRunExecutionV3Service { if (executionDuration >= run.organization.maximumExecutionTimePerRunInMs) { await this.#failRunExecution( tx, - "EXECUTE_JOB", run, { message: `Execution timed out after ${ @@ -1112,7 +975,6 @@ export class PerformRunExecutionV3Service { await this.#failRunExecution( tx, - "EXECUTE_JOB", run, { message: `Function timeout detected in ${ @@ -1147,102 +1009,73 @@ export class PerformRunExecutionV3Service { }); // The run has timed out, so we need to enqueue a new execution - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(run, tx); }); } - async #failRunExecutionWithRetry(output: Record): Promise { + async #failRunExecutionWithRetry( + run: FoundRun, + lastAttempt: boolean, + output: Record + ): Promise { + if (lastAttempt) { + return await this.#failRunExecution(this.#prismaClient, run, output); + } + + await this.#prismaClient.jobRun.update({ + where: { id: run.id }, + data: { + status: "WAITING_TO_EXECUTE", + }, + }); + throw new Error(JSON.stringify(output)); } async #failRunExecution( prisma: PrismaClientOrTransaction, - reason: "EXECUTE_JOB" | "PREPROCESS", run: FoundRun, output: Record, status: "FAILURE" | "ABORTED" | "TIMED_OUT" | "UNRESOLVED_AUTH" | "INVALID_PAYLOAD" = "FAILURE", durationInMs: number = 0 ): Promise { await $transaction(prisma, async (tx) => { - switch (reason) { - case "EXECUTE_JOB": { - // If the execution is an EXECUTE_JOB reason, we need to fail the run - await tx.jobRun.update({ - where: { id: run.id }, - data: { - completedAt: new Date(), - status, - output, - executionDuration: { - increment: durationInMs, - }, - tasks: { - updateMany: { - where: { - status: { - in: ["WAITING", "RUNNING", "PENDING"], - }, - }, - data: { - status: status === "TIMED_OUT" ? "CANCELED" : "ERRORED", - completedAt: new Date(), - }, + // If the execution is an EXECUTE_JOB reason, we need to fail the run + await tx.jobRun.update({ + where: { id: run.id }, + data: { + completedAt: new Date(), + status, + output, + executionDuration: { + increment: durationInMs, + }, + tasks: { + updateMany: { + where: { + status: { + in: ["WAITING", "RUNNING", "PENDING"], }, }, - forceYieldImmediately: false, - }, - }); - - await workerQueue.enqueue( - "deliverRunSubscriptions", - { - id: run.id, - }, - { tx } - ); - - break; - } - case "PREPROCESS": { - // If the status is ABORTED, we need to fail the run - if (status === "ABORTED") { - await tx.jobRun.update({ - where: { id: run.id }, data: { + status: status === "TIMED_OUT" ? "CANCELED" : "ERRORED", completedAt: new Date(), - status, - output, }, - }); - - break; - } - - await tx.jobRun.update({ - where: { - id: run.id, }, - data: { - status: "STARTED", - startedAt: new Date(), - }, - }); + }, + forceYieldImmediately: false, + }, + }); - await enqueueRunExecutionV3(run, tx, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); - - break; - } - } + await workerQueue.enqueue( + "deliverRunSubscriptions", + { + id: run.id, + }, + { tx } + ); }); } - - async #cancelExecution(run: FoundRun) { - return; - } } function prepareNoOpTasksBloomFilter(possibleTasks: FoundTask[]): string { diff --git a/apps/webapp/app/services/runs/resumeRun.server.ts b/apps/webapp/app/services/runs/resumeRun.server.ts new file mode 100644 index 000000000..6b99035ff --- /dev/null +++ b/apps/webapp/app/services/runs/resumeRun.server.ts @@ -0,0 +1,158 @@ +import { JobRun, RuntimeEnvironmentType } from "@trigger.dev/database"; +import { PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server"; +import { workerQueue } from "../worker.server"; +import { PerformRunExecutionV3Service, RunExecutionPriority } from "./performRunExecutionV3.server"; + +type FoundRun = NonNullable>>; + +export class ResumeRunService { + #prismaClient: PrismaClient; + + constructor(prismaClient: PrismaClient = prisma) { + this.#prismaClient = prismaClient; + } + + public async call(id: string) { + const run = await findRun(this.#prismaClient, id); + + if (!run) { + return; + } + + switch (run.status) { + case "ABORTED": + case "CANCELED": + case "FAILURE": + case "INVALID_PAYLOAD": + case "SUCCESS": + case "TIMED_OUT": + case "UNRESOLVED_AUTH": { + return; + } + case "QUEUED": { + await this.#resumeQueuedRun(run); + break; + } + case "WAITING_TO_EXECUTE": { + await this.#executeRun(run, "resume"); + break; + } + case "WAITING_TO_CONTINUE": { + await this.#resumeWaitingToContinueRun(run); + break; + } + case "STARTED": { + await this.#resumeStartedRun(run); + break; + } + case "PENDING": + case "PREPROCESSING": { + await this.#resumePendingRun(run); + break; + } + case "EXECUTING": { + throw new Error("Cannot resume a run that is currently executing"); + } + case "WAITING_ON_CONNECTIONS": { + throw new Error("Cannot resume a run that is waiting on connections"); + } + default: { + const _exhaustiveCheck: never = run.status; + throw new Error(`Non-exhaustive match for value: ${run.status}`); + } + } + } + + async #resumeQueuedRun(run: FoundRun) { + await this.#prismaClient.jobRun.update({ + where: { + id: run.id, + }, + data: { + startedAt: run.startedAt ?? new Date(), + }, + }); + + await this.#executeRun(run, "initial"); + } + + async #resumeStartedRun(run: FoundRun) { + await this.#prismaClient.jobRun.update({ + where: { + id: run.id, + }, + data: { + status: "WAITING_TO_EXECUTE", + }, + }); + + await this.#executeRun(run, "initial"); + } + + async #resumeWaitingToContinueRun(run: FoundRun) { + await this.#prismaClient.jobRun.update({ + where: { + id: run.id, + }, + data: { + status: "WAITING_TO_EXECUTE", + }, + }); + + await this.#executeRun(run, "resume"); + } + + async #resumePendingRun(run: FoundRun) { + await this.#prismaClient.jobRun.update({ + where: { + id: run.id, + }, + data: { + status: "QUEUED", + startedAt: new Date(), + }, + }); + + await this.#executeRun(run, "initial"); + } + + async #executeRun(run: FoundRun, priority: RunExecutionPriority) { + await PerformRunExecutionV3Service.enqueue(run, priority, this.#prismaClient, { + skipRetrying: run.version.environment.type === RuntimeEnvironmentType.DEVELOPMENT, + }); + } + + static async enqueue(run: JobRun, tx: PrismaClientOrTransaction, runAt?: Date) { + return await workerQueue.enqueue( + "resumeRun", + { + id: run.id, + }, + { + tx, + runAt: runAt ?? run.createdAt, + jobKey: `run_resume:${run.id}`, + } + ); + } + + static async dequeue(run: JobRun, tx: PrismaClientOrTransaction) { + await workerQueue.dequeue(`run_resume:${run.id}`, { + tx, + }); + } +} + +async function findRun(prisma: PrismaClientOrTransaction, id: string) { + return await prisma.jobRun.findUnique({ + where: { id }, + include: { + version: { + include: { + environment: true, + concurrencyLimitGroup: true, + }, + }, + }, + }); +} diff --git a/apps/webapp/app/services/runs/startRun.server.ts b/apps/webapp/app/services/runs/startRun.server.ts index 4ad39970b..62f9f0ed1 100644 --- a/apps/webapp/app/services/runs/startRun.server.ts +++ b/apps/webapp/app/services/runs/startRun.server.ts @@ -1,13 +1,13 @@ import { - RuntimeEnvironmentType, type ConnectionType, type Integration, type IntegrationConnection, } from "@trigger.dev/database"; import type { PrismaClient, PrismaClientOrTransaction } from "~/db.server"; -import { prisma } from "~/db.server"; -import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server"; +import { $transaction, prisma } from "~/db.server"; import { workerQueue } from "../worker.server"; +import { ResumeRunService } from "./resumeRun.server"; +import { createHash } from "node:crypto"; type FoundRun = NonNullable>>; type RunConnectionsByKey = Awaited>; @@ -59,23 +59,24 @@ export class StartRunService { : undefined ) .filter(Boolean); + const lockId = jobIdToLockId(run.jobId); - const updateRun = async () => { - if (run.preprocess) { - // Start the jobRun and increment the jobCount - return await this.#prismaClient.jobRun.update({ - where: { id }, - data: { - status: "PREPROCESSING", - runConnections: { - create: createRunConnections, - }, - }, + await $transaction( + this.#prismaClient, + async (tx) => { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(${lockId})`; + + const counter = await tx.jobCounter.upsert({ + where: { jobId: run.jobId }, + update: { lastNumber: { increment: 1 } }, + create: { jobId: run.jobId, lastNumber: 1 }, + select: { lastNumber: true }, }); - } else { - return await this.#prismaClient.jobRun.update({ + + const updatedRun = await this.#prismaClient.jobRun.update({ where: { id }, data: { + number: counter.lastNumber, status: "QUEUED", queuedAt: new Date(), runConnections: { @@ -83,14 +84,11 @@ export class StartRunService { }, }, }); - } - }; - const updatedRun = await updateRun(); - - await enqueueRunExecutionV3(updatedRun, this.#prismaClient, { - skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(updatedRun, tx); + }, + { timeout: 60000 } + ); } async #handleMissingConnections(id: string, runConnectionsByKey: RunConnectionsByKey) { @@ -237,3 +235,8 @@ async function createRunConnections(tx: PrismaClientOrTransaction, run: FoundRun function hasMissingConnections(runConnectionsByKey: RunConnectionsByKey) { return Object.values(runConnectionsByKey).some((connection) => connection.result === "missing"); } + +function jobIdToLockId(jobId: string): number { + // Convert jobId to a unique lock identifier + return parseInt(createHash("sha256").update(jobId).digest("hex").slice(0, 8), 16); +} diff --git a/apps/webapp/app/services/tasks/resumeTask.server.ts b/apps/webapp/app/services/tasks/resumeTask.server.ts index c47b04791..03a59b473 100644 --- a/apps/webapp/app/services/tasks/resumeTask.server.ts +++ b/apps/webapp/app/services/tasks/resumeTask.server.ts @@ -1,8 +1,7 @@ import { PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server"; -import { workerQueue } from "../worker.server"; -import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server"; -import { RuntimeEnvironmentType } from "@trigger.dev/database"; import { logger } from "../logger.server"; +import { ResumeRunService } from "../runs/resumeRun.server"; +import { workerQueue } from "../worker.server"; type FoundTask = Awaited>; @@ -81,9 +80,7 @@ export class ResumeTaskService { } } - await enqueueRunExecutionV3(task.run, this.#prismaClient, { - skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT, - }); + await ResumeRunService.enqueue(task.run, this.#prismaClient); } public static async enqueue(id: string, runAt?: Date, tx?: PrismaClientOrTransaction) { diff --git a/apps/webapp/app/services/tasks/runTask.server.ts b/apps/webapp/app/services/tasks/runTask.server.ts index eebce56cd..f0a8edd28 100644 --- a/apps/webapp/app/services/tasks/runTask.server.ts +++ b/apps/webapp/app/services/tasks/runTask.server.ts @@ -71,7 +71,7 @@ export class RunTaskService { status = "CANCELED"; } else { status = - delayUntilInFuture || callbackEnabled || taskBody.trigger + delayUntilInFuture || callbackEnabled ? "WAITING" : taskBody.noop ? "COMPLETED" @@ -180,7 +180,7 @@ export class RunTaskService { if (existingTask) { if (existingTask.status === "CANCELED") { const existingTaskStatus = - delayUntilInFuture || callbackEnabled || taskBody.trigger + delayUntilInFuture || callbackEnabled ? "WAITING" : taskBody.noop ? "COMPLETED" diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts index eae2ba59e..92ac8a1ca 100644 --- a/apps/webapp/app/services/worker.server.ts +++ b/apps/webapp/app/services/worker.server.ts @@ -26,6 +26,8 @@ import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.s import { ResumeTaskService } from "./tasks/resumeTask.server"; import { ExpireDispatcherService } from "./dispatchers/expireDispatcher.server"; import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralEventDispatcher.server"; +import { ResumeRunService } from "./runs/resumeRun.server"; +import { executionRateLimiter } from "./runExecutionRateLimiter.server"; import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server"; const workerCatalog = { @@ -97,6 +99,9 @@ const workerCatalog = { expireDispatcher: z.object({ id: z.string(), }), + resumeRun: z.object({ + id: z.string(), + }), }; const executionWorkerCatalog = { @@ -225,7 +230,6 @@ function getWorkerQueue() { "events.invokeDispatcher": { priority: 0, // smaller number = higher priority maxAttempts: 6, - queueName: (payload) => `dispatcher:${payload.id}`, // use a queue for a dispatcher so runs are created sequentially handler: async (payload, job) => { const service = new InvokeDispatcherService(); @@ -412,6 +416,15 @@ function getWorkerQueue() { handler: async (payload) => { const service = new ExpireDispatcherService(); + return await service.call(payload.id); + }, + }, + resumeRun: { + priority: 0, + maxAttempts: 10, + handler: async (payload, job) => { + const service = new ResumeRunService(); + return await service.call(payload.id); }, }, @@ -433,6 +446,7 @@ function getExecutionWorkerQueue() { }, shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT, schema: executionWorkerCatalog, + rateLimiter: executionRateLimiter, tasks: { performRunExecutionV2: { priority: 0, // smaller number = higher priority @@ -445,6 +459,7 @@ function getExecutionWorkerQueue() { reason: payload.reason, resumeTaskId: payload.resumeTaskId, isRetry: payload.isRetry, + lastAttempt: job.max_attempts === job.attempts, }); }, }, @@ -461,6 +476,7 @@ function getExecutionWorkerQueue() { id: payload.id, reason: payload.reason, isRetry: false, + lastAttempt: job.max_attempts === job.attempts, }, driftInMs ); diff --git a/apps/webapp/app/utils.ts b/apps/webapp/app/utils.ts index 5a50c6d84..bf8bbb621 100644 --- a/apps/webapp/app/utils.ts +++ b/apps/webapp/app/utils.ts @@ -158,19 +158,10 @@ export const obfuscateApiKey = (apiKey: string) => { return `${prefix}_${slug}_${"*".repeat(secretPart.length)}`; }; -export function appEnvTitleTag(appEnv?: "test" | "production" | "development" | "staging"): string { - if (!appEnv) { +export function appEnvTitleTag(appEnv?: string): string { + if (!appEnv || appEnv === "production") { return ""; } - switch (appEnv) { - case "test": - return " (test)"; - case "production": - return ""; - case "development": - return " (dev)"; - case "staging": - return " (staging)"; - } + return ` (${appEnv})` } diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index 8683d4ff5..a9417fb59 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -126,6 +126,10 @@ export function projectPath(organization: OrgForPath, project: ProjectForPath) { return `/orgs/${organizationParam(organization)}/projects/${projectParam(project)}`; } +export function projectRunsPath(organization: OrgForPath, project: ProjectForPath) { + return `${projectPath(organization, project)}/runs`; +} + export function projectSetupPath(organization: OrgForPath, project: ProjectForPath) { return `${projectPath(organization, project)}/setup`; } diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 82c60d8f0..03952397d 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -84,6 +84,7 @@ "highlight.run": "^7.3.4", "humanize-duration": "^3.27.3", "intl-parse-accept-language": "^1.0.0", + "ioredis": "^5.3.2", "isbot": "^3.6.5", "jsonpointer": "^5.0.1", "lodash.omit": "^4.5.0", diff --git a/docker/dev-compose.yml b/docker/dev-compose.yml index 015139b68..642510e28 100644 --- a/docker/dev-compose.yml +++ b/docker/dev-compose.yml @@ -2,6 +2,7 @@ version: "3" volumes: database-data: + redis-data: networks: app_network: @@ -42,3 +43,21 @@ services: PORT: 3030 networks: - app_network + + redis: + container_name: redis + image: redis:7 + restart: always + volumes: + - redis-data:/data + networks: + - app_network + ports: + - 6379:6379 + + redisinsight: + image: redislabs/redisinsight:latest + ports: + - "8001:8001" + volumes: + - redis-data:/redisinsight diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 41e935745..7c6f6cc58 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,6 +3,7 @@ version: "3" volumes: database-data: pgadmin-data: + redis-data: networks: app_network: @@ -41,3 +42,21 @@ services: - 5480:80 depends_on: - database + + redis: + container_name: redis + image: redis:7 + restart: always + volumes: + - redis-data:/data + networks: + - app_network + ports: + - 6379:6379 + + redisinsight: + image: redislabs/redisinsight:latest + ports: + - "8001:8001" + volumes: + - redis-data:/redisinsight diff --git a/docs/_snippets/jobs/options.mdx b/docs/_snippets/jobs/options.mdx index 49298360c..31a60f3aa 100644 --- a/docs/_snippets/jobs/options.mdx +++ b/docs/_snippets/jobs/options.mdx @@ -36,6 +36,9 @@ The `enabled` property is an optional property that specifies whether the Job is enabled or not. The Job will be enabled by default if you omit this property. When a job is disabled, no new runs will be triggered or resumed. In progress runs will continue to run until they are finished or delayed by using `io.wait`. + + The `concurrencyLimit` property is an optional property that specifies the maximum number of concurrent run executions. If this property is omitted, the job can potentially use up the full concurrency of an environment. You can also create a limit on a group of jobs by defining a [ConcurrencyLimit](/sdk/triggerclient/instancemethods/concurrency-limit) object. + The `onSuccess` property is an optional property that specifies a callback function to run when the Job finishes successfully. The callback function receives a [Run Notification](/sdk/run-notification) object as it's only parameter. diff --git a/docs/documentation/concepts/limits.mdx b/docs/documentation/concepts/limits.mdx index cb06a5ccd..c7b644564 100644 --- a/docs/documentation/concepts/limits.mdx +++ b/docs/documentation/concepts/limits.mdx @@ -16,7 +16,7 @@ The following limits apply to the Trigger.dev Cloud service and users of the sel | Connected Integrations | Up to 50 | Up to 1000 | Custom | | Task Output Size | 3MB | 3MB | 3MB | | [Tasks per Run](#tasks-per-runs) | Up to 250 | Up to 1000 | Custom | -| [Concurrent Run Executions](#concurrent-run-executions) | Up to 10 | Up to 10 | Custom | +| [Concurrent Run Executions per Environment](#concurrent-run-executions) | Up to 10 | Up to 100 | Custom | | [Maximum Task Duration](#maximum-task-duration) | < 2m | < 2m | < Deployment Grace Period | | [Maximum Run Execution Duration](#maximum-total-run-execution-duration) | up to 15m | up to 2 hrs | Custom | | [Yielded Executions per Run](#yielded-executions-per-run) | Up to 100 | Up to 100 | Custom | @@ -88,6 +88,51 @@ This does not include runs that are waiting for a [io.wait()](/sdk/io/wait) to c Going over this limit does not abort or cancel runs, but it will prevent new run executions until the number of concurrent executions drops below the limit. +You can limit the execution concurrency of a specific job like so: + +```ts +client.defineJob({ + id: `test-job-1`, + name: `Test Job 1`, + version: "1.0.0", + trigger: eventTrigger({ + name: "test", + }), + concurrencyLimit: 5, // Limit this job to 5 concurrent executions +}); +``` + +Alternatively, you can limit a group of jobs concurrency limit by defining a concurrency limit and passing it to the `defineJobs()` method: + +```ts +const concurrencyLimit = client.defineConcurrencyLimit({ + id: `test-shared`, + limit: 5, // Limit all jobs in this group to 5 concurrent executions +}); + +client.defineJob({ + id: `test-job-1`, + name: `Test Job 1`, + version: "1.0.0", + trigger: eventTrigger({ + name: "test", + }), + concurrencyLimit, +}); + +client.defineJob({ + id: `test-job-2`, + name: `Test Job 2`, + version: "1.0.0", + trigger: eventTrigger({ + name: "test", + }), + concurrencyLimit, +}); +``` + +The two jobs above will share the same concurrency limit, so between them they can only have 5 concurrent executions. + ### Maximum Task Duration The Maximum Task Duration is the maximum amount of time a single Task can run for. This limit is partly enforced by the Trigger.dev server, but also by the execution runtime of your deployed serverless function. diff --git a/docs/documentation/concepts/runs.mdx b/docs/documentation/concepts/runs.mdx index e65ebb870..95c90c001 100644 --- a/docs/documentation/concepts/runs.mdx +++ b/docs/documentation/concepts/runs.mdx @@ -24,7 +24,7 @@ client.defineJob({ run: async (payload, io, ctx) => { // 2. Regular code and Tasks // 3. Optionally return data from run execution - return { status: 'success' } + return { status: "success" }; }, }); ``` @@ -64,6 +64,52 @@ A few things you can do with `io`: The `context` object gives you access to information about the current Run, Job, Environment, Organization and Event. [View the full reference](/sdk/context) for `context`. +## Run Statuses + +### Pending + +The run has been created but has not started yet. This is the initial status of a run. + +### Queued + +The run is waiting to be executed. Runs can be queued because of [Run Execution Concurrency Limits](/documentation/concepts/limits#concurrent-run-executions) + +### Waiting on Connections + +If a run depends on a hosted integration, it will be in this status until the integration is ready. + +### Executing + +The run is currently executing. This means that the run function is running. + +### Waiting + +The run is waiting, either because of a call to `io.wait()` or because a task failed and will be retried at some point in the future. Runs in this state don't count towards concurrency limits. + +### Failed + +The run failed. This can happen if the run function throws an error or if a task fails and the run is not configured to retry. + +### Completed + +The run completed successfully. This means that the run function finished executing and all tasks completed successfully. + +### Cancelled + +The run was cancelled. This can happen if the run is cancelled manually. + +### Timed Out + +The run timed out. This can happen if the run exceeds the maximum run duration, or if we receive a serverless function execution timed out response when hitting your endpoint repeatedly with no new task creation. + +### Invalid Payload + +The run failed because the payload was invalid. + +### Unresolved Auth + +The run failed because the auth data could not be resolved when using a custom Auth Resolver. + ## References diff --git a/docs/mint.json b/docs/mint.json index cc99e1ade..b48bd8080 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -1,7 +1,9 @@ { "$schema": "https://mintlify.com/schema.json", "name": "Trigger.dev", - "openapi": ["/openapi.yml"], + "openapi": [ + "/openapi.yml" + ], "logo": { "dark": "/logo/dark.png", "light": "/logo/light.png", @@ -253,7 +255,10 @@ "pages": [ { "group": "Airtable", - "pages": ["integrations/apis/airtable", "integrations/apis/airtable-tasks"] + "pages": [ + "integrations/apis/airtable", + "integrations/apis/airtable-tasks" + ] }, { "group": "GitHub", @@ -279,16 +284,25 @@ }, { "group": "Plain", - "pages": ["integrations/apis/plain", "integrations/apis/plain-tasks"] + "pages": [ + "integrations/apis/plain", + "integrations/apis/plain-tasks" + ] }, "integrations/apis/replicate", { "group": "SendGrid", - "pages": ["integrations/apis/sendgrid", "integrations/apis/sendgrid-tasks"] + "pages": [ + "integrations/apis/sendgrid", + "integrations/apis/sendgrid-tasks" + ] }, { "group": "Resend", - "pages": ["integrations/apis/resend", "integrations/apis/resend-tasks"] + "pages": [ + "integrations/apis/resend", + "integrations/apis/resend-tasks" + ] }, { "group": "Shopify", @@ -300,7 +314,10 @@ }, { "group": "Slack", - "pages": ["integrations/apis/slack", "integrations/apis/slack-tasks"] + "pages": [ + "integrations/apis/slack", + "integrations/apis/slack-tasks" + ] }, "integrations/apis/stripe", { @@ -344,6 +361,7 @@ "sdk/triggerclient/instancemethods/define-dynamic-trigger", "sdk/triggerclient/instancemethods/define-dynamic-schedule", "sdk/triggerclient/instancemethods/define-auth-resolver", + "sdk/triggerclient/instancemethods/concurrency-limit", "sdk/triggerclient/instancemethods/on" ] } @@ -388,7 +406,10 @@ "sdk/dynamictrigger/constructor", { "group": "Instance methods", - "pages": ["sdk/dynamictrigger/register", "sdk/dynamictrigger/unregister"] + "pages": [ + "sdk/dynamictrigger/register", + "sdk/dynamictrigger/unregister" + ] } ] }, @@ -399,7 +420,10 @@ "sdk/dynamicschedule/constructor", { "group": "Instance methods", - "pages": ["sdk/dynamicschedule/register", "sdk/dynamicschedule/unregister"] + "pages": [ + "sdk/dynamicschedule/register", + "sdk/dynamicschedule/unregister" + ] } ] }, @@ -411,7 +435,9 @@ }, { "group": "HTTP Reference", - "pages": ["sdk/api-reference/events/create-an-event"] + "pages": [ + "sdk/api-reference/events/create-an-event" + ] }, { "group": "React SDK", @@ -425,7 +451,9 @@ }, { "group": "Overview", - "pages": ["examples/introduction"] + "pages": [ + "examples/introduction" + ] } ], "footerSocials": { @@ -438,4 +466,4 @@ "apiKey": "phc_hwYmedO564b3Ik8nhA4Csrb5SueY0EwFJWCbseGwWW" } } -} +} \ No newline at end of file diff --git a/docs/sdk/triggerclient/instancemethods/concurrency-limit.mdx b/docs/sdk/triggerclient/instancemethods/concurrency-limit.mdx new file mode 100644 index 000000000..48c8c1a17 --- /dev/null +++ b/docs/sdk/triggerclient/instancemethods/concurrency-limit.mdx @@ -0,0 +1,47 @@ +--- +title: "defineConcurrencyLimit()" +description: "Define a concurrency limit group to control the concurrency of your jobs." +--- + +You can control the concurrency of run executions for a group of jobs using a concurrency limit group. + + + +```ts example +const concurrencyLimit = client.defineConcurrencyLimit({ + id: `test-shared`, + limit: 5, // Limit all jobs in this group to 5 concurrent executions +}); + +client.defineJob({ + id: `test-job-1`, + name: `Test Job 1`, + version: "1.0.0", + trigger: eventTrigger({ + name: "test", + }), + concurrencyLimit, +}); + +client.defineJob({ + id: `test-job-2`, + name: `Test Job 2`, + version: "1.0.0", + trigger: eventTrigger({ + name: "test", + }), + concurrencyLimit, +}); +``` + + + +## Parameters + + + The ID of the concurrency limit group. + + + + The maximum number of concurrent executions allowed for this group. + diff --git a/packages/core/src/schemas/api.ts b/packages/core/src/schemas/api.ts index c09410b33..9a7882eae 100644 --- a/packages/core/src/schemas/api.ts +++ b/packages/core/src/schemas/api.ts @@ -273,6 +273,11 @@ export const QueueOptionsSchema = z.object({ export type QueueOptions = z.infer; +export const ConcurrencyLimitOptionsSchema = z.object({ + id: z.string(), + limit: z.number(), +}); + export const JobMetadataSchema = z.object({ id: z.string(), name: z.string(), @@ -284,6 +289,7 @@ export const JobMetadataSchema = z.object({ enabled: z.boolean(), startPosition: z.enum(["initial", "latest"]), preprocessRuns: z.boolean(), + concurrencyLimit: ConcurrencyLimitOptionsSchema.or(z.number().int().positive()).optional(), }); export type JobMetadata = z.infer; @@ -879,7 +885,6 @@ export const RunTaskOptionsSchema = z.object({ /** A No Operation means that the code won't be executed. This is used internally to implement features like [io.wait()](https://trigger.dev/docs/sdk/io/wait). */ noop: z.boolean().default(false), redact: RedactSchema.optional(), - trigger: TriggerMetadataSchema.optional(), parallel: z.boolean().optional(), }); diff --git a/packages/core/src/schemas/runs.ts b/packages/core/src/schemas/runs.ts index e62c1331b..8fc448b68 100644 --- a/packages/core/src/schemas/runs.ts +++ b/packages/core/src/schemas/runs.ts @@ -18,6 +18,9 @@ export const RunStatusSchema = z.union([ z.literal("CANCELED"), z.literal("UNRESOLVED_AUTH"), z.literal("INVALID_PAYLOAD"), + z.literal("EXECUTING"), + z.literal("WAITING_TO_CONTINUE"), + z.literal("WAITING_TO_EXECUTE"), ]); export const RunTaskSchema = z.object({ diff --git a/packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql b/packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql new file mode 100644 index 000000000..98fe1903c --- /dev/null +++ b/packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql @@ -0,0 +1,11 @@ +-- AlterEnum +-- This migration adds more than one value to an enum. +-- With PostgreSQL versions 11 and earlier, this is not possible +-- in a single migration. This can be worked around by creating +-- multiple migrations, each migration adding only one value to +-- the enum. + + +ALTER TYPE "JobRunStatus" ADD VALUE 'EXECUTING'; +ALTER TYPE "JobRunStatus" ADD VALUE 'WAITING_TO_CONTINUE'; +ALTER TYPE "JobRunStatus" ADD VALUE 'WAITING_TO_EXECUTE'; diff --git a/packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql b/packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql new file mode 100644 index 000000000..f8979aaa8 --- /dev/null +++ b/packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "JobRun" ALTER COLUMN "number" DROP NOT NULL; diff --git a/packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql b/packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql new file mode 100644 index 000000000..6f7b36264 --- /dev/null +++ b/packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql @@ -0,0 +1,7 @@ +-- CreateTable +CREATE TABLE "JobCounter" ( + "jobId" TEXT NOT NULL, + "lastNumber" INTEGER NOT NULL DEFAULT 0, + + CONSTRAINT "JobCounter_pkey" PRIMARY KEY ("jobId") +); diff --git a/packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql b/packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql new file mode 100644 index 000000000..8bc544ef5 --- /dev/null +++ b/packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql @@ -0,0 +1,10 @@ +-- This is an empty migration. +INSERT INTO + "JobCounter" ("jobId", "lastNumber") +SELECT + "jobId", + MAX(number) +FROM + "JobRun" +GROUP BY + "jobId"; \ No newline at end of file diff --git a/packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql b/packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql new file mode 100644 index 000000000..b04d0eefe --- /dev/null +++ b/packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql @@ -0,0 +1,30 @@ +-- AlterTable +ALTER TABLE "JobRun" ADD COLUMN "concurrencyLimitGroupId" TEXT; + +-- AlterTable +ALTER TABLE "JobVersion" ADD COLUMN "concurrencyLimit" INTEGER, +ADD COLUMN "concurrencyLimitGroupId" TEXT; + +-- CreateTable +CREATE TABLE "ConcurrencyLimitGroup" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "concurrencyLimit" INTEGER NOT NULL, + "environmentId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "ConcurrencyLimitGroup_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "ConcurrencyLimitGroup_environmentId_name_key" ON "ConcurrencyLimitGroup"("environmentId", "name"); + +-- AddForeignKey +ALTER TABLE "JobVersion" ADD CONSTRAINT "JobVersion_concurrencyLimitGroupId_fkey" FOREIGN KEY ("concurrencyLimitGroupId") REFERENCES "ConcurrencyLimitGroup"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ConcurrencyLimitGroup" ADD CONSTRAINT "ConcurrencyLimitGroup_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "JobRun" ADD CONSTRAINT "JobRun_concurrencyLimitGroupId_fkey" FOREIGN KEY ("concurrencyLimitGroupId") REFERENCES "ConcurrencyLimitGroup"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql b/packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql new file mode 100644 index 000000000..b59219304 --- /dev/null +++ b/packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql @@ -0,0 +1,17 @@ +-- DropForeignKey +ALTER TABLE "JobRun" DROP CONSTRAINT "JobRun_queueId_fkey"; + +-- DropForeignKey +ALTER TABLE "JobVersion" DROP CONSTRAINT "JobVersion_queueId_fkey"; + +-- AlterTable +ALTER TABLE "JobRun" ALTER COLUMN "queueId" DROP NOT NULL; + +-- AlterTable +ALTER TABLE "JobVersion" ALTER COLUMN "queueId" DROP NOT NULL; + +-- AddForeignKey +ALTER TABLE "JobVersion" ADD CONSTRAINT "JobVersion_queueId_fkey" FOREIGN KEY ("queueId") REFERENCES "JobQueue"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "JobRun" ADD CONSTRAINT "JobRun_queueId_fkey" FOREIGN KEY ("queueId") REFERENCES "JobQueue"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql b/packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql new file mode 100644 index 000000000..9bb2b98b7 --- /dev/null +++ b/packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql @@ -0,0 +1,11 @@ +/* + Warnings: + + - You are about to drop the column `concurrencyLimitGroupId` on the `JobRun` table. All the data in the column will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "JobRun" DROP CONSTRAINT "JobRun_concurrencyLimitGroupId_fkey"; + +-- AlterTable +ALTER TABLE "JobRun" DROP COLUMN "concurrencyLimitGroupId"; diff --git a/packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql b/packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql new file mode 100644 index 000000000..4529052e9 --- /dev/null +++ b/packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql @@ -0,0 +1,4 @@ +ALTER TABLE + "triggerdotdev_events"."run_executions" +ADD + COLUMN "concurrency_limit_group_id" text; \ No newline at end of file diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 810e76e14..a3ce71e17 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -328,6 +328,7 @@ model RuntimeEnvironment { scheduleSources ScheduleSource[] ExternalAccount ExternalAccount[] httpEndpointEnvironments TriggerHttpEndpointEnvironment[] + concurrencyLimitGroups ConcurrencyLimitGroup[] keyValueItems KeyValueItem[] webhookEnvironments WebhookEnvironment[] webhookRequestDeliveries WebhookRequestDelivery[] @@ -488,12 +489,16 @@ model JobVersion { project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String - queue JobQueue @relation(fields: [queueId], references: [id]) - queueId String + queue JobQueue? @relation(fields: [queueId], references: [id]) + queueId String? startPosition JobStartPosition @default(INITIAL) preprocessRuns Boolean @default(false) + concurrencyLimit Int? + concurrencyLimitGroup ConcurrencyLimitGroup? @relation(fields: [concurrencyLimitGroupId], references: [id]) + concurrencyLimitGroupId String? + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -532,6 +537,23 @@ model EventExample { @@unique([slug, jobVersionId]) } +model ConcurrencyLimitGroup { + id String @id @default(cuid()) + name String + + concurrencyLimit Int + + environment RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + environmentId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + jobVersion JobVersion[] + + @@unique([environmentId, name]) +} + model JobQueue { id String @id @default(cuid()) name String @@ -718,7 +740,7 @@ enum PayloadType { model JobRun { id String @id @default(cuid()) - number Int + number Int? internal Boolean @default(false) job Job @relation(fields: [jobId], references: [id], onDelete: Cascade, onUpdate: Cascade) @@ -742,8 +764,8 @@ model JobRun { project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String - queue JobQueue @relation(fields: [queueId], references: [id]) - queueId String + queue JobQueue? @relation(fields: [queueId], references: [id]) + queueId String? externalAccount ExternalAccount? @relation(fields: [externalAccountId], references: [id], onDelete: Cascade, onUpdate: Cascade) externalAccountId String? @@ -787,6 +809,9 @@ enum JobRunStatus { WAITING_ON_CONNECTIONS PREPROCESSING STARTED + EXECUTING + WAITING_TO_CONTINUE + WAITING_TO_EXECUTE SUCCESS FAILURE TIMED_OUT @@ -796,6 +821,11 @@ enum JobRunStatus { INVALID_PAYLOAD } +model JobCounter { + jobId String @id + lastNumber Int @default(0) +} + model JobRunAutoYieldExecution { id String @id @default(cuid()) diff --git a/packages/trigger-sdk/src/concurrencyLimit.ts b/packages/trigger-sdk/src/concurrencyLimit.ts new file mode 100644 index 000000000..98fdba209 --- /dev/null +++ b/packages/trigger-sdk/src/concurrencyLimit.ts @@ -0,0 +1,16 @@ +export type ConcurrencyLimitOptions = { + id: string; + limit: number; +}; + +export class ConcurrencyLimit { + constructor(private options: ConcurrencyLimitOptions) {} + + get id() { + return this.options.id; + } + + get limit() { + return this.options.limit; + } +} diff --git a/packages/trigger-sdk/src/job.ts b/packages/trigger-sdk/src/job.ts index b6336a376..da5f7423b 100644 --- a/packages/trigger-sdk/src/job.ts +++ b/packages/trigger-sdk/src/job.ts @@ -20,6 +20,7 @@ import type { import { slugifyId } from "./utils"; import { runLocalStorage } from "./runLocalStorage"; import { Prettify } from "@trigger.dev/core"; +import { ConcurrencyLimit } from "./concurrencyLimit"; export type JobOptions< TTrigger extends Trigger>, @@ -60,9 +61,16 @@ export type JobOptions< }); ``` */ integrations?: TIntegrations; - /** @deprecated This property is deprecated and no longer effects the execution of the Job - * */ - queue?: QueueOptions | string; + + /** + * The `concurrencyLimit` property is used to limit the number of concurrent run executions of a job. + * Can be a number which represents the limit or a `ConcurrencyLimit` instance which can be used to + * group together multiple jobs to share the same concurrency limit. + * + * If undefined the job will be limited only by the server's global concurrency limit, or if you are using the + * Trigger.dev Cloud service, the concurrency limit of your plan. + */ + concurrencyLimit?: number | ConcurrencyLimit; /** The `enabled` property is used to enable or disable the Job. If you disable a Job, it will not run. */ enabled?: boolean; /** This function gets called automatically when a Run is Triggered. @@ -174,6 +182,12 @@ export class Job< enabled: this.enabled, preprocessRuns: this.trigger.preprocessRuns, internal, + concurrencyLimit: + typeof this.options.concurrencyLimit === "number" + ? this.options.concurrencyLimit + : typeof this.options.concurrencyLimit === "object" + ? { id: this.options.concurrencyLimit.id, limit: this.options.concurrencyLimit.limit } + : undefined, }; } diff --git a/packages/trigger-sdk/src/triggerClient.ts b/packages/trigger-sdk/src/triggerClient.ts index 307cb8539..a64897e75 100644 --- a/packages/trigger-sdk/src/triggerClient.ts +++ b/packages/trigger-sdk/src/triggerClient.ts @@ -118,6 +118,7 @@ const registerSourceEvent: EventSpecification = { import EventEmitter from "node:events"; import * as packageJson from "../package.json"; +import { ConcurrencyLimit, ConcurrencyLimitOptions } from "./concurrencyLimit"; import { formatSchemaErrors } from "./utils/formatSchemaErrors"; import { WebhookDeliveryContext, WebhookSource } from "./triggers/webhook"; import { KeyValueStore } from "./store/keyValueStore"; @@ -742,6 +743,10 @@ export class TriggerClient { return endpoint; } + defineConcurrencyLimit(options: ConcurrencyLimitOptions) { + return new ConcurrencyLimit(options); + } + attach(job: Job, any>): void { this.#registeredJobs[job.id] = job; job.trigger.attachToJob(this, job); @@ -1788,6 +1793,12 @@ export class TriggerClient { enabled: job.enabled, preprocessRuns: job.trigger.preprocessRuns, internal, + concurrencyLimit: + typeof job.options.concurrencyLimit === "number" + ? job.options.concurrencyLimit + : typeof job.options.concurrencyLimit === "object" + ? { id: job.options.concurrencyLimit.id, limit: job.options.concurrencyLimit.limit } + : undefined, }; } diff --git a/perf/src/index.ts b/perf/src/index.ts index c3edeacad..4083f9d94 100644 --- a/perf/src/index.ts +++ b/perf/src/index.ts @@ -108,8 +108,8 @@ async function mainParallel() { async function mainParallelBulk() { const batches = 1; - const concurrency = 50; - const eventsPer = 20; + const concurrency = 10; + const eventsPer = 10; console.log("Preparing perf tests..."); @@ -169,7 +169,34 @@ async function mainSerial() { } } -mainParallelBulk().catch((err) => { +async function mainConcurrency() { + const batches = 1; + const concurrency = 10; + const eventsPer = 5; + + console.log("Preparing perf tests..."); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + console.log("Starting perf tests in 1 second..."); + + // wait for 1 seconds + await new Promise((resolve) => setTimeout(resolve, 1000)); + + // Send 5 events per second for 30 seconds (1 event == 10 runs) + for (let i = 0; i < batches; i++) { + console.log(`Sending ${concurrency} x ${eventsPer} events... batch ${i + 1}/${batches}`); + await Promise.all(new Array(concurrency).fill(0).map(() => sendEvents(eventsPer))); + + await new Promise((resolve) => setTimeout(resolve, 250)); + } +} + +async function mainSingle() { + await sendEvent(); +} + +mainConcurrency().catch((err) => { console.error(err); process.exit(1); }); diff --git a/perf/src/trigger.ts b/perf/src/trigger.ts index 6f8006fa5..3d79eb07f 100644 --- a/perf/src/trigger.ts +++ b/perf/src/trigger.ts @@ -6,6 +6,11 @@ export const triggerClient = new TriggerClient({ apiUrl: process.env.TRIGGER_API_URL!, }); +const concurrencyLimit = triggerClient.defineConcurrencyLimit({ + id: `perf-test-shared`, + limit: 5, +}); + triggerClient.defineJob({ id: `perf-test-1`, name: `Perf Test 1`, @@ -13,11 +18,12 @@ triggerClient.defineJob({ trigger: eventTrigger({ name: "perf.test", }), + concurrencyLimit, run: async (payload, io, ctx) => { await io.runTask( "task-1", async (task) => { - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 5000)); return { value: Math.random(), @@ -26,6 +32,8 @@ triggerClient.defineJob({ { name: "task 1" } ); + await io.wait("wait", 10); + await io.runTask( "task-2", async (task) => { @@ -35,5 +43,111 @@ triggerClient.defineJob({ }, { name: "task 2" } ); + + await io.runTask( + "task-3", + async (task) => { + await new Promise((resolve) => setTimeout(resolve, 2000)); + + return { + value: Math.random(), + }; + }, + { name: "task 3" } + ); + }, +}); + +triggerClient.defineJob({ + id: `perf-test-2`, + name: `Perf Test 2`, + version: "1.0.0", + trigger: eventTrigger({ + name: "perf.test", + }), + concurrencyLimit: 5, + run: async (payload, io, ctx) => { + await io.runTask( + "task-1", + async (task) => { + await new Promise((resolve) => setTimeout(resolve, 5000)); + + return { + value: Math.random(), + }; + }, + { name: "task 1" } + ); + + await io.wait("wait", 10); + + await io.runTask( + "task-2", + async (task) => { + return { + value: Math.random(), + }; + }, + { name: "task 2" } + ); + + await io.runTask( + "task-3", + async (task) => { + await new Promise((resolve) => setTimeout(resolve, 2000)); + + return { + value: Math.random(), + }; + }, + { name: "task 3" } + ); + }, +}); + +triggerClient.defineJob({ + id: `perf-test-3`, + name: `Perf Test 3`, + version: "1.0.0", + trigger: eventTrigger({ + name: "perf.test", + }), + concurrencyLimit, + run: async (payload, io, ctx) => { + await io.runTask( + "task-1", + async (task) => { + await new Promise((resolve) => setTimeout(resolve, 5000)); + + return { + value: Math.random(), + }; + }, + { name: "task 1" } + ); + + await io.wait("wait", 10); + + await io.runTask( + "task-2", + async (task) => { + return { + value: Math.random(), + }; + }, + { name: "task 2" } + ); + + await io.runTask( + "task-3", + async (task) => { + await new Promise((resolve) => setTimeout(resolve, 2000)); + + return { + value: Math.random(), + }; + }, + { name: "task 3" } + ); }, }); diff --git a/perf/tsconfig.json b/perf/tsconfig.json index 67f0df985..d0d1f2687 100644 --- a/perf/tsconfig.json +++ b/perf/tsconfig.json @@ -12,6 +12,8 @@ "@trigger.dev/express/*": ["../packages/express/src/*"], "@trigger.dev/core": ["../packages/core/src/index"], "@trigger.dev/core/*": ["../packages/core/src/*"], + "@trigger.dev/core-backend": ["../packages/core-backend/src/index"], + "@trigger.dev/core-backend/*": ["../packages/core-backend/src/*"], "@trigger.dev/integration-kit": ["../packages/integration-kit/src/index"], "@trigger.dev/integration-kit/*": ["../packages/integration-kit/src/*"], "@trigger.dev/github": ["../integrations/github/src/index"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a736f021a..f37df3fd5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -175,6 +175,7 @@ importers: highlight.run: ^7.3.4 humanize-duration: ^3.27.3 intl-parse-accept-language: ^1.0.0 + ioredis: ^5.3.2 isbot: ^3.6.5 jsonpointer: ^5.0.1 lodash.omit: ^4.5.0 @@ -283,6 +284,7 @@ importers: highlight.run: 7.3.4 humanize-duration: 3.27.3 intl-parse-accept-language: 1.0.0 + ioredis: 5.3.2 isbot: 3.6.5 jsonpointer: 5.0.1 lodash.omit: 4.5.0 @@ -7922,6 +7924,10 @@ packages: /@humanwhocodes/object-schema/1.2.1: resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==} + /@ioredis/commands/1.2.0: + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + dev: false + /@isaacs/cliui/8.0.2: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -18013,6 +18019,11 @@ packages: resolution: {integrity: sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q==} engines: {node: '>=6'} + /cluster-key-slot/1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /co/4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -18953,6 +18964,11 @@ packages: /delegates/1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} + /denque/2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dev: false + /depd/2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -23265,6 +23281,23 @@ packages: loose-envify: 1.4.0 dev: false + /ioredis/5.3.2: + resolution: {integrity: sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==} + engines: {node: '>=12.22.0'} + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.4 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + /ip/1.1.8: resolution: {integrity: sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==} @@ -25035,6 +25068,14 @@ packages: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} dev: true + /lodash.defaults/4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false + + /lodash.isarguments/3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false + /lodash.isplainobject/4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} dev: true @@ -29249,6 +29290,18 @@ packages: strip-indent: 3.0.0 dev: false + /redis-errors/1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + dev: false + + /redis-parser/3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + dependencies: + redis-errors: 1.2.0 + dev: false + /reduce-css-calc/2.1.8: resolution: {integrity: sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg==} dependencies: @@ -30773,6 +30826,10 @@ packages: get-source: 2.0.12 dev: true + /standard-as-callback/2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /static-extend/0.1.2: resolution: {integrity: sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==} engines: {node: '>=0.10.0'} From c4cb98af5c5f0243d1632bb38b5a102157a5ae25 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 28 Nov 2023 16:23:35 +0000 Subject: [PATCH 5/8] Clarified the run execution limits --- docs/documentation/concepts/limits.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/documentation/concepts/limits.mdx b/docs/documentation/concepts/limits.mdx index c7b644564..bf4334259 100644 --- a/docs/documentation/concepts/limits.mdx +++ b/docs/documentation/concepts/limits.mdx @@ -16,7 +16,7 @@ The following limits apply to the Trigger.dev Cloud service and users of the sel | Connected Integrations | Up to 50 | Up to 1000 | Custom | | Task Output Size | 3MB | 3MB | 3MB | | [Tasks per Run](#tasks-per-runs) | Up to 250 | Up to 1000 | Custom | -| [Concurrent Run Executions per Environment](#concurrent-run-executions) | Up to 10 | Up to 100 | Custom | +| [Concurrent Run Executions](#concurrent-run-executions) | Up to 10 | Up to 100 | Custom | | [Maximum Task Duration](#maximum-task-duration) | < 2m | < 2m | < Deployment Grace Period | | [Maximum Run Execution Duration](#maximum-total-run-execution-duration) | up to 15m | up to 2 hrs | Custom | | [Yielded Executions per Run](#yielded-executions-per-run) | Up to 100 | Up to 100 | Custom | From 1dcd87a2aa01c7146e8a4162c30129325f23c5c2 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 28 Nov 2023 18:05:15 +0000 Subject: [PATCH 6/8] URI encode keys (#758) --- .changeset/eleven-ghosts-applaud.md | 5 +++++ apps/webapp/app/routes/api.v1.store.$key.ts | 10 +++++----- packages/trigger-sdk/src/apiClient.ts | 6 ++++-- references/job-catalog/src/built-ins.ts | 7 +++++++ 4 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 .changeset/eleven-ghosts-applaud.md diff --git a/.changeset/eleven-ghosts-applaud.md b/.changeset/eleven-ghosts-applaud.md new file mode 100644 index 000000000..64fd117a5 --- /dev/null +++ b/.changeset/eleven-ghosts-applaud.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Fix: `Key-Value Store` keys will now be URI encoded diff --git a/apps/webapp/app/routes/api.v1.store.$key.ts b/apps/webapp/app/routes/api.v1.store.$key.ts index 2350cf029..f4249c128 100644 --- a/apps/webapp/app/routes/api.v1.store.$key.ts +++ b/apps/webapp/app/routes/api.v1.store.$key.ts @@ -42,14 +42,14 @@ export async function action({ request, params }: ActionFunctionArgs) { const store = new KeyValueStore(authenticatedEnv); - const { key } = parsedParams.data; + const decodedKey = decodeURIComponent(parsedParams.data.key); try { switch (parsedMethod.data) { case "DELETE": { - const deleted = await store.delete(key); + const deleted = await store.delete(decodedKey); - return json({ action: "DELETE", key, deleted }); + return json({ action: "DELETE", key: decodedKey, deleted }); } case "PUT": { const value = await request.text(); @@ -65,9 +65,9 @@ export async function action({ request, params }: ActionFunctionArgs) { ); } - const setValue = await store.set(key, value); + const setValue = await store.set(decodedKey, value); - return json({ action: "SET", key, value: setValue }); + return json({ action: "SET", key: decodedKey, value: setValue }); } default: { assertExhaustive(parsedMethod.data); diff --git a/packages/trigger-sdk/src/apiClient.ts b/packages/trigger-sdk/src/apiClient.ts index 18327326e..b5e94a55c 100644 --- a/packages/trigger-sdk/src/apiClient.ts +++ b/packages/trigger-sdk/src/apiClient.ts @@ -595,7 +595,9 @@ export class ApiClient { data, }); - const STORE_URL = `${this.#apiUrl}/api/v1/store/${data.key}`; + const encodedKey = encodeURIComponent(data.key); + + const STORE_URL = `${this.#apiUrl}/api/v1/store/${encodedKey}`; const authHeader: HeadersInit = { Authorization: `Bearer ${apiKey}`, @@ -627,7 +629,7 @@ export class ApiClient { return { action: "HAS", - key: data.key, + key: encodedKey, has: !!headResponse.ok, }; } diff --git a/references/job-catalog/src/built-ins.ts b/references/job-catalog/src/built-ins.ts index 159bc37d4..c2939af22 100644 --- a/references/job-catalog/src/built-ins.ts +++ b/references/job-catalog/src/built-ins.ts @@ -307,6 +307,13 @@ client.defineJob({ name: "store.example", }), run: async (payload, io, ctx) => { + // key tests + await io.store.job.set("set-emoji", "๐Ÿ”", "๐Ÿฎ"); + await io.store.job.get("get-emoji", "๐Ÿ”"); + + await io.store.job.set("set-url", "https://example.com/?foo=bar", "url"); + await io.store.job.get("get-url", "https://example.com/?foo=bar"); + // value tests await io.store.job.set("set-undefined", "test", undefined); await io.store.job.get("get-undefined", "test"); From 4dc46cbbe4f2c7928d6b82c3ad7f0a078d432876 Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Wed, 29 Nov 2023 00:12:50 +0530 Subject: [PATCH 7/8] fix: transform Headers object to Record type (#760) --- packages/core/src/requestFilterMatches.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/requestFilterMatches.ts b/packages/core/src/requestFilterMatches.ts index 94afa6a48..01bb0d4b0 100644 --- a/packages/core/src/requestFilterMatches.ts +++ b/packages/core/src/requestFilterMatches.ts @@ -10,7 +10,8 @@ export async function requestFilterMatches( return false; } - if (filter.headers && !eventFilterMatches(clonedRequest.headers, filter.headers)) { + const headersObj = Object.fromEntries(clonedRequest.headers.entries()); + if (filter.headers && !eventFilterMatches(headersObj, filter.headers)) { return false; } From ca05d5f60381498145e4e2001ce40be02a55312d Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 28 Nov 2023 22:48:31 +0000 Subject: [PATCH 8/8] @trigger.dev/openai: Adding additional assistant tasks --- .changeset/smart-frogs-help.md | 5 + apps/webapp/app/env.server.ts | 1 + .../runExecutionRateLimiter.server.ts | 2 +- docs/integrations/apis/openai/assistants.mdx | 64 ++++++- integrations/openai/src/assistants.ts | 173 +++++++++++++++++- integrations/openai/src/taskUtils.ts | 79 +++++--- integrations/openai/src/threads.ts | 72 +++++++- integrations/openai/src/types.ts | 2 +- references/job-catalog/src/openai.ts | 31 ++++ 9 files changed, 391 insertions(+), 38 deletions(-) create mode 100644 .changeset/smart-frogs-help.md diff --git a/.changeset/smart-frogs-help.md b/.changeset/smart-frogs-help.md new file mode 100644 index 000000000..914b9b7b7 --- /dev/null +++ b/.changeset/smart-frogs-help.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/openai": patch +--- + +Adding additional assistant tasks diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 3e3b7ca89..2027f880d 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -60,6 +60,7 @@ const EnvironmentSchema = z.object({ REDIS_PORT: z.coerce.number().optional(), REDIS_USERNAME: z.string().optional(), REDIS_PASSWORD: z.string().optional(), + REDIS_TLS_DISABLED: z.string().optional(), DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10), DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1), diff --git a/apps/webapp/app/services/runExecutionRateLimiter.server.ts b/apps/webapp/app/services/runExecutionRateLimiter.server.ts index b482c5c50..cf63c2c97 100644 --- a/apps/webapp/app/services/runExecutionRateLimiter.server.ts +++ b/apps/webapp/app/services/runExecutionRateLimiter.server.ts @@ -397,7 +397,7 @@ function getRateLimiter() { username: env.REDIS_USERNAME, password: env.REDIS_PASSWORD, enableAutoPipelining: true, - tls: {} + ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }) }, defaultConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT, }); diff --git a/docs/integrations/apis/openai/assistants.mdx b/docs/integrations/apis/openai/assistants.mdx index 73cdb48d5..7630b2db6 100644 --- a/docs/integrations/apis/openai/assistants.mdx +++ b/docs/integrations/apis/openai/assistants.mdx @@ -33,6 +33,54 @@ const assistant = await io.openai.beta.assistants.create("create-assistant", { }); ``` +### `update()` + +Update an assistant. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/assistants/modifyAssistant) + +```ts example.ts +const file = await io.openai.files.createAndWaitForProcessing("upload-file", { + purpose: "assistants", + file: fs.createReadStream("./fixtures/mydata.csv"), +}); + +const assistantId = "asst_abc123"; + +const assistant = await io.openai.beta.assistants.update("update-assistant", assistantId, { + file_ids: [file.id], // add a file to the assistant +}); +``` + +### `list()` + +List assistants. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/assistants/listAssistants) + +```ts example.ts +const assistants = await io.openai.beta.assistants.list("list"); + +// with pagination +const assistants = await io.openai.beta.assistants.list("list", { + limit: 10, + order: "desc", + after: "asst_abc123", +}); +``` + +### `retrieve()` + +Retrieve an assistant. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/assistants/getAssistant) + +```ts example.ts +const assistant = await io.openai.beta.assistants.retrieve("get-assistant", "asst_abc123"); +``` + +### `del()` + +Delete an assistant. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/assistants/deleteAssistant) + +```ts example.ts +const deletedAssistant = await io.openai.beta.assistants.del("delete-assistant", "asst_abc123"); +``` + ## Threads Create threads that assistants can interact with. [Official OpenAI docs](https://platform.openai.com/docs/api-reference/threads/createThread) @@ -132,12 +180,26 @@ Create messages within threads. [Official OpenAI docs](https://platform.openai.c ### `list()` -List all messages in a thread. +List messages in a thread. ```ts example.ts const messages = await io.openai.beta.threads.messages.list("list-messages", "thread_abc123"); +// with pagination +const messages = await io.openai.beta.threads.messages.list("list-messages", "thread_abc123", { + limit: 10, + order: "desc", + after: "message_abc123", +}); ``` +If you want to list all messages in a thread, you can use the `listAll()` helper: + +```ts example.ts +const messages = await io.openai.beta.threads.messages.listAll("list-messages", "thread_abc123"); +``` + +This will automatically paginate through all messages in the thread and return them as a single array. + ### `create()` Create a message. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/messages/createMessage) diff --git a/integrations/openai/src/assistants.ts b/integrations/openai/src/assistants.ts index c0a25cd15..1beeb058c 100644 --- a/integrations/openai/src/assistants.ts +++ b/integrations/openai/src/assistants.ts @@ -2,13 +2,13 @@ import { IntegrationTaskKey, Prettify } from "@trigger.dev/sdk"; import { OpenAIRunTask } from "./index"; import { OpenAIIntegrationOptions, OpenAIRequestOptions } from "./types"; import OpenAI from "openai"; -import { createTaskOutputProperties, handleOpenAIError } from "./taskUtils"; +import { createTaskOutputProperties, handleOpenAIError, isRequestOptions } from "./taskUtils"; export class Assistants { constructor( private runTask: OpenAIRunTask, private options: OpenAIIntegrationOptions - ) {} + ) { } async create( key: IntegrationTaskKey, @@ -54,4 +54,173 @@ export class Assistants { handleOpenAIError ); } + + async update( + key: IntegrationTaskKey, + id: string, + params: Prettify, + options: OpenAIRequestOptions = {} + ): Promise { + return this.runTask( + key, + async (client, task) => { + const { data, response } = await client.beta.assistants + .update(id, params, { + idempotencyKey: task.idempotencyKey, + ...options, + }) + .withResponse(); + + const outputProperties = createTaskOutputProperties(undefined, response.headers); + + task.outputProperties = [ + ...(outputProperties ?? []), + { + label: "assistantId", + text: data.id, + }, + ]; + + return data; + }, + { + name: "Update Assistant", + params, + properties: [ + ...(params.model ? [{ label: "model", text: params.model }] : []), + ...(params.name ? [{ label: "name", text: params.name }] : []), + ...(params.file_ids && params.file_ids.length > 0 + ? [{ label: "files", text: params.file_ids.join(", ") }] + : []), + ], + }, + handleOpenAIError + ); + } + + list( + key: IntegrationTaskKey, + params?: Prettify, + options?: OpenAIRequestOptions, + ): Promise; + list( + key: IntegrationTaskKey, + options?: OpenAIRequestOptions, + ): Promise; + async list( + key: IntegrationTaskKey, + params: Prettify | OpenAIRequestOptions = {}, + options: OpenAIRequestOptions | undefined = undefined + ): Promise { + return this.runTask( + key, + async (client, task) => { + if (isRequestOptions(params)) { + const { data, response } = await client.beta.assistants + .list({ + idempotencyKey: task.idempotencyKey, + ...params, + }) + .withResponse(); + + task.outputProperties = createTaskOutputProperties(undefined, response.headers); + + return data.data; + } + + const { data, response } = await client.beta.assistants + .list(params, { + idempotencyKey: task.idempotencyKey, + ...options, + }) + .withResponse(); + + task.outputProperties = createTaskOutputProperties(undefined, response.headers); + + return data.data; + + }, + { + name: "List Assistants", + params, + properties: !isRequestOptions(params) ? [ + ...(params.before ? [{ label: "before", text: params.before }] : []), + ...(params.order ? [{ label: "order", text: params.order }] : []), + ...(params.after ? [{ label: "after", text: params.after }] : []), + ...(params.limit ? [{ label: "limit", text: String(params.limit) }] : []), + ] : [], + }, + handleOpenAIError + ); + } + + async del( + key: IntegrationTaskKey, + id: string, + options: OpenAIRequestOptions = {} + ): Promise { + return this.runTask( + key, + async (client, task) => { + const { data, response } = await client.beta.assistants + .del(id, { + idempotencyKey: task.idempotencyKey, + ...options, + }) + .withResponse(); + + task.outputProperties = createTaskOutputProperties(undefined, response.headers); + + return data; + }, + { + name: "Delete Assistant", + params: { + id, + }, + properties: [ + { + label: "assistantId", + text: id, + }, + ], + }, + handleOpenAIError + ); + } + + async retrieve( + key: IntegrationTaskKey, + id: string, + options: OpenAIRequestOptions = {} + ): Promise { + return this.runTask( + key, + async (client, task) => { + const { data, response } = await client.beta.assistants + .retrieve(id, { + idempotencyKey: task.idempotencyKey, + ...options, + }) + .withResponse(); + + task.outputProperties = createTaskOutputProperties(undefined, response.headers); + + return data; + }, + { + name: "Retrieve Assistant", + params: { + id, + }, + properties: [ + { + label: "assistantId", + text: id, + }, + ], + }, + handleOpenAIError + ); + } } diff --git a/integrations/openai/src/taskUtils.ts b/integrations/openai/src/taskUtils.ts index 88b50a465..634140039 100644 --- a/integrations/openai/src/taskUtils.ts +++ b/integrations/openai/src/taskUtils.ts @@ -60,11 +60,11 @@ function createTaskUsageProperties( }, ...("completion_tokens" in usage ? [ - { - label: "Completion Usage", - text: String(usage.completion_tokens), - }, - ] + { + label: "Completion Usage", + text: String(usage.completion_tokens), + }, + ] : []), ]; } @@ -83,35 +83,35 @@ function createTaskRateLimitProperties(headers: Headers | undefined) { return [ ...(remainingRequests ? [ - { - label: "Remaining Requests", - text: remainingRequests ?? "Unknown", - }, - ] + { + label: "Remaining Requests", + text: remainingRequests ?? "Unknown", + }, + ] : []), ...(resetRequests ? [ - { - label: "Reset Requests", - text: resetRequests ?? "Unknown", - }, - ] + { + label: "Reset Requests", + text: resetRequests ?? "Unknown", + }, + ] : []), ...(remainingTokens ? [ - { - label: "Remaining Tokens", - text: remainingTokens ?? "Unknown", - }, - ] + { + label: "Remaining Tokens", + text: remainingTokens ?? "Unknown", + }, + ] : []), ...(resetTokens ? [ - { - label: "Reset Tokens", - text: resetTokens ?? "Unknown", - }, - ] + { + label: "Reset Tokens", + text: resetTokens ?? "Unknown", + }, + ] : []), ]; } @@ -282,3 +282,32 @@ export const backgroundTaskRetries: FetchRetryOptions = { randomize: true, }, }; + +type KeysEnum = { [P in keyof Required]: true }; + +const requestOptionsKeys: KeysEnum = { + method: true, + path: true, + query: true, + headers: true, + idempotencyKey: true, +}; + +export const isRequestOptions = (obj: unknown): obj is OpenAIRequestOptions => { + return ( + typeof obj === 'object' && + obj !== null && + !isEmptyObj(obj) && + Object.keys(obj).every((k) => hasOwn(requestOptionsKeys, k)) + ); +}; + +function isEmptyObj(obj: Object | null | undefined): boolean { + if (!obj) return true; + for (const _k in obj) return false; + return true; +} + +function hasOwn(obj: Object, key: string): boolean { + return Object.prototype.hasOwnProperty.call(obj, key); +} \ No newline at end of file diff --git a/integrations/openai/src/threads.ts b/integrations/openai/src/threads.ts index 537fd2c59..a22e7d635 100644 --- a/integrations/openai/src/threads.ts +++ b/integrations/openai/src/threads.ts @@ -7,6 +7,7 @@ import { createBackgroundFetchUrl, createTaskOutputProperties, handleOpenAIError, + isRequestOptions, } from "./taskUtils"; import { RunSubmitToolOutputsParams } from "openai/resources/beta/threads/runs/runs"; import { ThreadUpdateParams } from "openai/resources/beta/threads/threads"; @@ -15,7 +16,7 @@ export class Threads { constructor( private runTask: OpenAIRunTask, private options: OpenAIIntegrationOptions - ) {} + ) { } /** * Create a thread and run it in one task. @@ -261,7 +262,7 @@ class Runs { constructor( private runTask: OpenAIRunTask, private options: OpenAIIntegrationOptions - ) {} + ) { } /** * Creates a run and waits for it to complete by polling in the background. @@ -551,15 +552,70 @@ class Messages { constructor( private runTask: OpenAIRunTask, private options: OpenAIIntegrationOptions - ) {} + ) { } + + /** + * Returns messages for a given thread. + */ + list( + key: IntegrationTaskKey, + threadId: string, + params?: Prettify, + options?: OpenAIRequestOptions + ): Promise + list( + key: IntegrationTaskKey, + threadId: string, + options?: OpenAIRequestOptions + ): Promise + async list( + key: IntegrationTaskKey, + threadId: string, + params: Prettify | OpenAIRequestOptions = {}, + options: OpenAIRequestOptions | undefined = undefined + ): Promise { + return this.runTask( + key, + async (client, task, io) => { + if (isRequestOptions(params)) { + const { data: page, response } = await client.beta.threads.messages + .list(threadId, { + idempotencyKey: task.idempotencyKey, + ...params, + }) + .withResponse(); + + task.outputProperties = createTaskOutputProperties(undefined, response.headers); + + return page.data; + } + + const { data: page, response } = await client.beta.threads.messages + .list(threadId, params, { + idempotencyKey: task.idempotencyKey, + ...options, + }) + .withResponse(); + + task.outputProperties = createTaskOutputProperties(undefined, response.headers); + + return page.data; + }, + { + name: "List Messages", + properties: [{ label: "threadId", text: threadId }], + }, + handleOpenAIError + ); + } /** * Returns all messages for a given thread. */ - async list( + async listAll( key: IntegrationTaskKey, threadId: string, - options: OpenAIRequestOptions = {} + options: OpenAIRequestOptions = {}, ): Promise { return this.runTask( key, @@ -573,8 +629,8 @@ class Messages { const allMessages = []; - for await (const fineTuningJob of page) { - allMessages.push(fineTuningJob); + for await (const message of page) { + allMessages.push(message); } task.outputProperties = createTaskOutputProperties(undefined, response.headers); @@ -582,7 +638,7 @@ class Messages { return allMessages; }, { - name: "List Messages", + name: "List All Messages", properties: [{ label: "threadId", text: threadId }], }, handleOpenAIError diff --git a/integrations/openai/src/types.ts b/integrations/openai/src/types.ts index fabc24d71..2a060d873 100644 --- a/integrations/openai/src/types.ts +++ b/integrations/openai/src/types.ts @@ -34,4 +34,4 @@ export type OpenAIRequestOptions = { path?: string; headers?: OpenAIHeaders; idempotencyKey?: string; -}; +}; \ No newline at end of file diff --git a/references/job-catalog/src/openai.ts b/references/job-catalog/src/openai.ts index f8a53f9c2..c2b2c1405 100644 --- a/references/job-catalog/src/openai.ts +++ b/references/job-catalog/src/openai.ts @@ -228,6 +228,37 @@ client.defineJob({ }, }); +client.defineJob({ + id: "openai-manage-assistant", + name: "OpenAI GPT Manage Assistant", + version: "0.0.1", + trigger: invokeTrigger({ + schema: z.object({ + assistantId: z.string().optional(), + }), + }), + integrations: { + openai, + }, + run: async (payload, io, ctx) => { + const assistants = await io.openai.beta.assistants.list("list", { + limit: 10 + }); + + if (payload.assistantId) { + await io.openai.beta.assistants.retrieve("retrieve", payload.assistantId); + await io.openai.beta.assistants.update("update", payload.assistantId, { + name: "Updated name", + }); + await io.openai.beta.assistants.del("delete", payload.assistantId); + } + + for (const assistant of assistants) { + await io.openai.beta.assistants.del(`delete ${assistant.id}`, assistant.id); + } + }, +}); + client.defineJob({ id: "openai-use-assistant", name: "OpenAI GPT Use Assistant",