Improve the way we stream from tasks to the server

Skip all flaky tests 😡
Skip flaky realtimeClient tests in github CI for now
2024-12-11 22:22:04 +00:00 · 2024-12-11 11:20:12 +00:00 · 2024-12-11 11:10:52 +00:00 · 2024-12-11 11:01:35 +00:00 · 2024-12-11 11:00:46 +00:00 · 2024-12-10 15:53:45 +00:00
3240 changed files with 162702 additions and 133688 deletions
@@ -1,9 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
- Fix additionalFiles that aren't decendants
- Stop swallowing uncaught exceptions in prod
- Improve warnings and errors, fail early on critical warnings
- New arg to --save-logs even for successful builds
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-v3 CLI update command and package manager detection fix
@@ -1,6 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-better handle task metadata parse errors, and display nicely formatted errors
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Vastly improved dev command output
@@ -1,8 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"@trigger.dev/core-apps": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-add machine config and secure zod connection
@@ -1,5 +0,0 @@
---
-"@trigger.dev/core": patch
---
-
-Fix error stack traces
@@ -1,28 +1,19 @@
 {
  "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json",
-  "changelog": "@changesets/cli/changelog",
-  "commit": false,
-  "fixed": [
-    [
-      "@trigger.dev/*",
-      "trigger.dev"
-    ]
+  "changelog": [
+    "@remix-run/changelog-github",
+    {
+      "repo": "triggerdotdev/trigger.dev"
+    }
  ],
+  "commit": false,
+  "fixed": [["@trigger.dev/*", "trigger.dev"]],
  "linked": [],
  "access": "public",
  "baseBranch": "main",
  "updateInternalDependencies": "patch",
-  "ignore": [
-    "webapp",
-    "emails",
-    "proxy",
-    "yalt",
-    "@trigger.dev/database",
-    "coordinator",
-    "docker-provider",
-    "kubernetes-provider"
-  ],
+  "ignore": ["webapp", "proxy", "coordinator", "docker-provider", "kubernetes-provider"],
  "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
    "onlyUpdatePeerDependentsWhenOutOfRange": true
  }
-}
+}
@@ -1,7 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Configurable log levels in the config file and via env var
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Don’t swallow some error messages when deploying
@@ -1,6 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Add option to print console logs in the dev CLI locally (issue #1014)
@@ -1,5 +0,0 @@
---
-"@trigger.dev/core": patch
---
-
-Fixed batch otel flushing
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Remove the env var check during deploy (too many false negatives)
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fix permissions inside node_modules
@@ -1,9 +0,0 @@
---
-"@trigger.dev/sdk": patch
---
-
-Make msw a normal dependency (for now) to fix Module Not Found error in Next.js.
-
-It turns out that webpack will "hoist" dynamically imported modules and attempt to resolve them at build time, even though it's an optional peer dep:
-
-https://x.com/maverickdotdev/status/1782465214308319404
@@ -1,5 +0,0 @@
---
-"@trigger.dev/react": patch
---
-
-Fix for shared queryKey between useRunDetails and useRunStatuses
@@ -1,5 +0,0 @@
---
-"@trigger.dev/core": patch
---
-
-Retry 429, 500, and connection error API requests to the trigger.dev server
@@ -1,5 +0,0 @@
---
-"@trigger.dev/sdk": patch
---
-
-Export queue from the SDK
@@ -1,7 +0,0 @@
---
-"@trigger.dev/core-apps": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Display errors for runs and deployments
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-v3: fix digest extraction
@@ -1,5 +0,0 @@
---
-"@trigger.dev/nestjs": patch
---
-
-fix: [nestjs integration] fastify HTTP adapter detection now works correctly for response headers
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Changed "Worker" to "Version" in the dev command key
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Correctly handle self-hosted deploy command errors
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Default to retrying enabled in dev when running init
@@ -1,6 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Handle string and non-stringifiable outputs like functions
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Improve error messages during dev/deploy and handle deploy image build issues
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fixes an issue that was treating v2 trigger directories as v3
@@ -1,6 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"@trigger.dev/core": patch
---
-
-Added replayRun function to the SDK
@@ -1,6 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"@trigger.dev/core": patch
---
-
-Improve the SDK function types and expose a new APIError instead of the APIResult type
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Added a Node.js runtime check for the CLI
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fix post start hooks
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fix dev CLI output when not printing update messages
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Use the dashboard url instead of the API url for the View logs link
@@ -1,109 +0,0 @@
-{
-  "mode": "pre",
-  "tag": "beta",
-  "initialVersions": {
-    "coordinator": "0.0.1",
-    "docker-provider": "0.0.1",
-    "kubernetes-provider": "0.0.1",
-    "proxy": "0.0.11",
-    "webapp": "1.0.0",
-    "yalt": "0.0.1",
-    "@trigger.dev/airtable": "2.3.18",
-    "@trigger.dev/github": "2.3.18",
-    "@trigger.dev/linear": "2.3.18",
-    "@trigger.dev/openai": "2.3.18",
-    "@trigger.dev/plain": "2.3.18",
-    "@trigger.dev/replicate": "2.3.18",
-    "@trigger.dev/resend": "2.3.18",
-    "@trigger.dev/sendgrid": "2.3.18",
-    "@trigger.dev/shopify": "2.3.18",
-    "@trigger.dev/slack": "2.3.18",
-    "@trigger.dev/stripe": "2.3.18",
-    "@trigger.dev/supabase": "2.3.18",
-    "@trigger.dev/typeform": "2.3.18",
-    "@trigger.dev/astro": "2.3.18",
-    "@trigger.dev/cli": "2.3.18",
-    "trigger.dev": "2.3.18",
-    "@trigger.dev/core": "2.3.18",
-    "@trigger.dev/core-apps": "0.0.0",
-    "@trigger.dev/core-backend": "2.3.18",
-    "@trigger.dev/database": "0.0.1",
-    "emails": "1.0.0",
-    "@trigger.dev/eslint-plugin": "2.3.18",
-    "@trigger.dev/express": "2.3.18",
-    "@trigger.dev/hono": "2.3.18",
-    "@trigger.dev/integration-kit": "2.3.18",
-    "@trigger.dev/nestjs": "2.3.18",
-    "@trigger.dev/nextjs": "2.3.18",
-    "@trigger.dev/otlp-importer": "2.3.11",
-    "@trigger.dev/react": "2.3.18",
-    "@trigger.dev/remix": "2.3.18",
-    "@trigger.dev/sveltekit": "2.3.18",
-    "@trigger.dev/testing": "2.3.18",
-    "@trigger.dev/sdk": "2.3.18",
-    "@trigger.dev/yalt": "2.3.18"
-  },
-  "changesets": [
-    "angry-eagles-trade",
-    "beige-pens-dance",
-    "breezy-gorillas-mate",
-    "chilled-hornets-move",
-    "clean-pianos-listen",
-    "clever-apes-collect",
-    "cool-glasses-bake",
-    "cuddly-feet-approve",
-    "dry-walls-check",
-    "eight-pumas-float",
-    "eleven-paws-join",
-    "few-students-share",
-    "funny-swans-destroy",
-    "green-bags-wink",
-    "khaki-apricots-design",
-    "khaki-poems-lay",
-    "late-icons-lie",
-    "late-steaks-behave",
-    "lemon-jobs-repair",
-    "light-bulldogs-press",
-    "light-dragons-complain",
-    "loud-actors-remember",
-    "many-ligers-pump",
-    "mighty-camels-joke",
-    "nasty-jars-pump",
-    "new-rivers-tell",
-    "ninety-pets-travel",
-    "odd-poets-own",
-    "polite-ducks-switch",
-    "polite-rockets-matter",
-    "poor-flowers-cross",
-    "rare-roses-float",
-    "real-planets-stare",
-    "rich-kangaroos-unite",
-    "rotten-beers-refuse",
-    "rotten-dryers-exercise",
-    "selfish-ducks-sort",
-    "shaggy-spoons-taste",
-    "sharp-emus-compare",
-    "sharp-zebras-serve",
-    "shiny-coats-cry",
-    "silly-suits-switch",
-    "slow-buses-own",
-    "smart-needles-move",
-    "smart-olives-eat",
-    "spicy-lamps-smoke",
-    "strange-ghosts-matter",
-    "strong-lemons-add",
-    "stupid-bulldogs-applaud",
-    "sweet-lizards-press",
-    "swift-dragons-peel",
-    "tall-bees-wave",
-    "tame-guests-know",
-    "tender-oranges-rhyme",
-    "tidy-balloons-suffer",
-    "tidy-dryers-sleep",
-    "tiny-doors-type",
-    "tiny-elephants-scream",
-    "tricky-bulldogs-heal",
-    "tricky-ladybugs-unite",
-    "two-pumas-wait"
-  ]
-}
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Add openssl to prod worker image and allow passing auth token via env var for deploy
@@ -1,7 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Fixed incorrect span timings around checkpoints by implementing a precise wall clock that resets after restores
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fix a bug where revoking the CLI token would prevent you from ever logging in again with the CLI.
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Add git to prod worker image which fixes private package installs
@@ -1,7 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Adding task with a triggerSource of schedule
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix realtime streaming splits in v2
@@ -1,6 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
---
-
-Lock SDK and CLI deps on exact core version
@@ -1,56 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"@trigger.dev/core": patch
---
-
-Updates the `trigger`, `batchTrigger` and their `*AndWait` variants to use the first parameter for the payload/items, and the second parameter for options.
-
-Before:
-
-```ts
-await yourTask.trigger({ payload: { foo: "bar" }, options: { idempotencyKey: "key_1234" } });
-await yourTask.triggerAndWait({ payload: { foo: "bar" }, options: { idempotencyKey: "key_1234" } });
-
-await yourTask.batchTrigger({ items: [{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }] });
-await yourTask.batchTriggerAndWait({ items: [{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }] });
-```
-
-After:
-
-```ts
-await yourTask.trigger({ foo: "bar" }, { idempotencyKey: "key_1234" });
-await yourTask.triggerAndWait({ foo: "bar" }, { idempotencyKey: "key_1234" });
-
-await yourTask.batchTrigger([{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }]);
-await yourTask.batchTriggerAndWait([{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }]);
-```
-
-We've also changed the API of the `triggerAndWait` result. Before, if the subtask that was triggered finished with an error, we would automatically "rethrow" the error in the parent task.
-
-Now instead we're returning a `TaskRunResult` object that allows you to discriminate between successful and failed runs in the subtask:
-
-Before:
-
-```ts
-try {
-  const result = await yourTask.triggerAndWait({ foo: "bar" });
-
-  // result is the output of your task
-  console.log("result", result);
-
-} catch (error) {
-  // handle subtask errors here
-}
-```
-
-After:
-
-```ts
-const result = await yourTask.triggerAndWait({ foo: "bar" });
-
-if (result.ok) {
-  console.log(`Run ${result.id} succeeded with output`, result.output);
-} else {
-  console.log(`Run ${result.id} failed with error`, result.error);
-}
-```
@@ -1,13 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-When using idempotency keys, triggerAndWait and batchTriggerAndWait will still work even if the existing runs have already been completed (or even partially completed, in the case of batchTriggerAndWait)
-
- TaskRunExecutionResult.id is now the run friendlyId, not the attempt friendlyId
- A single TaskRun can now have many batchItems, in the case of batchTriggerAndWait while using idempotency keys
- A run’s idempotencyKey is now added to the ctx as well as the TaskEvent and displayed in the span view
- When resolving batchTriggerAndWait, the runtimes no longer reject promises, leading to an error in the parent task
-
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Update trigger.dev CLI for new batch otel support
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Add additional logging around cleaning up dev workers, and always kill them after 5 seconds if they haven't already exited
@@ -1,6 +0,0 @@
---
-"@trigger.dev/otlp-importer": patch
-"trigger.dev": patch
---
-
-Fix package builds and CLI commands on Windows
@@ -1,8 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
-"@trigger.dev/cli": patch
---
-
-Move to our global system from AsyncLocalStorage for the current task context storage
@@ -1,5 +0,0 @@
---
-"@trigger.dev/sdk": patch
---
-
-Remove unimplemented batchOptions
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fix CLI logout and add list-profiles command
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fixing an issue with bundling @trigger.dev/core/v3 in dev when using pnpm
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Added DEBUG to the ignored env vars
@@ -1,5 +0,0 @@
---
-"@trigger.dev/core": patch
---
-
-Make optional schedule object fields nullish
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Fixed an issue where the trigger.dev package was not being built before publishing to npm
@@ -1,8 +0,0 @@
---
-"trigger.dev": major
-"@trigger.dev/core": major
-"@trigger.dev/otlp-importer": major
-"@trigger.dev/sdk": major
---
-
-Updates to support Trigger.dev v3
@@ -1,5 +0,0 @@
---
-"@trigger.dev/core": patch
---
-
-Added JSDocs to the schedule SDK types
@@ -1,7 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Dynamically import superjson and fix some bundling issues
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Changed the binary name from trigger.dev to triggerdev to fix a Windows issue
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Stop swallowing deployment errors and display them better
@@ -1,5 +0,0 @@
---
-"trigger.dev": patch
---
-
-Init command was failing on Windows because of bad template paths
@@ -1,10 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Fixes an issue that caused failed tasks when resuming after calling `triggerAndWait` or `batchTriggerAndWait` in prod/staging (this doesn't effect dev).
-
-The version of Node.js we use for deployed workers (latest 20) would crash with an out-of-memory error when the checkpoint was restored. This crash does not happen on Node 18x or Node21x, so we've decided to upgrade the worker version to Node.js21x, to mitigate this issue.
-
-You'll need to re-deploy to production to fix the issue.
@@ -1,6 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"@trigger.dev/core": patch
---
-
-Added cancelRun to the SDK
@@ -1,7 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
- Add graceful exit for prod workers
- Prevent overflow in long waits
@@ -1,7 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Added a new global - Task Catalog - to better handle task metadata
@@ -1,8 +0,0 @@
---
-"@trigger.dev/sdk": patch
-"@trigger.dev/core-apps": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
---
-
-Extracting out all the non-SDK related features from the main @trigger.dev/core/v3 export
@@ -1,6 +0,0 @@
---
-"trigger.dev": patch
-"@trigger.dev/cli": patch
---
-
-Add support for tasks located in subdirectories inside trigger dirs
@@ -0,0 +1,36 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "moduleDetection": "force",
+    "verbatimModuleSyntax": false,
+    "jsx": "react",
+
+    "strict": true,
+    "alwaysStrict": true,
+    "strictPropertyInitialization": false,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitAny": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+
+    "noFallthroughCasesInSwitch": true,
+    "resolveJsonModule": true,
+
+    "removeComments": false,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": false,
+    "experimentalDecorators": false,
+    "downlevelIteration": true,
+    "isolatedModules": true,
+    "noUncheckedIndexedAccess": true,
+
+    "pretty": true,
+    "customConditions": ["@triggerdotdev/source"]
+  }
+}
@@ -37,6 +37,7 @@ packages/\*\*/dist
 .vercel
 .cache
 .output
+.trigger
 apps/\*\*/public/build

 cypress/screenshots
@@ -10,16 +10,23 @@ DIRECT_URL=${DATABASE_URL}
 REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
+ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
+V3_ENABLED=true

-# Redis is used for concurrency control
-# REDIS_HOST="localhost"
-# REDIS_PORT="6379"
-# REDIS_TLS_DISABLED="true"
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"

 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
@@ -64,7 +71,7 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
 # OBJECT_STORE_SECRET_ACCESS_KEY=
-# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+# CHECKPOINT_THRESHOLD_IN_MS=10000

 # These control the server-side internal telemetry
 # INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
@@ -72,4 +79,4 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
 # INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
 # INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
-# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,
@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [triggerdotdev]
@@ -0,0 +1,84 @@
+name: "#️⃣ Get image tag (action)"
+
+description: This action gets the image tag from the commit ref or input (if provided)
+
+outputs:
+  tag:
+    description: The image tag
+    value: ${{ steps.get_tag.outputs.tag }}
+  is_semver:
+    description: Whether the tag is a semantic version
+    value: ${{ steps.check_semver.outputs.is_semver }}
+
+inputs:
+  tag:
+    description: The image tag. If this is set it will return the tag as is.
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: "#️⃣ Get image tag (step)"
+      id: get_tag
+      shell: bash
+      run: |
+        if [[ -n "${{ inputs.tag }}" ]]; then
+          tag="${{ inputs.tag }}"
+        elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+          if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            version="${GITHUB_REF_NAME#v.docker.}"
+            tag="v${version}"
+          elif [[ "${{ github.ref_name }}" == build-* ]]; then
+            tag="${GITHUB_REF_NAME#build-}"
+          else
+            echo "Invalid git tag: ${{ github.ref_name }}"
+            exit 1
+          fi
+        elif [[ "${{ github.ref_name }}" == "main" ]]; then
+          tag="main"
+        else
+          echo "Invalid git ref: ${{ github.ref }}"
+          exit 1
+        fi
+        echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+    - name: 🔍 Check for validity
+      id: check_validity
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      run: |
+        if [[ "${tag}" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then
+          echo "Tag is valid: ${tag}"
+        else
+          echo "Tag is not valid: ${tag}"
+          exit 1
+        fi
+
+    - name: 🆚 Check for semver
+      id: check_semver
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      # Will match most semver formats except build metadata, i.e. v1.2.3+build.1
+      # Valid matches:
+      # v1.2.3
+      # v1.2.3-alpha
+      # v1.2.3-alpha.1
+      # v1.2.3-rc.1
+      # v1.2.3-beta-1
+      run: |
+        if [[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
+          echo "Tag is a semantic version: ${tag}"
+          is_semver=true
+        else
+          echo "Tag is not a semantic version: ${tag}"
+          is_semver=false
+        fi
+        echo "is_semver=${is_semver}" >> "$GITHUB_OUTPUT"
@@ -0,0 +1 @@
+This is the repo for Trigger.dev, a background jobs platform written in TypeScript. Our webapp at apps/webapp is a Remix 2.1 app that uses Node.js v20. Our SDK is an isomorphic TypeScript SDK at packages/trigger-sdk. Always prefer using isomorphic code like fetch, ReadableStream, etc. instead of Node.js specific code. Our tests are all vitest. We use prisma in internal-packages/database for our database interactions using PostgreSQL. For TypeScript, we usually use types over interfaces. We use zod a lot in packages/core and in the webapp. Avoid enums. Use strict mode. No default exports, use function declarations.
@@ -0,0 +1,70 @@
+# GitHub Action Tests
+
+This directory contains necessary files to allow local testing of GitHub Actions workflows, composite actions, etc. You will need to install [act](https://github.com/nektos/act) to perform tests.
+
+## Workflow tests
+
+Trigger specific workflow files by specifying their full path:
+
+```
+act -W .github/workflow/release.yml
+```
+
+You will likely need to override any custom runners we use, e.g. buildjet. For example:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override
+
+# override multiple images at the same time
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override
+```
+
+Trigger with specific event payloads to test pushing to branches or tags:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+# simulate push to main
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-main.json
+
+# simulate a `build-` prefixed tag
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-buld.json
+```
+
+By default, `act` will send a push event. To trigger a different event:
+
+```
+# basic syntax
+act <EVENT> ...
+
+# simulate a pull request
+act pull_request
+
+# only trigger a specific workflow
+act pull_request -W .github/workflow/pr_checks.yml
+```
+
+## Composite action tests
+
+The composite (custom) action tests can be run by triggering the `test-actions` workflow:
+
+```
+act -W .github/test/test-actions.yml
+```
+
+## Helpful flags
+
+- `--pull=false` - perform fully offline tests if all images are already present
+- `-j <job_name>` - run the specified job only
+- `-l push` - list all workflows with push triggers
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/build-buildtag"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.nonsemver"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.1.2.3"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-prod-anything"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-test-anything"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/1.2.3"
+}
@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/standard-tag"
+}
@@ -0,0 +1,152 @@
+name: Test Actions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  get-image-tag-none:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        uses: ./.github/actions/get-image-tag
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-null:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        with:
+          # this should behave exactly as when no tag is provided
+          tag: null
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-override:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with tag override
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "abc-123"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-string:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid string
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          # does not end with alphanumeric character
+          tag: "abc-123-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-prerelease:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with prerelease semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3-beta.4"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with basic semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          tag: "v1.2.3-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
@@ -0,0 +1,42 @@
+name: 📚 Docs Checks
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check-broken-links:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./docs
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 📦 Cache npm
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+          key: |
+            ${{ runner.os }}-mintlify
+          restore-keys: |
+            ${{ runner.os }}-mintlify
+
+      - name: 🔗 Check for broken links
+        run: npx mintlify@4.0.222 broken-links
@@ -1,74 +1,53 @@
-name: "🧪 E2E Tests"
+name: "E2E"
+
 on:
  workflow_call:
-jobs:
-  e2e:
-    name: "🧪 E2E Tests"
-    runs-on: buildjet-16vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME || vars.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN || vars.DOCKERHUB_TOKEN }}
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string

+jobs:
+  cli-v3:
+    name: "🧪 CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [buildjet-8vcpu-ubuntu-2204, windows-latest]
+        package-manager: ["npm", "pnpm"]
+    steps:
      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          fetch-depth: 0
-          submodules: recursive

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
        with:
          node-version: 20.11.1
-          cache: "pnpm"
-
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
+        run: pnpm install --frozen-lockfile --filter trigger.dev...

-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...

-      - name: Run Playwright tests
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Run E2E Tests
+        shell: bash
        run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @references/nextjs-test^...
-          cd apps/webapp && pnpm run build:server
-          cd ../..
-          pnpm --filter @trigger.dev/database generate
-          pnpm --filter @trigger.dev/otlp-importer generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+          LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e
@@ -2,13 +2,9 @@ name: 🤖 PR Checks

 on:
  pull_request:
-    branches:
-      - main
+    types: [opened, synchronize, reopened]
    paths-ignore:
-      - "**.md"
-      - "**.mdx"
-      - ".github/CODEOWNERS"
-      - ".github/ISSUE_TEMPLATE/**"
+      - "docs/**"

 concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -27,6 +23,39 @@ jobs:
    uses: ./.github/workflows/unit-tests.yml
    secrets: inherit

-  # e2e:
-  #   uses: ./.github/workflows/e2e.yml
-  #   secrets: inherit
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
+    secrets: inherit
+
+  preview-release:
+    name: Preview Release
+    needs: [typecheck, units, e2e]
+    if: github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: ⚡ Publish preview release
+        run: npx pkg-pr-new publish --no-template $(ls -d ./packages/*)
@@ -1,59 +0,0 @@
-name: "🐳 Publish Docker"
-on:
-  workflow_call:
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.get_version.outputs.version }}
-      short_sha: ${{ steps.get_commit.outputs.sha_short }}
-    steps:
-      - name: Setup Depot CLI
-        uses: depot/setup-action@v1
-        
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: 🆚 Get the version
-        id: get_version
-        run: |
-          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            if [[ $IMAGE_TAG == v.docker.* ]]; then
-              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
-              IMAGE_TAG="v${ORIGINAL_VERSION}"
-            elif [[ $IMAGE_TAG == build-* ]]; then
-              IMAGE_TAG="${IMAGE_TAG#build-}"
-            fi
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/main ]]; then
-            # Handle main branch specifically
-            IMAGE_TAG="main"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          else
-            echo "Invalid reference: ${GITHUB_REF}"
-            exit 1
-          fi
-          echo "::set-output name=version::${IMAGE_TAG}"
-      - name: 🔢 Get the commit hash
-        id: get_commit
-        run: |
-          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
-
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐳 Build image and push to GitHub Container Registry
-        uses: depot/build-push-action@v1
-        with:
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }}
-          push: true
@@ -1,94 +0,0 @@
-name: "🚢 Publish Infra Images"
-
-on:
-  push:
-    tags:
-      - "infra-dev-*"
-      - "infra-test-*"
-      - "infra-prod-*"
-    paths:
-      - ".github/workflows/publish.yml"
-      - "packages/**"
-      - "!packages/**/*.md"
-      - "!packages/**/*.eslintrc"
-      - "apps/**"
-      - "!apps/**/*.md"
-      - "!apps/**/*.eslintrc"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "!integrations/**/*.eslintrc"
-      - "pnpm-lock.yaml"
-      - "pnpm-workspace.yaml"
-      - "turbo.json"
-      - "docker/Dockerfile"
-      - "docker/scripts/**"
-      - "tests/**"
-
-permissions:
-  id-token: write
-  packages: write
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-
-env:
-  AWS_REGION: us-east-1
-
-jobs:
-  build:
-    strategy:
-      matrix:
-        package: [coordinator, kubernetes-provider]
-    runs-on: buildjet-16vcpu-ubuntu-2204
-    env:
-      DOCKER_BUILDKIT: "1"
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Generate image reference
-        id: prep
-        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
-        run: |
-          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
-          sha=${GITHUB_SHA::7}
-          ts=$(date +%s)
-          if [[ "${{ matrix.package }}" == *-provider ]]; then
-            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
-            repository=provider/${provider_type}
-          else
-            repository=${{ matrix.package }}
-          fi
-          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
-          echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # ..to avoid rate limits when pulling images
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: 🚢 Build Container Image
-        run: |
-          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
-
-      # ..to push image
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag infra_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
-          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
-          IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}
@@ -0,0 +1,66 @@
+name: "🐳 Publish Webapp"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
+    outputs:
+      version: ${{ steps.get_tag.outputs.tag }}
+      short_sha: ${{ steps.get_commit.outputs.sha_short }}
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: "#️⃣ Get the image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🔢 Get the commit hash
+        id: get_commit
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v3
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            image_tags=$image_tags,$ref_without_tag:v3
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true
@@ -0,0 +1,87 @@
+name: "⚒️ Publish Worker"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
+
+permissions:
+  packages: write
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, docker-provider, kubernetes-provider]
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🐋 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
+
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
+        if: steps.get_tag.outputs.is_semver == 'true'
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+          docker push "$REGISTRY/$REPOSITORY:v3"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
@@ -1,6 +1,12 @@
 name: 🚀 Publish Trigger.dev Docker

 on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        required: true
+        type: string
  push:
    branches:
      - main
@@ -8,11 +14,13 @@ on:
      - "v.docker.*"
      - "build-*"
    paths:
+      - ".github/actions/**/*.yml"
      - ".github/workflows/publish.yml"
      - ".github/workflows/typecheck.yml"
      - ".github/workflows/unit-tests.yml"
      - ".github/workflows/e2e.yml"
-      - ".github/workflows/publish-docker.yml"
+      - ".github/workflows/publish-webapp.yml"
+      - ".github/workflows/publish-worker.yml"
      - "packages/**"
      - "!packages/**/*.md"
      - "!packages/**/*.eslintrc"
@@ -49,11 +57,16 @@ jobs:
    uses: ./.github/workflows/unit-tests.yml
    secrets: inherit

-  # e2e:
-  #   uses: ./.github/workflows/e2e.yml
-  #   secrets: inherit
-
-  publish:
+  publish-webapp:
    needs: [typecheck, units]
-    uses: ./.github/workflows/publish-docker.yml
+    uses: ./.github/workflows/publish-webapp.yml
    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
+
+  publish-worker:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-worker.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
@@ -5,50 +5,50 @@ on:
    branches:
      - main
    paths-ignore:
+      - "docs/**"
      - "**.md"
      - ".github/CODEOWNERS"
      - ".github/ISSUE_TEMPLATE/**"

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
  release:
    name: 🦋 Changesets Release
-    runs-on: buildjet-8vcpu-ubuntu-2204
-    if: |
-      github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: ubuntu-latest
+    if: github.repository == 'triggerdotdev/trigger.dev'
    outputs:
-      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
      published: ${{ steps.changesets.outputs.published }}
+      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
+      published_package_version: ${{ steps.get_version.outputs.package_version }}
    steps:
-      - name: 🛑 Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-
      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
        with:
          node-version: 20.11.1
          cache: "pnpm"

-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
      - name: 📥 Download deps
        run: pnpm install --frozen-lockfile

      - name: 📀 Generate Prisma Client
        run: pnpm run generate

+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
      - name: 🔎 Type check
        run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"

@@ -63,6 +63,7 @@ jobs:
        # PR is merged, the workflow will run again and this action will build +
        # publish to npm.
      - name: 🚀 PR / Publish
+        if: ${{ !env.ACT }}
        id: changesets
        uses: changesets/action@v1
        with:
@@ -70,7 +71,31 @@ jobs:
          commit: "chore: Update version for release"
          title: "chore: Update version for release"
          publish: pnpm run changeset:release
-          createGithubReleases: false
+          createGithubReleases: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # - name: 🚀 PR / Publish (mock)
+      #   if: ${{ env.ACT }}
+      #   id: changesets
+      #   run: |
+      #     echo "published=true" >> "$GITHUB_OUTPUT"
+      #     echo "publishedPackages=[{\"name\": \"@xx/xx\", \"version\": \"1.2.0\"}, {\"name\": \"@xx/xy\", \"version\": \"0.8.9\"}]" >> "$GITHUB_OUTPUT"
+
+      - name: 📦 Get package version
+        if: steps.changesets.outputs.published == 'true'
+        id: get_version
+        run: |
+          package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
+          echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    needs: release
+    uses: ./.github/workflows/publish.yml
+    secrets: inherit
+    # if: needs.release.outputs.published == 'true'
+    # disable automatic publishing for now
+    if: false
+    with:
+      image_tag: v${{ needs.release.outputs.published_package_version }}
@@ -1,32 +1,29 @@
 name: "ʦ TypeScript"
+
 on:
  workflow_call:
+
 jobs:
  typecheck:
    runs-on: buildjet-8vcpu-ubuntu-2204

    steps:
      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
        with:
          node-version: 20.11.1
          cache: "pnpm"

-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
      - name: 📥 Download deps
        run: pnpm install --frozen-lockfile

@@ -34,4 +31,7 @@ jobs:
        run: pnpm run generate

      - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
+        run: pnpm run typecheck
+
+      - name: 🔎 Check exports
+        run: pnpm run check-exports
@@ -1,45 +1,46 @@
 name: "🧪 Unit Tests"
+
 on:
  workflow_call:
+
 jobs:
  unitTests:
    name: "🧪 Unit Tests"
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: buildjet-16vcpu-ubuntu-2204
    steps:
      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
        with:
          node-version: 20.11.1
          cache: "pnpm"

-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: ⎔ Setup Deno
-        uses: denoland/setup-deno@v1
-        with:
-          deno-version: v1.x
-
-      - name: ⎔ Setup bun
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: "1.0.15"
-
      - name: 📥 Download deps
        run: pnpm install --frozen-lockfile

-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test --filter webapp
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "secret"
+      
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test --filter "@trigger.dev/*"
+      
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test --filter "@internal/*"
@@ -12,7 +12,6 @@ coverage
 # next.js
 .next/
 out/
-build
 dist
 packages/**/dist

@@ -54,4 +53,8 @@ apps/**/public/build
 /playwright/.cache/

 .cosine
-.trigger/
+.trigger
+.tshy*
+.yarn
+*.tsbuildinfo
+/packages/cli-v3/src/package.json
@@ -1,3 +1,3 @@
-[submodule "packages/otlp-importer/protos"]
-	path = packages/otlp-importer/protos
+[submodule "internal-packages/otlp-importer/protos"]
+	path = internal-packages/otlp-importer/protos
 	url = https://github.com/open-telemetry/opentelemetry-proto.git
@@ -1,2 +1,3 @@
 link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
+public-hoist-pattern[]=*prisma*
+prefer-workspace-packages=true
@@ -7,5 +7,4 @@ tailwind.css
 **/.react-email/
 **/storybook-static/
 **/.changeset/
-**/build/
 **/dist/
@@ -1,9 +1,4 @@
 {
-	"recommendations": [
-        "astro-build.astro-vscode",
-        "denoland.vscode-deno"
-    ],
-	"unwantedRecommendations": [
-		
-	]
-}
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
+}
@@ -13,6 +13,15 @@
      "cwd": "${workspaceFolder}",
      "sourceMaps": true
    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug realtimeStreams.test.ts",
+      "command": "pnpm run test -t RealtimeStreams",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
    {
      "type": "chrome",
      "request": "launch",
@@ -23,25 +32,64 @@
    {
      "type": "node-terminal",
      "request": "launch",
-      "name": "Debug v2 job catalog",
-      "command": "pnpm run byo-auth",
-      "envFile": "${workspaceFolder}/references/job-catalog/.env",
-      "cwd": "${workspaceFolder}/references/job-catalog",
+      "name": "Debug V3 init CLI",
+      "command": "pnpm exec trigger init",
+      "cwd": "${workspaceFolder}/references/init-shell",
      "sourceMaps": true
    },
    {
      "type": "node-terminal",
      "request": "launch",
      "name": "Debug V3 Dev CLI",
-      "command": "pnpm exec triggerdev dev --log-level debug",
+      "command": "pnpm exec trigger dev",
      "cwd": "${workspaceFolder}/references/v3-catalog",
      "sourceMaps": true
    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug Dev Next.js Realtime",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/nextjs-realtime",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug prisma-catalog deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/prisma-catalog",
+      "sourceMaps": true
+    },
    {
      "type": "node-terminal",
      "request": "launch",
      "name": "Debug V3 Deploy CLI",
-      "command": "pnpm exec triggerdev deploy",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 list-profiles CLI",
+      "command": "pnpm exec trigger list-profiles --log-level debug",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 update CLI",
+      "command": "pnpm exec trigger update",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
      "cwd": "${workspaceFolder}/references/v3-catalog",
      "sourceMaps": true
    },
@@ -52,6 +100,22 @@
      "port": 9229,
      "restart": true,
      "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug CLI e2e tests",
+      "command": "MOD=otel-telemetry-loader pnpm run test:e2e",
+      "cwd": "${workspaceFolder}/packages/cli-v3",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "debug v3 hello-world dev",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
    }
  ]
 }
@@ -1,4 +1,9 @@
 {
  "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"],
-  "debug.toolBarLocation": "commandCenter"
+  "debug.toolBarLocation": "commandCenter",
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "search.exclude": {
+    "**/node_modules/**": true,
+    "packages/cli-v3/e2e": true
+  }
 }
@@ -10,13 +10,14 @@ Thank you for helping us make Trigger.dev even better! 🤩

 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.

 ### Prerequisites

- [Node.js](https://nodejs.org/en) version >=16.x
- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)

 ### Setup

@@ -33,15 +34,19 @@ branch are tagged into a release monthly.
   ```
   cd trigger.dev
   ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
   ```
   pnpm i
   ```
-4. Create your `.env` file
+6. Create your `.env` file
   ```
   cp .env.example .env
   ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:

   `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:

@@ -51,7 +56,7 @@ branch are tagged into a release monthly.

   Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.

-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)

   ```
   pnpm run docker
@@ -59,19 +64,15 @@ branch are tagged into a release monthly.

   This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.

-7. Migrate the database
+9. Migrate the database
   ```
   pnpm run db:migrate
   ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.

 ## Running

@@ -83,99 +84,96 @@ branch are tagged into a release monthly.

   It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)

-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.

-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog

-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).

-## Adding and running migrations
+### First-time setup

-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:

-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.

-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.

-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.

-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI

 ```sh
-cd packages/cli
-pnpm run dev
-```
-
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in references directory
-
-```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
 pnpm i
-pnpm exec trigger-cli init
 ```

-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:

 ```sh
-pnpm run dev
+cd references/v3-catalog
+cp .env.example .env
+pnpm exec trigger login -a http://localhost:3030
 ```

-7. Open a new terminal window, and then run the `dev` command like so:
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:

 ```sh
-pnpm exec trigger-cli dev
+cd references/v3-catalog
+pnpm exec trigger login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec trigger dev --profile local
+pnpm exec trigger deploy --profile local
 ```

-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+### Running

-## Running end-to-end webapp tests
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev
+```
+
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
+
+```sh
+pnpm run build --filter trigger.dev
+```
+
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
+
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec trigger dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec trigger dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)

 To run the end-to-end tests, follow the steps below:

@@ -223,6 +221,26 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```

+## Adding and running migrations
+
+1. Modify internal-packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs

 The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.
@@ -5,86 +5,86 @@
  <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
  
-### The open source background jobs framework
+### Open source background jobs with no timeouts

 [Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)

 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)

 </div>

-# About Trigger.dev
+## About Trigger.dev

-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs. Write normal async code, deploy, and never hit a timeout.

-## Long running Jobs on serverless
+### Key features:

-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
-
- Auto-resume after a function timeout
- Auto-resume after a server outage
- Add delays of up to a year
+- JavaScript and TypeScript SDK
+- Write reliable code by default
+- No timeouts
+- Retries (with exponential backoff)
+- Queues and concurrency controls
+- Schedules and crons
+- Full Observability; logs, live trace views, advanced filtering
+- Custom alerts, get notified by email, Slack or webhooks
+- No infrastructure to manage
+- Elastic (scaling)
+- Works with your existing tech stack

 ## In your codebase

-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.

-## Secure by design
+```ts
+import { task } from "@trigger.dev/sdk/v3";

-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```

-## Don't worry about deployment
+## Deployment

-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always self-host.
+
+## Environments
+
+We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.

 ## Full visibility of every job run

-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.

-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
-
-## Our progress
-
-We’re building the most comprehensive and easy-to-use background jobs framework for developers.
-
-Click the links to join the discussions about our upcoming features.
-
-| Feature                                                                              | What it does                                        | Status |
-| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
-| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
-| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
-| Cloud                                                                                | Just write code, no deployment required             | ✅     |
-| Dashboard                                                                            | View every Task in every Run                        | ✅     |
-| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
-| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
-| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
-| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
-| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
-| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
-| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
-| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
-| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
-| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
+![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)

 # Getting started

-Visit our docs [here](https://trigger.dev/docs).
+The quickest way to get started is to create an account and project in our [web app](https://cloud.trigger.dev), and follow the instructions in the onboarding. Build and deploy your first task in minutes.

-## Self-host
+### Useful links:

-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+- [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
+- [How it works](https://trigger.dev/docs/v3/how-it-works) - understand how Trigger.dev works under the hood
+- [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
+
+## Self-hosting
+
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+We also have a dedicated self-hosting channel in our [Discord server](https://trigger.dev/discord) for support.

 ## Development

 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).

-## Meet the Amazing People Behind This Project 🚀
+## Meet the Amazing People Behind This Project:

 <a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
  <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				This is the repo for Trigger.dev, a background jobs platform written in TypeScript. Our webapp at apps/webapp is a Remix 2.1 app that uses Node.js v20. Our SDK is an isomorphic TypeScript SDK at packages/trigger-sdk. Always prefer using isomorphic code like fetch, ReadableStream, etc. instead of Node.js specific code. Our tests are all vitest. We use prisma in internal-packages/database for our database interactions using PostgreSQL. For TypeScript, we usually use types over interfaces. We use zod a lot in packages/core and in the webapp. Avoid enums. Use strict mode. No default exports, use function declarations.