pythonExtension and python runtime improvements (#1734)
* pythonExtension and python runtime improvements * Adding streaming support * Use writeFileSync * Restructure extension docs and add python extension docs * Fix broken link * Update docs/config/extensions/overview.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Update docs/config/extensions/aptGet.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Update docs/config/extensions/custom.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Add environment variable support --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
Eric Allam
+9
-452
@@ -315,479 +315,36 @@ Build extension allow you to hook into the build system and customize the build
|
||||
|
||||
#### additionalFiles
|
||||
|
||||
Import the `additionalFiles` build extension and use it in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { additionalFiles } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This will copy the files specified in the `files` array to the build directory. The `files` array can contain globs. The output paths will match the path of the file, relative to the root of the project.
|
||||
|
||||
<Note>The root of the project is the directory that contains the trigger.config.ts file</Note>
|
||||
See the [additionalFiles documentation](/config/extensions/additionalFiles) for more information.
|
||||
|
||||
#### `additionalPackages`
|
||||
|
||||
Import the `additionalPackages` build extension and use it in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { additionalPackages } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [additionalPackages({ packages: ["wrangler"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This allows you to include additional packages in the build that are not automatically included via imports. This is useful if you want to install a package that includes a CLI tool that you want to invoke in your tasks via `exec`. We will try to automatically resolve the version of the package but you can specify the version by using the `@` symbol:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [additionalPackages({ packages: ["wrangler@1.19.0"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
See the [additionalPackages documentation](/config/extensions/additionalPackages) for more information.
|
||||
|
||||
#### `emitDecoratorMetadata`
|
||||
|
||||
If you need support for the `emitDecoratorMetadata` typescript compiler option, import the `emitDecoratorMetadata` build extension and use it in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [emitDecoratorMetadata()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This is usually required if you are using certain ORMs, like TypeORM, that require this option to be enabled. It's not enabled by default because there is a performance cost to enabling it.
|
||||
|
||||
<Note>
|
||||
emitDecoratorMetadata works by hooking into the esbuild bundle process and using the TypeScript
|
||||
compiler API to compile files where we detect the use of decorators. This means you must have
|
||||
`emitDecoratorMetadata` enabled in your `tsconfig.json` file, as well as `typescript` installed in
|
||||
your `devDependencies`.
|
||||
</Note>
|
||||
See the [emitDecoratorMetadata documentation](/config/extensions/emitDecoratorMetadata) for more information.
|
||||
|
||||
#### Prisma
|
||||
|
||||
If you are using Prisma, you should use the prisma build extension.
|
||||
|
||||
- Automatically handles copying Prisma files to the build directory
|
||||
- Generates the Prisma client during the deploy process
|
||||
- Optionally will migrate the database during the deploy process
|
||||
- Support for TypedSQL and multiple schema files
|
||||
- You can use `prismaSchemaFolder` to specify just the directory containing your schema file, instead of the full path
|
||||
- You can add the extension twice if you have multiple separate schemas in the same project (example below)
|
||||
|
||||
You can use it for a simple Prisma setup like this:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
version: "5.19.0", // optional, we'll automatically detect the version if not provided
|
||||
schema: "prisma/schema.prisma",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
<Note>
|
||||
This does not have any effect when running the `dev` command, only when running the `deploy`
|
||||
command.
|
||||
</Note>
|
||||
|
||||
If you want to also run migrations during the build process, you can pass in the `migrate` option:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
schema: "prisma/schema.prisma",
|
||||
migrate: true,
|
||||
directUrlEnvVarName: "DATABASE_URL_UNPOOLED", // optional - the name of the environment variable that contains the direct database URL if you are using a direct database URL
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated. Some examples where you may need to do this include when using the `prisma-kysely` or `prisma-json-types-generator` generators.
|
||||
|
||||
<CodeGroup>
|
||||
|
||||
```prisma schema.prisma
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("DATABASE_URL")
|
||||
directUrl = env("DATABASE_URL_UNPOOLED")
|
||||
}
|
||||
|
||||
// We only want to generate the prisma-client-js generator
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
generator kysely {
|
||||
provider = "prisma-kysely"
|
||||
output = "../../src/kysely"
|
||||
enumFileName = "enums.ts"
|
||||
fileName = "types.ts"
|
||||
}
|
||||
|
||||
generator json {
|
||||
provider = "prisma-json-types-generator"
|
||||
}
|
||||
```
|
||||
|
||||
```ts trigger.config.ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
schema: "prisma/schema.prisma",
|
||||
clientGenerator: "client",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
</CodeGroup>
|
||||
|
||||
If you are using [TypedSQL](https://www.prisma.io/typedsql), you'll need to enable it via the `typedSql` option:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
schema: "prisma/schema.prisma",
|
||||
typedSql: true,
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
<Note>
|
||||
The `prismaExtension` will inject the `DATABASE_URL` environment variable into the build process. Learn more about setting environment variables for deploying in our [Environment Variables](/deploy-environment-variables) guide.
|
||||
|
||||
These environment variables are only used during the build process and are not embedded in the final container image.
|
||||
|
||||
</Note>
|
||||
|
||||
If you have multiple separate schemas in the same project you can add the extension multiple times:
|
||||
|
||||
```ts
|
||||
prismaExtension({
|
||||
schema: 'prisma/schema/main.prisma',
|
||||
version: '6.2.0',
|
||||
migrate: false,
|
||||
}),
|
||||
prismaExtension({
|
||||
schema: 'prisma/schema/secondary.prisma',
|
||||
version: '6.2.0',
|
||||
migrate: false,
|
||||
}),
|
||||
```
|
||||
See the [prismaExtension documentation](/config/extensions/prismaExtension) for more information.
|
||||
|
||||
#### syncEnvVars
|
||||
|
||||
The `syncEnvVars` build extension replaces the deprecated `resolveEnvVars` export. Check out our [syncEnvVars documentation](/deploy-environment-variables#sync-env-vars-from-another-service) for more information.
|
||||
|
||||
```ts
|
||||
import { syncEnvVars } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [syncEnvVars()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
#### syncVercelEnvVars
|
||||
|
||||
The `syncVercelEnvVars` build extension syncs environment variables from your Vercel project to Trigger.dev.
|
||||
|
||||
<Note>
|
||||
You need to set the `VERCEL_ACCESS_TOKEN` and `VERCEL_PROJECT_ID` environment variables, or pass
|
||||
in the token and project ID as arguments to the `syncVercelEnvVars` build extension. If you're
|
||||
working with a team project, you'll also need to set `VERCEL_TEAM_ID`, which can be found in your
|
||||
team settings. You can find / generate the `VERCEL_ACCESS_TOKEN` in your Vercel
|
||||
[dashboard](https://vercel.com/account/settings/tokens). Make sure the scope of the token covers
|
||||
the project with the environment variables you want to sync.
|
||||
</Note>
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [syncVercelEnvVars()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
#### audioWaveform
|
||||
|
||||
Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [audioWaveform()], // uses verson 1.1.0 of audiowaveform by default
|
||||
},
|
||||
});
|
||||
```
|
||||
See the [syncEnvVars documentation](/config/extensions/syncEnvVars) for more information.
|
||||
|
||||
#### puppeteer
|
||||
|
||||
<ScrapingWarning />
|
||||
|
||||
To use Puppeteer in your project, add these build settings to your `trigger.config.ts` file:
|
||||
|
||||
```ts trigger.config.ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { puppeteer } from "@trigger.dev/build/extensions/puppeteer";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [puppeteer()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
And add the following environment variable in your Trigger.dev dashboard on the Environment Variables page:
|
||||
|
||||
```bash
|
||||
PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable",
|
||||
```
|
||||
|
||||
Follow [this example](/guides/examples/puppeteer) to get setup with Trigger.dev and Puppeteer in your project.
|
||||
See the [puppeteer documentation](/config/extensions/puppeteer) for more information.
|
||||
|
||||
#### ffmpeg
|
||||
|
||||
You can add the `ffmpeg` build extension to your build process:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { ffmpeg } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [ffmpeg()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
By default, this will install the version of `ffmpeg` that is available in the Debian package manager. If you need a specific version, you can pass in the version as an argument:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { ffmpeg } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [ffmpeg({ version: "6.0-4" })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This extension will also add the `FFMPEG_PATH` and `FFPROBE_PATH` to your environment variables, making it easy to use popular ffmpeg libraries like `fluent-ffmpeg`.
|
||||
|
||||
Note that `fluent-ffmpeg` needs to be added to [`external`](/config/config-file#external) in your `trigger.config.ts` file.
|
||||
|
||||
Follow [this example](/guides/examples/ffmpeg-video-processing) to get setup with Trigger.dev and FFmpeg in your project.
|
||||
See the [ffmpeg documentation](/config/extensions/ffmpeg) for more information.
|
||||
|
||||
#### esbuild plugins
|
||||
|
||||
You can easily add existing or custom esbuild plugins to your build process using the `esbuildPlugin` extension:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { esbuildPlugin } from "@trigger.dev/build/extensions";
|
||||
import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
esbuildPlugin(
|
||||
sentryEsbuildPlugin({
|
||||
org: process.env.SENTRY_ORG,
|
||||
project: process.env.SENTRY_PROJECT,
|
||||
authToken: process.env.SENTRY_AUTH_TOKEN,
|
||||
}),
|
||||
// optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins
|
||||
{ placement: "last", target: "deploy" }
|
||||
),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
See the [esbuild plugins documentation](/config/extensions/esbuildPlugin) for more information.
|
||||
|
||||
#### aptGet
|
||||
|
||||
You can install system packages into the deployed image using using the `aptGet` extension:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { aptGet } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [aptGet({ packages: ["ffmpeg"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
If you want to install a specific version of a package, you can specify the version like this:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [aptGet({ packages: ["ffmpeg=6.0-4"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
#### Custom extensions
|
||||
|
||||
You can create your own extensions to further customize the build process. Extensions are an object with a `name` and zero or more lifecycle hooks (`onBuildStart` and `onBuildComplete`) that allow you to modify the `BuildContext` object that is passed to the build process through adding layers. For example, this is how the `aptGet` extension is implemented:
|
||||
|
||||
```ts
|
||||
import { BuildExtension } from "@trigger.dev/core/v3/build";
|
||||
|
||||
export type AptGetOptions = {
|
||||
packages: string[];
|
||||
};
|
||||
|
||||
export function aptGet(options: AptGetOptions): BuildExtension {
|
||||
return {
|
||||
name: "aptGet",
|
||||
onBuildComplete(context) {
|
||||
if (context.target === "dev") {
|
||||
return;
|
||||
}
|
||||
|
||||
context.logger.debug("Adding apt-get layer", {
|
||||
pkgs: options.packages,
|
||||
});
|
||||
|
||||
context.addLayer({
|
||||
id: "apt-get",
|
||||
image: {
|
||||
pkgs: options.packages,
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
Instead of creating this function and worrying about types, you can define an extension inline in your `trigger.config.ts` file:
|
||||
|
||||
```ts trigger.config.ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "aptGet",
|
||||
onBuildComplete(context) {
|
||||
if (context.target === "dev") {
|
||||
return;
|
||||
}
|
||||
|
||||
context.logger.debug("Adding apt-get layer", {
|
||||
pkgs: ["ffmpeg"],
|
||||
});
|
||||
|
||||
context.addLayer({
|
||||
id: "apt-get",
|
||||
image: {
|
||||
pkgs: ["ffmpeg"],
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
We'll be expanding the documentation on how to create custom extensions in the future, but for now you are encouraged to look at the existing extensions in the `@trigger.dev/build` package for inspiration, which you can see in our repo [here](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/build/src/extensions)
|
||||
See the [aptGet documentation](/config/extensions/aptGet) for more information.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
title: "Additional Files"
|
||||
sidebarTitle: "additionalFiles"
|
||||
description: "Use the additionalFiles build extension to copy additional files to the build directory"
|
||||
---
|
||||
|
||||
Import the `additionalFiles` build extension and use it in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { additionalFiles } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This will copy the files specified in the `files` array to the build directory. The `files` array can contain globs. The output paths will match the path of the file, relative to the root of the project.
|
||||
|
||||
This extension effects both the `dev` and the `deploy` commands, and the resulting paths will be the same for both.
|
||||
|
||||
<Note>The root of the project is the directory that contains the trigger.config.ts file</Note>
|
||||
@@ -0,0 +1,36 @@
|
||||
---
|
||||
title: "Additional Packages"
|
||||
sidebarTitle: "additionalPackages"
|
||||
description: "Use the additionalPackages build extension to include additional packages in the build"
|
||||
---
|
||||
|
||||
Import the `additionalPackages` build extension and use it in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { additionalPackages } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [additionalPackages({ packages: ["wrangler"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This allows you to include additional packages in the build that are not automatically included via imports. This is useful if you want to install a package that includes a CLI tool that you want to invoke in your tasks via `exec`. We will try to automatically resolve the version of the package but you can specify the version by using the `@` symbol:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [additionalPackages({ packages: ["wrangler@1.19.0"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This extension does not do anything in `dev` mode, but it will install the packages in the build directory when you run `deploy`. The packages will be installed in the `node_modules` directory in the build directory.
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
title: "apt-get"
|
||||
sidebarTitle: "aptGet"
|
||||
description: "Use the aptGet build extension to install system packages into the deployed image"
|
||||
---
|
||||
|
||||
You can install system packages into the deployed image using the `aptGet` extension:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { aptGet } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [aptGet({ packages: ["ffmpeg"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
If you want to install a specific version of a package, you can specify the version like this:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [aptGet({ packages: ["ffmpeg=6.0-4"] })],
|
||||
},
|
||||
});
|
||||
```
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
title: "Audio Waveform"
|
||||
sidebarTitle: "audioWaveform"
|
||||
description: "Use the audioWaveform build extension to add support for Audio Waveform in your project"
|
||||
---
|
||||
|
||||
Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [audioWaveform()], // uses verson 1.1.0 of audiowaveform by default
|
||||
},
|
||||
});
|
||||
```
|
||||
@@ -0,0 +1,380 @@
|
||||
---
|
||||
title: "Custom build extensions"
|
||||
sidebarTitle: "Custom"
|
||||
description: "Customize how your project is built and deployed to Trigger.dev with your own custom build extensions"
|
||||
---
|
||||
|
||||
Build extensions allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). See our [build extension overview](/config/extensions/overview) for more information on how to install and use our built-in extensions. Build extensions can do the following:
|
||||
|
||||
- Add additional files to the build
|
||||
- Add dependencies to the list of externals
|
||||
- Add esbuild plugins
|
||||
- Add additional npm dependencies
|
||||
- Add additional system packages to the image build container
|
||||
- Add commands to run in the image build container
|
||||
- Add environment variables to the image build container
|
||||
- Sync environment variables to your Trigger.dev project
|
||||
|
||||
## Creating a build extension
|
||||
|
||||
Build extensions are added to your `trigger.config.ts` file, with a required `name` and optional build hook functions. Here's a simple example of a build extension that just logs a message when the build starts:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
console.log("Build starting!");
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
You can also extract that out into a function instead of defining it inline, in which case you will need to import the `BuildExtension` type from the `@trigger.dev/build` package:
|
||||
|
||||
<Note>
|
||||
You'll need to add the `@trigger.dev/build` package to your `devDependencies` before the below
|
||||
code will work. Make sure it's version matches that of the installed `@trigger.dev/sdk` package.
|
||||
</Note>
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { BuildExtension } from "@trigger.dev/build";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [myExtension()],
|
||||
},
|
||||
});
|
||||
|
||||
function myExtension(): BuildExtension {
|
||||
return {
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
console.log("Build starting!");
|
||||
},
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
## Build hooks
|
||||
|
||||
### externalsForTarget
|
||||
|
||||
This allows the extension to add additional dependencies to the list of externals for the build. This is useful for dependencies that are not included in the bundle, but are expected to be available at runtime.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
externalsForTarget: async (target) => {
|
||||
return ["my-dependency"];
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### onBuildStart
|
||||
|
||||
This hook runs before the build starts. It receives the `BuildContext` object as an argument.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
console.log("Build starting!");
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
If you want to add an esbuild plugin, you must do so in the `onBuildStart` hook. Here's an example of adding a custom esbuild plugin:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
context.registerPlugin({
|
||||
name: "my-plugin",
|
||||
setup(build) {
|
||||
build.onLoad({ filter: /.*/, namespace: "file" }, async (args) => {
|
||||
return {
|
||||
contents: "console.log('Hello, world!')",
|
||||
loader: "js",
|
||||
};
|
||||
});
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
You can use the `BuildContext.target` property to determine if the build is for `dev` or `deploy`:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
if (context.target === "dev") {
|
||||
console.log("Building for dev");
|
||||
} else {
|
||||
console.log("Building for deploy");
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### onBuildComplete
|
||||
|
||||
This hook runs after the build completes. It receives the `BuildContext` object and a `BuildManifest` object as arguments. This is where you can add in one or more `BuildLayer`'s to the context.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildComplete: async (context, manifest) => {
|
||||
context.addLayer({
|
||||
id: "more-dependencies",
|
||||
dependencies,
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
See the [addLayer](#addlayer) documentation for more information on how to use `addLayer`.
|
||||
|
||||
## BuildTarget
|
||||
|
||||
Can either be `dev` or `deploy`, matching the CLI command name that is being run.
|
||||
|
||||
```sh
|
||||
npx trigger.dev@latest dev # BuildTarget is "dev"
|
||||
npx trigger.dev@latest deploy # BuildTarget is "deploy"
|
||||
```
|
||||
|
||||
## BuildContext
|
||||
|
||||
### addLayer()
|
||||
|
||||
<ParamField path="layer" type="BuildLayer">
|
||||
The layer to add to the build context. See the [BuildLayer](#buildlayer) documentation for more
|
||||
information.
|
||||
</ParamField>
|
||||
|
||||
### registerPlugin()
|
||||
|
||||
<ParamField path="plugin" type="esbuild.Plugin" required>
|
||||
The esbuild plugin to register.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="options" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="target" type="BuildTarget">
|
||||
An optional target to register the plugin for. If not provided, the plugin will be registered
|
||||
for all targets.
|
||||
</ResponseField>
|
||||
<ResponseField name="placement" type="first | last">
|
||||
An optional placement for the plugin. If not provided, the plugin will be registered in place.
|
||||
This allows you to control the order of plugins.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
### resolvePath()
|
||||
|
||||
Resolves a path relative to the project's working directory.
|
||||
|
||||
<ParamField path="path" type="string">
|
||||
The path to resolve.
|
||||
</ParamField>
|
||||
|
||||
```ts
|
||||
const resolvedPath = context.resolvePath("my-other-dependency");
|
||||
```
|
||||
|
||||
### properties
|
||||
|
||||
<ParamField path="target" type="BuildTarget">
|
||||
The target of the build, either `dev` or `deploy`.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="config" type="ResolvedConfig">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="runtime" type="string">
|
||||
The runtime of the project (either node or bun)
|
||||
</ResponseField>
|
||||
<ResponseField name="project" type="string">
|
||||
The project ref
|
||||
</ResponseField>
|
||||
<ResponseField name="dirs" type="string[]">
|
||||
The trigger directories to search for tasks
|
||||
</ResponseField>
|
||||
<ResponseField name="build" type="object">
|
||||
The build configuration object
|
||||
</ResponseField>
|
||||
<ResponseField name="workingDir" type="string">
|
||||
The working directory of the project
|
||||
</ResponseField>
|
||||
<ResponseField name="workspaceDir" type="string">
|
||||
The root workspace directory of the project
|
||||
</ResponseField>
|
||||
<ResponseField name="packageJsonPath" type="string">
|
||||
The path to the package.json file
|
||||
</ResponseField>
|
||||
<ResponseField name="lockfilePath" type="string">
|
||||
The path to the lockfile (package-lock.json, yarn.lock, or pnpm-lock.yaml)
|
||||
</ResponseField>
|
||||
<ResponseField name="configFile" type="string">
|
||||
The path to the trigger.config.ts file
|
||||
</ResponseField>
|
||||
<ResponseField name="tsconfigPath" type="string">
|
||||
The path to the tsconfig.json file
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="logger" type="BuildLogger">
|
||||
A logger object that can be used to log messages to the console.
|
||||
</ParamField>
|
||||
|
||||
## BuildLayer
|
||||
|
||||
<ParamField path="id" type="string">
|
||||
A unique identifier for the layer.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="commands" type="string[]">
|
||||
An array of commands to run in the image build container.
|
||||
|
||||
```ts
|
||||
commands: ["echo 'Hello, world!'"];
|
||||
```
|
||||
|
||||
These commands are run after packages have been installed and the code copied into the container in the "build" stage of the Dockerfile. This means you cannot install system packages in these commands because they won't be available in the final stage. To do that, please use the `pkgs` property of the `image` object.
|
||||
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="image" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="pkgs" type="string[]">
|
||||
An array of system packages to install in the image build container.
|
||||
</ResponseField>
|
||||
<ResponseField name="instructions" type="string[]">
|
||||
An array of instructions to add to the Dockerfile.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="build" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="env" type="Record<string, string>">
|
||||
Environment variables to add to the image build container, but only during the "build" stage
|
||||
of the Dockerfile. This is where you'd put environment variables that are needed when running
|
||||
any of the commands in the `commands` array.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="deploy" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="env" type="Record<string, string>">
|
||||
Environment variables that should sync to the Trigger.dev project, which will then be avalable
|
||||
in your tasks at runtime. Importantly, these are NOT added to the image build container, but
|
||||
are instead added to the Trigger.dev project and stored securely.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="dependencies" type="Record<string, string>">
|
||||
An object of dependencies to add to the build. The key is the package name and the value is the
|
||||
version.
|
||||
|
||||
```ts
|
||||
dependencies: {
|
||||
"my-dependency": "^1.0.0",
|
||||
};
|
||||
```
|
||||
|
||||
</ParamField>
|
||||
|
||||
### examples
|
||||
|
||||
Add a command that will echo the value of an environment variable:
|
||||
|
||||
```ts
|
||||
context.addLayer({
|
||||
id: "my-layer",
|
||||
commands: [`echo $MY_ENV_VAR`],
|
||||
build: {
|
||||
env: {
|
||||
MY_ENV_VAR: "Hello, world!",
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
When creating a build extension, you may run into issues with the build process. One thing that can help is turning on `debug` logging when running either `dev` or `deploy`:
|
||||
|
||||
```sh
|
||||
npx trigger.dev@latest dev --log-level debug
|
||||
npx trigger.dev@latest deploy --log-level debug
|
||||
```
|
||||
|
||||
Another helpful tool is the `--dry-run` flag on the `deploy` command, which will bundle your project and generate the Containerfile (e.g. the Dockerfile) without actually deploying it. This can help you see what the final image will look like and debug any issues with the build process.
|
||||
|
||||
```sh
|
||||
npx trigger.dev@latest deploy --dry-run
|
||||
```
|
||||
|
||||
You should also take a look at our built in extensions for inspiration on how to create your own. You can find them in in [the source code here](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/build/src/extensions).
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
title: "Emit Decorator Metadata"
|
||||
sidebarTitle: "emitDecoratorMetadata"
|
||||
description: "Use the emitDecoratorMetadata build extension to enable support for the emitDecoratorMetadata TypeScript compiler option"
|
||||
---
|
||||
|
||||
If you need support for the `emitDecoratorMetadata` typescript compiler option, import the `emitDecoratorMetadata` build extension and use it in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [emitDecoratorMetadata()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This is usually required if you are using certain ORMs, like TypeORM, that require this option to be enabled. It's not enabled by default because there is a performance cost to enabling it.
|
||||
|
||||
<Note>
|
||||
emitDecoratorMetadata works by hooking into the esbuild bundle process and using the TypeScript
|
||||
compiler API to compile files where we detect the use of decorators. This means you must have
|
||||
`emitDecoratorMetadata` enabled in your `tsconfig.json` file, as well as `typescript` installed in
|
||||
your `devDependencies`.
|
||||
</Note>
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
title: "esbuild Plugin"
|
||||
sidebarTitle: "esbuildPlugin"
|
||||
description: "Use the esbuildPlugin build extension to add existing or custom esbuild plugins to your build process"
|
||||
---
|
||||
|
||||
You can easily add existing or custom esbuild plugins to your build process using the `esbuildPlugin` extension:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { esbuildPlugin } from "@trigger.dev/build/extensions";
|
||||
import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
esbuildPlugin(
|
||||
sentryEsbuildPlugin({
|
||||
org: process.env.SENTRY_ORG,
|
||||
project: process.env.SENTRY_PROJECT,
|
||||
authToken: process.env.SENTRY_AUTH_TOKEN,
|
||||
}),
|
||||
// optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins
|
||||
{ placement: "last", target: "deploy" }
|
||||
),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
title: "FFmpeg"
|
||||
sidebarTitle: "ffmpeg"
|
||||
description: "Use the ffmpeg build extension to include FFmpeg in your project"
|
||||
---
|
||||
|
||||
You can add the `ffmpeg` build extension to your build process:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { ffmpeg } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [ffmpeg()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
By default, this will install the version of `ffmpeg` that is available in the Debian package manager. If you need a specific version, you can pass in the version as an argument:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { ffmpeg } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [ffmpeg({ version: "6.0-4" })],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This extension will also add the `FFMPEG_PATH` and `FFPROBE_PATH` to your environment variables, making it easy to use popular ffmpeg libraries like `fluent-ffmpeg`.
|
||||
|
||||
Note that `fluent-ffmpeg` needs to be added to [`external`](/config/config-file#external) in your `trigger.config.ts` file.
|
||||
|
||||
Follow [this example](/guides/examples/ffmpeg-video-processing) to get setup with Trigger.dev and FFmpeg in your project.
|
||||
@@ -1,22 +1,14 @@
|
||||
---
|
||||
title: "Build extensions"
|
||||
sidebarTitle: "Overview"
|
||||
description: "Customize how your project is built and deployed to Trigger.dev with build extensions"
|
||||
---
|
||||
|
||||
Build extension allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). See our [trigger.config.ts reference](/config/config-file#extensions) for more information on how to install and use our built-in extensions. Build extensions can do the following:
|
||||
Build extensions allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying).
|
||||
|
||||
- Add additional files to the build
|
||||
- Add dependencies to the list of externals
|
||||
- Add esbuild plugins
|
||||
- Add additional npm dependencies
|
||||
- Add additional system packages to the image build container
|
||||
- Add commands to run in the image build container
|
||||
- Add environment variables to the image build container
|
||||
- Sync environment variables to your Trigger.dev project
|
||||
You can use pre-built extensions by installing the `@trigger.dev/build` package into your `devDependencies`, or you can create your own.
|
||||
|
||||
## Creating a build extension
|
||||
|
||||
Build extensions are added to your `trigger.config.ts` file, with a required `name` and optional build hook functions. Here's a simple example of a build extension that just logs a message when the build starts:
|
||||
Build extensions are added to your `trigger.config.ts` file under the `build.extensions` property:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
@@ -36,344 +28,39 @@ export default defineConfig({
|
||||
});
|
||||
```
|
||||
|
||||
You can also extract that out into a function instead of defining it inline, in which case you will need to import the `BuildExtension` type from the `@trigger.dev/build` package:
|
||||
|
||||
<Note>
|
||||
You'll need to add the `@trigger.dev/build` package to your `devDependencies` before the below
|
||||
code will work. Make sure it's version matches that of the installed `@trigger.dev/sdk` package.
|
||||
</Note>
|
||||
If you are using a pre-built extension, you can import it from the `@trigger.dev/build` package:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { BuildExtension } from "@trigger.dev/build";
|
||||
import { ffmpeg } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [myExtension()],
|
||||
},
|
||||
});
|
||||
|
||||
function myExtension(): BuildExtension {
|
||||
return {
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
console.log("Build starting!");
|
||||
},
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
## Build hooks
|
||||
|
||||
### externalsForTarget
|
||||
|
||||
This allows the extension to add additional dependencies to the list of externals for the build. This is useful for dependencies that are not included in the bundle, but are expected to be available at runtime.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
externalsForTarget: async (target) => {
|
||||
return ["my-dependency"];
|
||||
},
|
||||
},
|
||||
],
|
||||
extensions: [ffmpeg()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### onBuildStart
|
||||
## Built-in extensions
|
||||
|
||||
This hook runs before the build starts. It receives the `BuildContext` object as an argument.
|
||||
Trigger.dev provides a set of built-in extensions that you can use to customize how your project is built and deployed. These extensions are available out of the box and can be configured in your `trigger.config.ts` file.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
| Extension | Description |
|
||||
| :-------------------------------------------------------------------- | :----------------------------------------------------------------------------- |
|
||||
| [prismaExtension](/config/extensions/prismaExtension) | Using prisma in your Trigger.dev tasks |
|
||||
| [pythonExtension](/config/extensions/pythonExtension) | Execute Python scripts in your project |
|
||||
| [puppeteer](/config/extensions/puppeteer) | Use Puppeteer in your Trigger.dev tasks |
|
||||
| [ffmpeg](/config/extensions/ffmpeg) | Use FFmpeg in your Trigger.dev tasks |
|
||||
| [aptGet](/config/extensions/aptGet) | Install system packages in your build image |
|
||||
| [additionalFiles](/config/extensions/additionalFiles) | Copy additional files to your build image |
|
||||
| [additionalPackages](/config/extensions/additionalPackages) | Install additional npm packages in your build image |
|
||||
| [syncEnvVars](/config/extensions/syncEnvVars) | Automatically sync environment variables from external services to Trigger.dev |
|
||||
| [syncVercelEnvVars](/config/extensions/syncEnvVars#syncVercelEnvVars) | Automatically sync environment variables from Vercel to Trigger.dev |
|
||||
| [esbuildPlugin](/config/extensions/esbuildPlugin) | Add existing or custom esbuild extensions to customize your build process |
|
||||
| [emitDecoratorMetadata](/config/extensions/emitDecoratorMetadata) | Enable `emitDecoratorMetadata` in your TypeScript build |
|
||||
| [audioWaveform](/config/extensions/audioWaveform) | Add Audio Waveform to your build image |
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
console.log("Build starting!");
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
## Custom extensions
|
||||
|
||||
If you want to add an esbuild plugin, you must do so in the `onBuildStart` hook. Here's an example of adding a custom esbuild plugin:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
context.registerPlugin({
|
||||
name: "my-plugin",
|
||||
setup(build) {
|
||||
build.onLoad({ filter: /.*/, namespace: "file" }, async (args) => {
|
||||
return {
|
||||
contents: "console.log('Hello, world!')",
|
||||
loader: "js",
|
||||
};
|
||||
});
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
You can use the `BuildContext.target` property to determine if the build is for `dev` or `deploy`:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildStart: async (context) => {
|
||||
if (context.target === "dev") {
|
||||
console.log("Building for dev");
|
||||
} else {
|
||||
console.log("Building for deploy");
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### onBuildComplete
|
||||
|
||||
This hook runs after the build completes. It receives the `BuildContext` object and a `BuildManifest` object as arguments. This is where you can add in one or more `BuildLayer`'s to the context.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "my-project",
|
||||
build: {
|
||||
extensions: [
|
||||
{
|
||||
name: "my-extension",
|
||||
onBuildComplete: async (context, manifest) => {
|
||||
context.addLayer({
|
||||
id: "more-dependencies",
|
||||
dependencies,
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
See the [addLayer](#addlayer) documentation for more information on how to use `addLayer`.
|
||||
|
||||
## BuildTarget
|
||||
|
||||
Can either be `dev` or `deploy`, matching the CLI command name that is being run.
|
||||
|
||||
```sh
|
||||
npx trigger.dev@latest dev # BuildTarget is "dev"
|
||||
npx trigger.dev@latest deploy # BuildTarget is "deploy"
|
||||
```
|
||||
|
||||
## BuildContext
|
||||
|
||||
### addLayer()
|
||||
|
||||
<ParamField path="layer" type="BuildLayer">
|
||||
The layer to add to the build context. See the [BuildLayer](#buildlayer) documentation for more
|
||||
information.
|
||||
</ParamField>
|
||||
|
||||
### registerPlugin()
|
||||
|
||||
<ParamField path="plugin" type="esbuild.Plugin" required>
|
||||
The esbuild plugin to register.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="options" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="target" type="BuildTarget">
|
||||
An optional target to register the plugin for. If not provided, the plugin will be registered
|
||||
for all targets.
|
||||
</ResponseField>
|
||||
<ResponseField name="placement" type="first | last">
|
||||
An optional placement for the plugin. If not provided, the plugin will be registered in place.
|
||||
This allows you to control the order of plugins.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
### resolvePath()
|
||||
|
||||
Resolves a path relative to the project's working directory.
|
||||
|
||||
<ParamField path="path" type="string">
|
||||
The path to resolve.
|
||||
</ParamField>
|
||||
|
||||
```ts
|
||||
const resolvedPath = context.resolvePath("my-other-dependency");
|
||||
```
|
||||
|
||||
### properties
|
||||
|
||||
<ParamField path="target" type="BuildTarget">
|
||||
The target of the build, either `dev` or `deploy`.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="config" type="ResolvedConfig">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="runtime" type="string">
|
||||
The runtime of the project (either node or bun)
|
||||
</ResponseField>
|
||||
<ResponseField name="project" type="string">
|
||||
The project ref
|
||||
</ResponseField>
|
||||
<ResponseField name="dirs" type="string[]">
|
||||
The trigger directories to search for tasks
|
||||
</ResponseField>
|
||||
<ResponseField name="build" type="object">
|
||||
The build configuration object
|
||||
</ResponseField>
|
||||
<ResponseField name="workingDir" type="string">
|
||||
The working directory of the project
|
||||
</ResponseField>
|
||||
<ResponseField name="workspaceDir" type="string">
|
||||
The root workspace directory of the project
|
||||
</ResponseField>
|
||||
<ResponseField name="packageJsonPath" type="string">
|
||||
The path to the package.json file
|
||||
</ResponseField>
|
||||
<ResponseField name="lockfilePath" type="string">
|
||||
The path to the lockfile (package-lock.json, yarn.lock, or pnpm-lock.yaml)
|
||||
</ResponseField>
|
||||
<ResponseField name="configFile" type="string">
|
||||
The path to the trigger.config.ts file
|
||||
</ResponseField>
|
||||
<ResponseField name="tsconfigPath" type="string">
|
||||
The path to the tsconfig.json file
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="logger" type="BuildLogger">
|
||||
A logger object that can be used to log messages to the console.
|
||||
</ParamField>
|
||||
|
||||
## BuildLayer
|
||||
|
||||
<ParamField path="id" type="string">
|
||||
A unique identifier for the layer.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="commands" type="string[]">
|
||||
An array of commands to run in the image build container.
|
||||
|
||||
```ts
|
||||
commands: ["echo 'Hello, world!'"];
|
||||
```
|
||||
|
||||
These commands are run after packages have been installed and the code copied into the container in the "build" stage of the Dockerfile. This means you cannot install system packages in these commands because they won't be available in the final stage. To do that, please use the `pkgs` property of the `image` object.
|
||||
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="image" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="pkgs" type="string[]">
|
||||
An array of system packages to install in the image build container.
|
||||
</ResponseField>
|
||||
<ResponseField name="instructions" type="string[]">
|
||||
An array of instructions to add to the Dockerfile.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="build" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="env" type="Record<string, string>">
|
||||
Environment variables to add to the image build container, but only during the "build" stage
|
||||
of the Dockerfile. This is where you'd put environment variables that are needed when running
|
||||
any of the commands in the `commands` array.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="deploy" type="object">
|
||||
<Expandable title="properties">
|
||||
<ResponseField name="env" type="Record<string, string>">
|
||||
Environment variables that should sync to the Trigger.dev project, which will then be avalable
|
||||
in your tasks at runtime. Importantly, these are NOT added to the image build container, but
|
||||
are instead added to the Trigger.dev project and stored securely.
|
||||
</ResponseField>
|
||||
</Expandable>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="dependencies" type="Record<string, string>">
|
||||
An object of dependencies to add to the build. The key is the package name and the value is the
|
||||
version.
|
||||
|
||||
```ts
|
||||
dependencies: {
|
||||
"my-dependency": "^1.0.0",
|
||||
};
|
||||
```
|
||||
|
||||
</ParamField>
|
||||
|
||||
### examples
|
||||
|
||||
Add a command that will echo the value of an environment variable:
|
||||
|
||||
```ts
|
||||
context.addLayer({
|
||||
id: "my-layer",
|
||||
commands: [`echo $MY_ENV_VAR`],
|
||||
build: {
|
||||
env: {
|
||||
MY_ENV_VAR: "Hello, world!",
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
When creating a build extension, you may run into issues with the build process. One thing that can help is turning on `debug` logging when running either `dev` or `deploy`:
|
||||
|
||||
```sh
|
||||
npx trigger.dev@latest dev --log-level debug
|
||||
npx trigger.dev@latest deploy --log-level debug
|
||||
```
|
||||
|
||||
Another helpful tool is the `--dry-run` flag on the `deploy` command, which will bundle your project and generate the Containerfile (e.g. the Dockerfile) without actually deploying it. This can help you see what the final image will look like and debug any issues with the build process.
|
||||
|
||||
```sh
|
||||
npx trigger.dev@latest deploy --dry-run
|
||||
```
|
||||
|
||||
You should also take a look at our built in extensions for inspiration on how to create your own. You can find them in in [the source code here](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/build/src/extensions).
|
||||
If one of the built-in extensions doesn't meet your needs, you can create your own custom extension. See our [guide on creating custom build extensions](/config/extensions/custom) for more information.
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
---
|
||||
title: "Prisma"
|
||||
sidebarTitle: "prismaExtension"
|
||||
description: "Use the prismaExtension build extension to use Prisma with Trigger.dev"
|
||||
---
|
||||
|
||||
If you are using Prisma, you should use the prisma build extension.
|
||||
|
||||
- Automatically handles copying Prisma files to the build directory
|
||||
- Generates the Prisma client during the deploy process
|
||||
- Optionally will migrate the database during the deploy process
|
||||
- Support for TypedSQL and multiple schema files
|
||||
- You can use `prismaSchemaFolder` to specify just the directory containing your schema file, instead of the full path
|
||||
- You can add the extension twice if you have multiple separate schemas in the same project (example below)
|
||||
|
||||
You can use it for a simple Prisma setup like this:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
version: "5.19.0", // optional, we'll automatically detect the version if not provided
|
||||
schema: "prisma/schema.prisma",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
<Note>
|
||||
This does not have any effect when running the `dev` command, only when running the `deploy`
|
||||
command.
|
||||
</Note>
|
||||
|
||||
### Migrations
|
||||
|
||||
If you want to also run migrations during the build process, you can pass in the `migrate` option:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
schema: "prisma/schema.prisma",
|
||||
migrate: true,
|
||||
directUrlEnvVarName: "DATABASE_URL_UNPOOLED", // optional - the name of the environment variable that contains the direct database URL if you are using a direct database URL
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### clientGenerator
|
||||
|
||||
If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated. Some examples where you may need to do this include when using the `prisma-kysely` or `prisma-json-types-generator` generators.
|
||||
|
||||
<CodeGroup>
|
||||
|
||||
```prisma schema.prisma
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("DATABASE_URL")
|
||||
directUrl = env("DATABASE_URL_UNPOOLED")
|
||||
}
|
||||
|
||||
// We only want to generate the prisma-client-js generator
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
generator kysely {
|
||||
provider = "prisma-kysely"
|
||||
output = "../../src/kysely"
|
||||
enumFileName = "enums.ts"
|
||||
fileName = "types.ts"
|
||||
}
|
||||
|
||||
generator json {
|
||||
provider = "prisma-json-types-generator"
|
||||
}
|
||||
```
|
||||
|
||||
```ts trigger.config.ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
schema: "prisma/schema.prisma",
|
||||
clientGenerator: "client",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
</CodeGroup>
|
||||
|
||||
### TypedSQL
|
||||
|
||||
If you are using [TypedSQL](https://www.prisma.io/typedsql), you'll need to enable it via the `typedSql` option:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
prismaExtension({
|
||||
schema: "prisma/schema.prisma",
|
||||
typedSql: true,
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
<Note>
|
||||
The `prismaExtension` will inject the `DATABASE_URL` environment variable into the build process. Learn more about setting environment variables for deploying in our [Environment Variables](/deploy-environment-variables) guide.
|
||||
|
||||
These environment variables are only used during the build process and are not embedded in the final container image.
|
||||
|
||||
</Note>
|
||||
|
||||
### Multiple schemas
|
||||
|
||||
If you have multiple separate schemas in the same project you can add the extension multiple times:
|
||||
|
||||
```ts
|
||||
prismaExtension({
|
||||
schema: 'prisma/schema/main.prisma',
|
||||
version: '6.2.0',
|
||||
migrate: false,
|
||||
}),
|
||||
prismaExtension({
|
||||
schema: 'prisma/schema/secondary.prisma',
|
||||
version: '6.2.0',
|
||||
migrate: false,
|
||||
}),
|
||||
```
|
||||
@@ -0,0 +1,30 @@
|
||||
---
|
||||
title: "Puppeteer"
|
||||
sidebarTitle: "puppeteer"
|
||||
description: "Use the puppeteer build extension to enable support for Puppeteer in your project"
|
||||
---
|
||||
|
||||
<ScrapingWarning />
|
||||
|
||||
To use Puppeteer in your project, add these build settings to your `trigger.config.ts` file:
|
||||
|
||||
```ts trigger.config.ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { puppeteer } from "@trigger.dev/build/extensions/puppeteer";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [puppeteer()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
And add the following environment variable in your Trigger.dev dashboard on the Environment Variables page:
|
||||
|
||||
```bash
|
||||
PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable",
|
||||
```
|
||||
|
||||
Follow [this example](/guides/examples/puppeteer) to get setup with Trigger.dev and Puppeteer in your project.
|
||||
@@ -0,0 +1,182 @@
|
||||
---
|
||||
title: "Python"
|
||||
sidebarTitle: "pythonExtension"
|
||||
description: "Use the python build extension to add support for executing Python scripts in your project"
|
||||
---
|
||||
|
||||
If you need to execute Python scripts in your Trigger.dev project, you can use the `pythonExtension` build extension via the `@trigger.dev/python` package.
|
||||
|
||||
First, you'll need to install the `@trigger.dev/python` package:
|
||||
|
||||
```bash
|
||||
npm add @trigger.dev/python
|
||||
```
|
||||
|
||||
Then, you can use the `pythonExtension` build extension in your `trigger.config.ts` file:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
build: {
|
||||
extensions: [pythonExtension()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This will take care of adding python to the build image and setting up the necessary environment variables to execute Python scripts. You can then use our `python` utilities in the `@trigger.dev/python` package to execute Python scripts in your tasks. For example, running a Python script inline in a task:
|
||||
|
||||
```ts
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const myScript = task({
|
||||
id: "my-python-script",
|
||||
run: async () => {
|
||||
const result = await python.runInline(`print("Hello, world!")`);
|
||||
return result.stdout;
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Adding python scripts
|
||||
|
||||
You can automatically add python scripts to your project using the `scripts` option in the `pythonExtension` function. This will copy the specified scripts to the build directory during the deploy process. For example:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
build: {
|
||||
extensions: [
|
||||
pythonExtension({
|
||||
scripts: ["./python/**/*.py"],
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This will copy all Python files in the `python` directory to the build directory during the deploy process. You can then execute these scripts using the `python.runScript` function:
|
||||
|
||||
```ts
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const myScript = task({
|
||||
id: "my-python-script",
|
||||
run: async () => {
|
||||
const result = await python.runScript("./python/my_script.py", ["hello", "world"]);
|
||||
return result.stdout;
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
<Note>
|
||||
The pythonExtension will also take care of moving the scripts to the correct location during `dev`
|
||||
mode, so you can use the same exact path in development as you do in production.
|
||||
</Note>
|
||||
|
||||
## Using requirements files
|
||||
|
||||
If you have a `requirements.txt` file in your project, you can use the `requirementsFile` option in the `pythonExtension` function to install the required packages during the build process. For example:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
build: {
|
||||
extensions: [
|
||||
pythonExtension({
|
||||
requirementsFile: "./requirements.txt",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This will install the packages specified in the `requirements.txt` file during the build process. You can then use these packages in your Python scripts.
|
||||
|
||||
<Note>
|
||||
The `requirementsFile` option is only available in production mode. In development mode, you can
|
||||
install the required packages manually using the `pip` command.
|
||||
</Note>
|
||||
|
||||
## Virtual environments
|
||||
|
||||
If you are using a virtual environment in your project, you can use the `devPythonBinaryPath` option in the `pythonExtension` function to specify the path to the Python binary in the virtual environment. For example:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
build: {
|
||||
extensions: [
|
||||
pythonExtension({
|
||||
devPythonBinaryPath: ".venv/bin/python",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
This has no effect in production mode, but in development mode, it will use the specified Python binary to execute Python scripts.
|
||||
|
||||
## Streaming output
|
||||
|
||||
All of the `python` functions have a streaming version that allows you to stream the output of the Python script as it runs. For example:
|
||||
|
||||
```ts
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const myStreamingScript = task({
|
||||
id: "my-streaming-python-script",
|
||||
run: async () => {
|
||||
// You don't need to await the result
|
||||
const result = python.stream.runScript("./python/my_script.py", ["hello", "world"]);
|
||||
|
||||
// result is an async iterable/readable stream
|
||||
for await (const chunk of streamingResult) {
|
||||
console.log(chunk);
|
||||
}
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Environment variables
|
||||
|
||||
We automatically inject the environment variables in the `process.env` object when running Python scripts. You can access these environment variables in your Python scripts using the `os.environ` dictionary. For example:
|
||||
|
||||
```python
|
||||
import os
|
||||
|
||||
print(os.environ["MY_ENV_VAR"])
|
||||
```
|
||||
|
||||
You can also pass additional environment variables to the Python script using the `env` option in the `python.runScript` function. For example:
|
||||
|
||||
```ts
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const myScript = task({
|
||||
id: "my-python-script",
|
||||
run: async () => {
|
||||
const result = await python.runScript("./python/my_script.py", ["hello", "world"], {
|
||||
env: {
|
||||
MY_ENV_VAR: "my value",
|
||||
},
|
||||
});
|
||||
return result.stdout;
|
||||
},
|
||||
});
|
||||
```
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
title: "Sync env vars"
|
||||
sidebarTitle: "syncEnvVars"
|
||||
description: "Use the syncEnvVars build extension to automatically sync environment variables to Trigger.dev"
|
||||
---
|
||||
|
||||
The `syncEnvVars` build extension will sync env vars from another service into Trigger.dev before the deployment starts. This is useful if you are using a secret store service like Infisical or AWS Secrets Manager to store your secrets.
|
||||
|
||||
`syncEnvVars` takes an async callback function, and any env vars returned from the callback will be synced to Trigger.dev.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { syncEnvVars } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
build: {
|
||||
extensions: [
|
||||
syncEnvVars(async (ctx) => {
|
||||
return [
|
||||
{ name: "SECRET_KEY", value: "secret-value" },
|
||||
{ name: "ANOTHER_SECRET", value: "another-secret-value" },
|
||||
];
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
The callback is passed a context object with the following properties:
|
||||
|
||||
- `environment`: The environment name that the task is being deployed to (e.g. `production`, `staging`, etc.)
|
||||
- `projectRef`: The project ref of the Trigger.dev project
|
||||
- `env`: The environment variables that are currently set in the Trigger.dev project
|
||||
|
||||
### Example: Sync env vars from Infisical
|
||||
|
||||
In this example we're using env vars from [Infisical](https://infisical.com).
|
||||
|
||||
```ts trigger.config.ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { syncEnvVars } from "@trigger.dev/build/extensions/core";
|
||||
import { InfisicalSDK } from "@infisical/sdk";
|
||||
|
||||
export default defineConfig({
|
||||
build: {
|
||||
extensions: [
|
||||
syncEnvVars(async (ctx) => {
|
||||
const client = new InfisicalSDK();
|
||||
|
||||
await client.auth().universalAuth.login({
|
||||
clientId: process.env.INFISICAL_CLIENT_ID!,
|
||||
clientSecret: process.env.INFISICAL_CLIENT_SECRET!,
|
||||
});
|
||||
|
||||
const { secrets } = await client.secrets().listSecrets({
|
||||
environment: ctx.environment,
|
||||
projectId: process.env.INFISICAL_PROJECT_ID!,
|
||||
});
|
||||
|
||||
return secrets.map((secret) => ({
|
||||
name: secret.secretKey,
|
||||
value: secret.secretValue,
|
||||
}));
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### syncVercelEnvVars
|
||||
|
||||
The `syncVercelEnvVars` build extension syncs environment variables from your Vercel project to Trigger.dev.
|
||||
|
||||
<Note>
|
||||
You need to set the `VERCEL_ACCESS_TOKEN` and `VERCEL_PROJECT_ID` environment variables, or pass
|
||||
in the token and project ID as arguments to the `syncVercelEnvVars` build extension. If you're
|
||||
working with a team project, you'll also need to set `VERCEL_TEAM_ID`, which can be found in your
|
||||
team settings. You can find / generate the `VERCEL_ACCESS_TOKEN` in your Vercel
|
||||
[dashboard](https://vercel.com/account/settings/tokens). Make sure the scope of the token covers
|
||||
the project with the environment variables you want to sync.
|
||||
</Note>
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
// This will automatically use the VERCEL_ACCESS_TOKEN and VERCEL_PROJECT_ID environment variables
|
||||
extensions: [syncVercelEnvVars()],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
Or you can pass in the token and project ID as arguments:
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
// Your other config settings...
|
||||
build: {
|
||||
extensions: [
|
||||
syncVercelEnvVars({
|
||||
projectId: "your-vercel-project-id",
|
||||
vercelAccessToken: "your-vercel-access-token",
|
||||
vercelTeamId: "your-vercel-team-id", // optional
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
+61
-35
@@ -39,36 +39,7 @@
|
||||
},
|
||||
"triggering",
|
||||
"runs",
|
||||
"apikeys",
|
||||
{
|
||||
"group": "Configuration",
|
||||
"pages": [
|
||||
"config/config-file",
|
||||
"config/extensions/overview"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Development",
|
||||
"pages": [
|
||||
"cli-dev",
|
||||
"run-tests"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Deployment",
|
||||
"pages": [
|
||||
"deployment/overview",
|
||||
"deploy-environment-variables",
|
||||
"github-actions",
|
||||
"deployment/atomic-deployment",
|
||||
{
|
||||
"group": "Deployment integrations",
|
||||
"pages": [
|
||||
"vercel-integration"
|
||||
]
|
||||
}
|
||||
"apikeys"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -91,14 +62,61 @@
|
||||
"versioning",
|
||||
"machines",
|
||||
"idempotency",
|
||||
"replaying",
|
||||
"runs/max-duration",
|
||||
"tags",
|
||||
"runs/metadata",
|
||||
"run-usage",
|
||||
"context",
|
||||
"bulk-actions",
|
||||
"examples"
|
||||
"context"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Configuration",
|
||||
"pages": [
|
||||
"config/config-file",
|
||||
{
|
||||
"group": "Build extensions",
|
||||
"pages": [
|
||||
"config/extensions/overview",
|
||||
{
|
||||
"group": "Built-in extensions",
|
||||
"pages": [
|
||||
"config/extensions/prismaExtension",
|
||||
"config/extensions/pythonExtension",
|
||||
"config/extensions/puppeteer",
|
||||
"config/extensions/ffmpeg",
|
||||
"config/extensions/aptGet",
|
||||
"config/extensions/additionalFiles",
|
||||
"config/extensions/additionalPackages",
|
||||
"config/extensions/syncEnvVars",
|
||||
"config/extensions/esbuildPlugin",
|
||||
"config/extensions/emitDecoratorMetadata",
|
||||
"config/extensions/audioWaveform"
|
||||
]
|
||||
},
|
||||
"config/extensions/custom"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Development",
|
||||
"pages": [
|
||||
"cli-dev"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Deployment",
|
||||
"pages": [
|
||||
"deployment/overview",
|
||||
"deploy-environment-variables",
|
||||
"github-actions",
|
||||
"deployment/atomic-deployment",
|
||||
{
|
||||
"group": "Deployment integrations",
|
||||
"pages": [
|
||||
"vercel-integration"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -146,6 +164,15 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Using the Dashboard",
|
||||
"pages": [
|
||||
"run-tests",
|
||||
"troubleshooting-alerts",
|
||||
"replaying",
|
||||
"bulk-actions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Troubleshooting",
|
||||
"pages": [
|
||||
@@ -153,7 +180,6 @@
|
||||
"troubleshooting-debugging-in-vscode",
|
||||
"upgrading-packages",
|
||||
"upgrading-beta",
|
||||
"troubleshooting-alerts",
|
||||
"troubleshooting-uptime-status",
|
||||
"troubleshooting-github-issues",
|
||||
"request-feature"
|
||||
|
||||
@@ -29,7 +29,7 @@ export default defineConfig({
|
||||
```
|
||||
|
||||
<Note>
|
||||
[Build extensions](/config/config-file#extensions) allow you to hook into the build system and
|
||||
[Build extensions](/config/extensions/overview) allow you to hook into the build system and
|
||||
customize the build process or the resulting bundle and container image (in the case of
|
||||
deploying). You can use pre-built extensions or create your own.
|
||||
</Note>
|
||||
|
||||
@@ -34,7 +34,7 @@ export default defineConfig({
|
||||
```
|
||||
|
||||
<Note>
|
||||
[Build extensions](/config/config-file#extensions) allow you to hook into the build system and
|
||||
[Build extensions](/config/extensions/overview) allow you to hook into the build system and
|
||||
customize the build process or the resulting bundle and container image (in the case of
|
||||
deploying). You can use pre-built extensions or create your own.
|
||||
</Note>
|
||||
|
||||
@@ -12,7 +12,7 @@ This example demonstrates how to use Trigger.dev to turn a PDF into a series of
|
||||
|
||||
## Update your build configuration
|
||||
|
||||
To use this example, add these build settings below to your `trigger.config.ts` file. They ensure that the `mutool` and `curl` packages are installed when you deploy your task. You can learn more about this and see more build settings [here](/config/config-file#aptget).
|
||||
To use this example, add these build settings below to your `trigger.config.ts` file. They ensure that the `mutool` and `curl` packages are installed when you deploy your task. You can learn more about this and see more build settings [here](/config/extensions/aptGet).
|
||||
|
||||
```ts trigger.config.ts
|
||||
export default defineConfig({
|
||||
@@ -100,5 +100,3 @@ To test this task in the dashboard, you can use the following payload:
|
||||
```
|
||||
|
||||
<LocalDevelopment packages={"mupdf-tools from MuPDF"} />
|
||||
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ There are 3 example tasks to follow on this page:
|
||||
|
||||
<ScrapingWarning />
|
||||
|
||||
## Build configurations
|
||||
## Build configuration
|
||||
|
||||
To use all examples on this page, you'll first need to add these build settings to your `trigger.config.ts` file:
|
||||
|
||||
@@ -40,7 +40,7 @@ export default defineConfig({
|
||||
});
|
||||
```
|
||||
|
||||
Learn more about [build configurations](/config/config-file#build-configuration) including setting default retry settings, customizing the build environment, and more.
|
||||
Learn more about the [trigger.config.ts](/config/config-file) file including setting default retry settings, customizing the build environment, and more.
|
||||
|
||||
## Set an environment variable
|
||||
|
||||
|
||||
@@ -12,9 +12,9 @@ import ScrapingWarning from "/snippets/web-scraping-warning.mdx";
|
||||
height="315"
|
||||
src="https://www.youtube.com/embed/6azvzrZITKY?si=muKtsBiS9TJGGKWg"
|
||||
title="YouTube video player"
|
||||
allow="accelerometer; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
|
||||
referrerPolicy="strict-origin-when-cross-origin"
|
||||
allowFullScreen
|
||||
allow="accelerometer; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
|
||||
referrerPolicy="strict-origin-when-cross-origin"
|
||||
allowFullScreen
|
||||
/>
|
||||
|
||||
## Overview
|
||||
@@ -63,7 +63,7 @@ export default defineConfig({
|
||||
});
|
||||
```
|
||||
|
||||
Learn more about [build configurations](/config/config-file#build-configuration) including setting default retry settings, customizing the build environment, and more.
|
||||
Learn more about the [trigger.config.ts](/config/config-file) file including setting default retry settings, customizing the build environment, and more.
|
||||
|
||||
### Environment variables
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ export default defineConfig({
|
||||
```
|
||||
|
||||
<Note>
|
||||
[Build extensions](/config/config-file#extensions) allow you to hook into the build system and
|
||||
[Build extensions](/config/extensions/overview) allow you to hook into the build system and
|
||||
customize the build process or the resulting bundle and container image (in the case of
|
||||
deploying). You can use pre-built extensions or create your own.
|
||||
</Note>
|
||||
|
||||
@@ -34,7 +34,7 @@ export default defineConfig({
|
||||
```
|
||||
|
||||
<Note>
|
||||
[Build extensions](/config/config-file#extensions) allow you to hook into the build system and
|
||||
[Build extensions](/config/extensions/overview) allow you to hook into the build system and
|
||||
customize the build process or the resulting bundle and container image (in the case of
|
||||
deploying). You can use pre-built extensions or create your own.
|
||||
</Note>
|
||||
|
||||
@@ -84,7 +84,7 @@ Next, configure the Prisma [build extension](https://trigger.dev/docs/config/ext
|
||||
|
||||
This will ensure that the Prisma client is available when the task runs.
|
||||
|
||||
For a full list of options available in the Prisma build extension, see the [Prisma build extension documentation](https://trigger.dev/docs/config/config-file#prisma).
|
||||
For a full list of options available in the Prisma build extension, see the [Prisma build extension documentation](https://trigger.dev/docs/config/extensions/prismaExtension).
|
||||
|
||||
```js /trigger.config.js
|
||||
export default defineConfig({
|
||||
@@ -103,7 +103,7 @@ export default defineConfig({
|
||||
```
|
||||
|
||||
<Note>
|
||||
[Build extensions](/config/config-file#extensions) allow you to hook into the build system and
|
||||
[Build extensions](/config/extensions/overview) allow you to hook into the build system and
|
||||
customize the build process or the resulting bundle and container image (in the case of
|
||||
deploying). You can use pre-built extensions or create your own.
|
||||
</Note>
|
||||
|
||||
@@ -260,7 +260,7 @@ export default defineConfig({
|
||||
```
|
||||
|
||||
<Note>
|
||||
[Build extensions](/config/config-file#extensions) allow you to hook into the build system and
|
||||
[Build extensions](/config/extensions/overview) allow you to hook into the build system and
|
||||
customize the build process or the resulting bundle and container image (in the case of
|
||||
deploying). You can use pre-built extensions or create your own.
|
||||
</Note>
|
||||
|
||||
@@ -94,13 +94,14 @@ Your code is deployed separately from the rest of your app(s) so you need to mak
|
||||
|
||||
### `Error: @prisma/client did not initialize yet.`
|
||||
|
||||
Prisma uses code generation to create the client from your schema file. This means you need to add a bit of config so we can generate this file before your tasks run: [Read the guide](/config/config-file#prisma).
|
||||
Prisma uses code generation to create the client from your schema file. This means you need to add a bit of config so we can generate this file before your tasks run: [Read the guide](/config/extensions/prismaExtension).
|
||||
|
||||
### `Parallel waits are not supported`
|
||||
|
||||
In the current version, you can't perform more that one "wait" in parallel.
|
||||
|
||||
Waits include:
|
||||
|
||||
- `wait.for()`
|
||||
- `wait.until()`
|
||||
- `task.triggerAndWait()`
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": "./src/index.ts",
|
||||
"./internal": "./src/internal.ts",
|
||||
"./extensions": "./src/extensions/index.ts",
|
||||
"./extensions/core": "./src/extensions/core.ts",
|
||||
"./extensions/prisma": "./src/extensions/prisma.ts",
|
||||
@@ -36,6 +37,9 @@
|
||||
},
|
||||
"typesVersions": {
|
||||
"*": {
|
||||
"internal": [
|
||||
"dist/commonjs/internal.d.ts"
|
||||
],
|
||||
"extensions": [
|
||||
"dist/commonjs/extensions/index.d.ts"
|
||||
],
|
||||
@@ -95,6 +99,17 @@
|
||||
"default": "./dist/commonjs/index.js"
|
||||
}
|
||||
},
|
||||
"./internal": {
|
||||
"import": {
|
||||
"@triggerdotdev/source": "./src/internal.ts",
|
||||
"types": "./dist/esm/internal.d.ts",
|
||||
"default": "./dist/esm/internal.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/commonjs/internal.d.ts",
|
||||
"default": "./dist/commonjs/internal.js"
|
||||
}
|
||||
},
|
||||
"./extensions": {
|
||||
"import": {
|
||||
"@triggerdotdev/source": "./src/extensions/index.ts",
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import { relative, join, posix, dirname } from "node:path";
|
||||
import { glob } from "tinyglobby";
|
||||
import { copyFile, mkdir } from "node:fs/promises";
|
||||
import { BuildExtension } from "@trigger.dev/core/v3/build";
|
||||
import { addAdditionalFilesToBuild } from "../../internal/additionalFiles.js";
|
||||
|
||||
export type AdditionalFilesOptions = {
|
||||
files: string[];
|
||||
@@ -11,86 +9,7 @@ export function additionalFiles(options: AdditionalFilesOptions): BuildExtension
|
||||
return {
|
||||
name: "additionalFiles",
|
||||
async onBuildComplete(context, manifest) {
|
||||
// Copy any static assets to the destination
|
||||
const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, {
|
||||
cwd: context.workingDir,
|
||||
});
|
||||
|
||||
for (const { assets, matcher } of staticAssets) {
|
||||
if (assets.length === 0) {
|
||||
console.warn("No files found for matcher", matcher);
|
||||
}
|
||||
}
|
||||
|
||||
await copyStaticAssets(staticAssets);
|
||||
await addAdditionalFilesToBuild("additionalFiles", options, context, manifest);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type MatchedStaticAssets = { source: string; destination: string }[];
|
||||
|
||||
type FoundStaticAssetFiles = Array<{
|
||||
matcher: string;
|
||||
assets: MatchedStaticAssets;
|
||||
}>;
|
||||
|
||||
async function findStaticAssetFiles(
|
||||
matchers: string[],
|
||||
destinationPath: string,
|
||||
options?: { cwd?: string; ignore?: string[] }
|
||||
): Promise<FoundStaticAssetFiles> {
|
||||
const result: FoundStaticAssetFiles = [];
|
||||
|
||||
for (const matcher of matchers) {
|
||||
const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options);
|
||||
|
||||
result.push({ matcher, assets });
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function findStaticAssetsForMatcher(
|
||||
matcher: string,
|
||||
destinationPath: string,
|
||||
options?: { cwd?: string; ignore?: string[] }
|
||||
): Promise<MatchedStaticAssets> {
|
||||
const result: MatchedStaticAssets = [];
|
||||
|
||||
const files = await glob({
|
||||
patterns: [matcher],
|
||||
cwd: options?.cwd,
|
||||
ignore: options?.ignore ?? [],
|
||||
onlyFiles: true,
|
||||
absolute: true,
|
||||
});
|
||||
|
||||
let matches = 0;
|
||||
|
||||
for (const file of files) {
|
||||
matches++;
|
||||
|
||||
const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file)
|
||||
.split(posix.sep)
|
||||
.filter((p) => p !== "..")
|
||||
.join(posix.sep);
|
||||
|
||||
const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir);
|
||||
|
||||
result.push({
|
||||
source: file,
|
||||
destination: relativeDestinationPath,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function copyStaticAssets(staticAssetFiles: FoundStaticAssetFiles): Promise<void> {
|
||||
for (const { assets } of staticAssetFiles) {
|
||||
for (const { source, destination } of assets) {
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
await copyFile(source, destination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./internal/additionalFiles.js";
|
||||
@@ -0,0 +1,106 @@
|
||||
import { BuildManifest } from "@trigger.dev/core/v3";
|
||||
import { BuildContext } from "@trigger.dev/core/v3/build";
|
||||
import { copyFile, mkdir } from "node:fs/promises";
|
||||
import { dirname, join, posix, relative } from "node:path";
|
||||
import { glob } from "tinyglobby";
|
||||
|
||||
export type AdditionalFilesOptions = {
|
||||
files: string[];
|
||||
};
|
||||
|
||||
export async function addAdditionalFilesToBuild(
|
||||
source: string,
|
||||
options: AdditionalFilesOptions,
|
||||
context: BuildContext,
|
||||
manifest: BuildManifest
|
||||
) {
|
||||
// Copy any static assets to the destination
|
||||
const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, {
|
||||
cwd: context.workingDir,
|
||||
});
|
||||
|
||||
for (const { assets, matcher } of staticAssets) {
|
||||
if (assets.length === 0) {
|
||||
context.logger.warn(`[${source}] No files found for matcher`, matcher);
|
||||
} else {
|
||||
context.logger.debug(`[${source}] Found ${assets.length} files for matcher`, matcher);
|
||||
}
|
||||
}
|
||||
|
||||
await copyStaticAssets(staticAssets, source, context);
|
||||
}
|
||||
|
||||
type MatchedStaticAssets = { source: string; destination: string }[];
|
||||
|
||||
type FoundStaticAssetFiles = Array<{
|
||||
matcher: string;
|
||||
assets: MatchedStaticAssets;
|
||||
}>;
|
||||
|
||||
async function findStaticAssetFiles(
|
||||
matchers: string[],
|
||||
destinationPath: string,
|
||||
options?: { cwd?: string; ignore?: string[] }
|
||||
): Promise<FoundStaticAssetFiles> {
|
||||
const result: FoundStaticAssetFiles = [];
|
||||
|
||||
for (const matcher of matchers) {
|
||||
const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options);
|
||||
|
||||
result.push({ matcher, assets });
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function findStaticAssetsForMatcher(
|
||||
matcher: string,
|
||||
destinationPath: string,
|
||||
options?: { cwd?: string; ignore?: string[] }
|
||||
): Promise<MatchedStaticAssets> {
|
||||
const result: MatchedStaticAssets = [];
|
||||
|
||||
const files = await glob({
|
||||
patterns: [matcher],
|
||||
cwd: options?.cwd,
|
||||
ignore: options?.ignore ?? [],
|
||||
onlyFiles: true,
|
||||
absolute: true,
|
||||
});
|
||||
|
||||
let matches = 0;
|
||||
|
||||
for (const file of files) {
|
||||
matches++;
|
||||
|
||||
const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file)
|
||||
.split(posix.sep)
|
||||
.filter((p) => p !== "..")
|
||||
.join(posix.sep);
|
||||
|
||||
const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir);
|
||||
|
||||
result.push({
|
||||
source: file,
|
||||
destination: relativeDestinationPath,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function copyStaticAssets(
|
||||
staticAssetFiles: FoundStaticAssetFiles,
|
||||
sourceName: string,
|
||||
context: BuildContext
|
||||
): Promise<void> {
|
||||
for (const { assets } of staticAssetFiles) {
|
||||
for (const { source, destination } of assets) {
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
|
||||
context.logger.debug(`[${sourceName}] Copying ${source} to ${destination}`);
|
||||
|
||||
await copyFile(source, destination);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,14 +61,13 @@ import {
|
||||
SubscribeToRunsQueryParams,
|
||||
UpdateEnvironmentVariableParams,
|
||||
} from "./types.js";
|
||||
import type { AsyncIterableStream } from "./stream.js";
|
||||
import { AsyncIterableStream } from "../streams/asyncIterableStream.js";
|
||||
|
||||
export type {
|
||||
CreateEnvironmentVariableParams,
|
||||
ImportEnvironmentVariablesParams,
|
||||
SubscribeToRunsQueryParams,
|
||||
UpdateEnvironmentVariableParams,
|
||||
AsyncIterableStream,
|
||||
};
|
||||
|
||||
export type ClientTriggerOptions = {
|
||||
|
||||
@@ -16,12 +16,11 @@ import {
|
||||
} from "../utils/ioSerialization.js";
|
||||
import { ApiError } from "./errors.js";
|
||||
import { ApiClient } from "./index.js";
|
||||
import { LineTransformStream, zodShapeStream } from "./stream.js";
|
||||
import {
|
||||
AsyncIterableStream,
|
||||
createAsyncIterableReadable,
|
||||
LineTransformStream,
|
||||
zodShapeStream,
|
||||
} from "./stream.js";
|
||||
} from "../streams/asyncIterableStream.js";
|
||||
|
||||
export type RunShape<TRunTypes extends AnyRunTypes> = TRunTypes extends AnyRunTypes
|
||||
? {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
type Row,
|
||||
type ShapeStreamInterface,
|
||||
} from "@electric-sql/client";
|
||||
import { AsyncIterableStream, createAsyncIterableStream } from "../streams/asyncIterableStream.js";
|
||||
|
||||
export type ZodShapeStreamOptions = {
|
||||
headers?: Record<string, string>;
|
||||
@@ -82,57 +83,6 @@ export function zodShapeStream<TShapeSchema extends z.ZodTypeAny>(
|
||||
};
|
||||
}
|
||||
|
||||
export type AsyncIterableStream<T> = AsyncIterable<T> & ReadableStream<T>;
|
||||
|
||||
export function createAsyncIterableStream<S, T>(
|
||||
source: ReadableStream<S>,
|
||||
transformer: Transformer<S, T>
|
||||
): AsyncIterableStream<T> {
|
||||
const transformedStream: any = source.pipeThrough(new TransformStream(transformer));
|
||||
|
||||
transformedStream[Symbol.asyncIterator] = () => {
|
||||
const reader = transformedStream.getReader();
|
||||
return {
|
||||
async next(): Promise<IteratorResult<string>> {
|
||||
const { done, value } = await reader.read();
|
||||
return done ? { done: true, value: undefined } : { done: false, value };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
return transformedStream;
|
||||
}
|
||||
|
||||
export function createAsyncIterableReadable<S, T>(
|
||||
source: ReadableStream<S>,
|
||||
transformer: Transformer<S, T>,
|
||||
signal: AbortSignal
|
||||
): AsyncIterableStream<T> {
|
||||
return new ReadableStream<T>({
|
||||
async start(controller) {
|
||||
const transformedStream = source.pipeThrough(new TransformStream(transformer));
|
||||
const reader = transformedStream.getReader();
|
||||
|
||||
signal.addEventListener("abort", () => {
|
||||
queueMicrotask(() => {
|
||||
reader.cancel();
|
||||
controller.close();
|
||||
});
|
||||
});
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
controller.close();
|
||||
break;
|
||||
}
|
||||
|
||||
controller.enqueue(value);
|
||||
}
|
||||
},
|
||||
}) as AsyncIterableStream<T>;
|
||||
}
|
||||
|
||||
class ReadableShapeStream<T extends Row<unknown> = Row> {
|
||||
readonly #stream: ShapeStreamInterface<T>;
|
||||
readonly #currentState: Map<string, T> = new Map();
|
||||
|
||||
@@ -22,6 +22,7 @@ export * from "./types/index.js";
|
||||
export { links } from "./links.js";
|
||||
export * from "./jwt.js";
|
||||
export * from "./idempotencyKeys.js";
|
||||
export * from "./streams/asyncIterableStream.js";
|
||||
export * from "./utils/getEnv.js";
|
||||
export {
|
||||
formatDuration,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { NoopTaskLogger, TaskLogger } from "./taskLogger.js";
|
||||
import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js";
|
||||
import { Span } from "@opentelemetry/api";
|
||||
import { Span, SpanOptions } from "@opentelemetry/api";
|
||||
|
||||
const API_NAME = "logger";
|
||||
|
||||
@@ -47,8 +47,12 @@ export class LoggerAPI implements TaskLogger {
|
||||
this.#getTaskLogger().error(message, metadata);
|
||||
}
|
||||
|
||||
public trace<T>(name: string, fn: (span: Span) => Promise<T>): Promise<T> {
|
||||
return this.#getTaskLogger().trace(name, fn);
|
||||
public trace<T>(name: string, fn: (span: Span) => Promise<T>, options?: SpanOptions): Promise<T> {
|
||||
return this.#getTaskLogger().trace(name, fn, options);
|
||||
}
|
||||
|
||||
public startSpan(name: string, options?: SpanOptions): Span {
|
||||
return this.#getTaskLogger().startSpan(name, options);
|
||||
}
|
||||
|
||||
#getTaskLogger(): TaskLogger {
|
||||
|
||||
@@ -24,6 +24,7 @@ export interface TaskLogger {
|
||||
warn(message: string, properties?: Record<string, unknown>): void;
|
||||
error(message: string, properties?: Record<string, unknown>): void;
|
||||
trace<T>(name: string, fn: (span: Span) => Promise<T>, options?: SpanOptions): Promise<T>;
|
||||
startSpan(name: string, options?: SpanOptions): Span;
|
||||
}
|
||||
|
||||
export class OtelTaskLogger implements TaskLogger {
|
||||
@@ -90,6 +91,10 @@ export class OtelTaskLogger implements TaskLogger {
|
||||
return this._config.tracer.startActiveSpan(name, fn, options);
|
||||
}
|
||||
|
||||
startSpan(name: string, options?: SpanOptions): Span {
|
||||
return this._config.tracer.startSpan(name, options);
|
||||
}
|
||||
|
||||
#getTimestampInHrTime(): ClockTime {
|
||||
return clock.preciseNow();
|
||||
}
|
||||
@@ -104,6 +109,9 @@ export class NoopTaskLogger implements TaskLogger {
|
||||
trace<T>(name: string, fn: (span: Span) => Promise<T>): Promise<T> {
|
||||
return fn({} as Span);
|
||||
}
|
||||
startSpan(): Span {
|
||||
return {} as Span;
|
||||
}
|
||||
}
|
||||
|
||||
function safeJsonProcess(value?: Record<string, unknown>): Record<string, unknown> | undefined {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { DeserializedJson } from "../../schemas/json.js";
|
||||
import { AsyncIterableStream } from "../apiClient/stream.js";
|
||||
import { AsyncIterableStream } from "../streams/asyncIterableStream.js";
|
||||
import { getGlobal, registerGlobal } from "../utils/globals.js";
|
||||
import { ApiRequestOptions } from "../zodfetch.js";
|
||||
import { NoopRunMetadataManager } from "./noopManager.js";
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { dequal } from "dequal/lite";
|
||||
import { DeserializedJson } from "../../schemas/json.js";
|
||||
import { ApiClient } from "../apiClient/index.js";
|
||||
import { AsyncIterableStream } from "../apiClient/stream.js";
|
||||
import { FlushedRunMetadata, RunMetadataChangeOperation } from "../schemas/common.js";
|
||||
import { ApiRequestOptions } from "../zodfetch.js";
|
||||
import { MetadataStream } from "./metadataStream.js";
|
||||
import { applyMetadataOperations } from "./operations.js";
|
||||
import { RunMetadataManager, RunMetadataUpdater } from "./types.js";
|
||||
import { AsyncIterableStream } from "../streams/asyncIterableStream.js";
|
||||
|
||||
const MAXIMUM_ACTIVE_STREAMS = 5;
|
||||
const MAXIMUM_TOTAL_STREAMS = 10;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { DeserializedJson } from "../../schemas/json.js";
|
||||
import { AsyncIterableStream } from "../apiClient/stream.js";
|
||||
import { AsyncIterableStream } from "../streams/asyncIterableStream.js";
|
||||
import { ApiRequestOptions } from "../zodfetch.js";
|
||||
import type { RunMetadataManager, RunMetadataUpdater } from "./types.js";
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { DeserializedJson } from "../../schemas/json.js";
|
||||
import { AsyncIterableStream } from "../apiClient/stream.js";
|
||||
import { AsyncIterableStream } from "../streams/asyncIterableStream.js";
|
||||
import { ApiRequestOptions } from "../zodfetch.js";
|
||||
|
||||
export interface RunMetadataUpdater {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
export type AsyncIterableStream<T> = AsyncIterable<T> & ReadableStream<T>;
|
||||
|
||||
export function createAsyncIterableStream<S, T>(
|
||||
source: ReadableStream<S>,
|
||||
transformer: Transformer<S, T>
|
||||
): AsyncIterableStream<T> {
|
||||
const transformedStream: any = source.pipeThrough(new TransformStream(transformer));
|
||||
|
||||
transformedStream[Symbol.asyncIterator] = () => {
|
||||
const reader = transformedStream.getReader();
|
||||
return {
|
||||
async next(): Promise<IteratorResult<string>> {
|
||||
const { done, value } = await reader.read();
|
||||
return done ? { done: true, value: undefined } : { done: false, value };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
return transformedStream;
|
||||
}
|
||||
|
||||
export function createAsyncIterableReadable<S, T>(
|
||||
source: ReadableStream<S>,
|
||||
transformer: Transformer<S, T>,
|
||||
signal: AbortSignal
|
||||
): AsyncIterableStream<T> {
|
||||
return new ReadableStream<T>({
|
||||
async start(controller) {
|
||||
const transformedStream = source.pipeThrough(new TransformStream(transformer));
|
||||
const reader = transformedStream.getReader();
|
||||
|
||||
signal.addEventListener("abort", () => {
|
||||
queueMicrotask(() => {
|
||||
reader.cancel();
|
||||
controller.close();
|
||||
});
|
||||
});
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
controller.close();
|
||||
break;
|
||||
}
|
||||
|
||||
controller.enqueue(value);
|
||||
}
|
||||
},
|
||||
}) as AsyncIterableStream<T>;
|
||||
}
|
||||
|
||||
export function createAsyncIterableStreamFromAsyncIterable<T>(
|
||||
asyncIterable: AsyncIterable<T>,
|
||||
transformer: Transformer<T, T>,
|
||||
signal?: AbortSignal
|
||||
): AsyncIterableStream<T> {
|
||||
const stream = new ReadableStream<T>({
|
||||
async start(controller) {
|
||||
try {
|
||||
if (signal) {
|
||||
signal.addEventListener("abort", () => {
|
||||
controller.close();
|
||||
});
|
||||
}
|
||||
|
||||
const iterator = asyncIterable[Symbol.asyncIterator]();
|
||||
|
||||
while (true) {
|
||||
if (signal?.aborted) {
|
||||
break;
|
||||
}
|
||||
|
||||
const { done, value } = await iterator.next();
|
||||
|
||||
if (done) {
|
||||
controller.close();
|
||||
break;
|
||||
}
|
||||
|
||||
controller.enqueue(value);
|
||||
}
|
||||
} catch (error) {
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
// If the stream is a tinyexec process with a kill method, kill it
|
||||
if ("kill" in asyncIterable) {
|
||||
(asyncIterable as any).kill();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const transformedStream = stream.pipeThrough(new TransformStream(transformer));
|
||||
|
||||
return transformedStream as AsyncIterableStream<T>;
|
||||
}
|
||||
@@ -14,7 +14,7 @@ This extension introduces the <code>pythonExtension</code> build extension, whic
|
||||
- <code>run</code>: Executes Python commands with proper environment setup.
|
||||
- <code>runInline</code>: Executes inline Python code directly from Node.
|
||||
- <code>runScript</code>: Executes standalone <code>.py</code> script files.
|
||||
- **Custom Python Path:** In development, you can configure <code>pythonBinaryPath</code> to point to a custom Python installation.
|
||||
- **Custom Python Path:** In development, you can configure <code>devPythonBinaryPath</code> to point to a custom Python installation.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -22,7 +22,7 @@ This extension introduces the <code>pythonExtension</code> build extension, whic
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import pythonExtension from "@trigger.dev/python/extension";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
@@ -30,8 +30,8 @@ export default defineConfig({
|
||||
extensions: [
|
||||
pythonExtension({
|
||||
requirementsFile: "./requirements.txt", // Optional: Path to your requirements file
|
||||
pythonBinaryPath: path.join(rootDir, `.venv/bin/python`), // Optional: Custom Python binary path
|
||||
scripts: ["my_script.py"], // List of Python scripts to include
|
||||
devPythonBinaryPath: ".venv/bin/python", // Optional: Custom Python binary path
|
||||
scripts: ["src/python/**/*.py"], // Glob pattern for Python scripts
|
||||
}),
|
||||
],
|
||||
},
|
||||
@@ -40,13 +40,34 @@ export default defineConfig({
|
||||
|
||||
2. (Optional) Create a <code>requirements.txt</code> file in your project root with the necessary Python dependencies.
|
||||
|
||||
```plaintext title="requirements.txt"
|
||||
pandas==1.3.3
|
||||
numpy==1.21.2
|
||||
```
|
||||
|
||||
```typescript title="trigger.config.ts"
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
project: "<project ref>",
|
||||
build: {
|
||||
extensions: [
|
||||
pythonExtension({
|
||||
requirementsFile: "./requirements.txt",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
3. Execute Python scripts within your tasks using one of the provided functions:
|
||||
|
||||
### Running a Python Script
|
||||
|
||||
```typescript
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import python from "@trigger.dev/python";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const myScript = task({
|
||||
id: "my-python-script",
|
||||
@@ -55,13 +76,29 @@ export const myScript = task({
|
||||
return result.stdout;
|
||||
},
|
||||
});
|
||||
|
||||
export const myStreamingScript = task({
|
||||
id: "my-streaming-python-script",
|
||||
run: async () => {
|
||||
// You can also stream the output of the script
|
||||
const result = python.stream.runScript("my_script.py", ["hello", "world"]);
|
||||
|
||||
// result is an async iterable/readable stream
|
||||
for await (const chunk of streamingResult) {
|
||||
logger.debug("convert-url-to-markdown", {
|
||||
url: payload.url,
|
||||
chunk,
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Running Inline Python Code
|
||||
|
||||
```typescript
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import python from "@trigger.dev/python";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const myTask = task({
|
||||
id: "to_datetime-task",
|
||||
@@ -69,7 +106,7 @@ export const myTask = task({
|
||||
const result = await python.runInline(`
|
||||
import pandas as pd
|
||||
|
||||
pandas.to_datetime("${+new Date() / 1000}")
|
||||
pd.to_datetime("${+new Date() / 1000}")
|
||||
`);
|
||||
return result.stdout;
|
||||
},
|
||||
@@ -80,7 +117,7 @@ pandas.to_datetime("${+new Date() / 1000}")
|
||||
|
||||
```typescript
|
||||
import { task } from "@trigger.dev/sdk/v3";
|
||||
import python from "@trigger.dev/python";
|
||||
import { python } from "@trigger.dev/python";
|
||||
|
||||
export const pythonVersionTask = task({
|
||||
id: "python-version-task",
|
||||
@@ -94,7 +131,6 @@ export const pythonVersionTask = task({
|
||||
## Limitations
|
||||
|
||||
- This is a **partial implementation** and does not provide full Python support as an execution runtime for tasks.
|
||||
- Only basic Python script execution is supported; scripts are not automatically copied to staging/production containers.
|
||||
- Manual intervention may be required for installing and configuring binary dependencies in development environments.
|
||||
|
||||
## Additional Information
|
||||
|
||||
@@ -45,9 +45,7 @@
|
||||
"check-exports": "attw --pack ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@trigger.dev/build": "workspace:3.3.16",
|
||||
"@trigger.dev/core": "workspace:3.3.16",
|
||||
"@trigger.dev/sdk": "workspace:3.3.16",
|
||||
"tinyexec": "^0.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -57,7 +55,13 @@
|
||||
"typescript": "^5.5.4",
|
||||
"tsx": "4.17.0",
|
||||
"esbuild": "^0.23.0",
|
||||
"@arethetypeswrong/cli": "^0.15.4"
|
||||
"@arethetypeswrong/cli": "^0.15.4",
|
||||
"@trigger.dev/build": "workspace:3.3.16",
|
||||
"@trigger.dev/sdk": "workspace:3.3.16"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@trigger.dev/sdk": "workspace:^3.3.16",
|
||||
"@trigger.dev/build": "workspace:^3.3.16"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.20.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import fs from "node:fs";
|
||||
import assert from "node:assert";
|
||||
import { additionalFiles } from "@trigger.dev/build/extensions/core";
|
||||
import { addAdditionalFilesToBuild } from "@trigger.dev/build/internal";
|
||||
import { BuildManifest } from "@trigger.dev/core/v3";
|
||||
import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build";
|
||||
|
||||
@@ -16,7 +16,7 @@ export type PythonOptions = {
|
||||
*
|
||||
* Example: `/usr/bin/python3` or `C:\\Python39\\python.exe`
|
||||
*/
|
||||
pythonBinaryPath?: string;
|
||||
devPythonBinaryPath?: string;
|
||||
/**
|
||||
* An array of glob patterns that specify which Python scripts are allowed to be executed.
|
||||
*
|
||||
@@ -57,13 +57,18 @@ class PythonExtension implements BuildExtension {
|
||||
}
|
||||
|
||||
async onBuildComplete(context: BuildContext, manifest: BuildManifest) {
|
||||
await additionalFiles({
|
||||
files: this.options.scripts ?? [],
|
||||
}).onBuildComplete!(context, manifest);
|
||||
await addAdditionalFilesToBuild(
|
||||
"pythonExtension",
|
||||
{
|
||||
files: this.options.scripts ?? [],
|
||||
},
|
||||
context,
|
||||
manifest
|
||||
);
|
||||
|
||||
if (context.target === "dev") {
|
||||
if (this.options.pythonBinaryPath) {
|
||||
process.env.PYTHON_BIN_PATH = this.options.pythonBinaryPath;
|
||||
if (this.options.devPythonBinaryPath) {
|
||||
process.env.PYTHON_BIN_PATH = this.options.devPythonBinaryPath;
|
||||
}
|
||||
|
||||
return;
|
||||
@@ -93,27 +98,59 @@ class PythonExtension implements BuildExtension {
|
||||
},
|
||||
});
|
||||
|
||||
context.addLayer({
|
||||
id: "python-dependencies",
|
||||
build: {
|
||||
env: {
|
||||
REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "",
|
||||
if (this.options.requirementsFile) {
|
||||
if (this.options.requirements) {
|
||||
context.logger.warn(
|
||||
`[pythonExtension] Both options.requirements and options.requirementsFile are specified. requirements will be ignored.`
|
||||
);
|
||||
}
|
||||
|
||||
// Copy requirements file to the container
|
||||
await addAdditionalFilesToBuild(
|
||||
"pythonExtension",
|
||||
{
|
||||
files: [this.options.requirementsFile],
|
||||
},
|
||||
},
|
||||
image: {
|
||||
instructions: splitAndCleanComments(`
|
||||
context,
|
||||
manifest
|
||||
);
|
||||
|
||||
// Add a layer to the build that installs the requirements
|
||||
context.addLayer({
|
||||
id: "python-dependencies",
|
||||
image: {
|
||||
instructions: splitAndCleanComments(`
|
||||
# Copy the requirements file
|
||||
COPY ${this.options.requirementsFile} .
|
||||
# Install dependencies
|
||||
RUN pip install --no-cache-dir -r ${this.options.requirementsFile}
|
||||
`),
|
||||
},
|
||||
deploy: {
|
||||
override: true,
|
||||
},
|
||||
});
|
||||
} else if (this.options.requirements) {
|
||||
context.addLayer({
|
||||
id: "python-dependencies",
|
||||
build: {
|
||||
env: {
|
||||
REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "",
|
||||
},
|
||||
},
|
||||
image: {
|
||||
instructions: splitAndCleanComments(`
|
||||
ARG REQUIREMENTS_CONTENT
|
||||
RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt
|
||||
|
||||
# Install dependencies
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
`),
|
||||
},
|
||||
deploy: {
|
||||
override: true,
|
||||
},
|
||||
});
|
||||
},
|
||||
deploy: {
|
||||
override: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default pythonExtension;
|
||||
|
||||
+267
-53
@@ -1,63 +1,277 @@
|
||||
import fs from "node:fs";
|
||||
import assert from "node:assert";
|
||||
import {
|
||||
AsyncIterableStream,
|
||||
createAsyncIterableStreamFromAsyncIterable,
|
||||
SemanticInternalAttributes,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { logger } from "@trigger.dev/sdk/v3";
|
||||
import { x, Options as XOptions, Result } from "tinyexec";
|
||||
import assert from "node:assert";
|
||||
import fs from "node:fs";
|
||||
import { Result, x, Options as XOptions } from "tinyexec";
|
||||
import { createTempFileSync, withTempFile } from "./utils/tempFiles.js";
|
||||
|
||||
export const run = async (
|
||||
scriptArgs: string[] = [],
|
||||
options: Partial<XOptions> = {}
|
||||
): Promise<Result> => {
|
||||
const pythonBin = process.env.PYTHON_BIN_PATH || "python";
|
||||
|
||||
return await logger.trace("Python call", async (span) => {
|
||||
span.addEvent("Properties", {
|
||||
command: `${pythonBin} ${scriptArgs.join(" ")}`,
|
||||
});
|
||||
|
||||
const result = await x(pythonBin, scriptArgs, {
|
||||
...options,
|
||||
throwOnError: false, // Ensure errors are handled manually
|
||||
});
|
||||
|
||||
span.addEvent("Output", { ...result });
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
logger.error(result.stderr, { ...result });
|
||||
throw new Error(`Python command exited with non-zero code ${result.exitCode}`);
|
||||
}
|
||||
|
||||
return result;
|
||||
});
|
||||
export type PythonExecOptions = Partial<XOptions> & {
|
||||
env?: { [key: string]: string | undefined };
|
||||
};
|
||||
|
||||
export const runScript = (
|
||||
scriptPath: string,
|
||||
scriptArgs: string[] = [],
|
||||
options: Partial<XOptions> = {}
|
||||
) => {
|
||||
assert(scriptPath, "Script path is required");
|
||||
assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`);
|
||||
export const python = {
|
||||
async run(scriptArgs: string[] = [], options: PythonExecOptions = {}): Promise<Result> {
|
||||
const pythonBin = process.env.PYTHON_BIN_PATH || "python";
|
||||
|
||||
return run([scriptPath, ...scriptArgs], options);
|
||||
};
|
||||
return await logger.trace(
|
||||
"python.run()",
|
||||
async (span) => {
|
||||
const result = await x(pythonBin, scriptArgs, {
|
||||
...options,
|
||||
nodeOptions: {
|
||||
...(options.nodeOptions || {}),
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
},
|
||||
throwOnError: false, // Ensure errors are handled manually
|
||||
});
|
||||
|
||||
export const runInline = async (scriptContent: string, options: Partial<XOptions> = {}) => {
|
||||
assert(scriptContent, "Script content is required");
|
||||
if (result.exitCode) {
|
||||
span.setAttribute("exitCode", result.exitCode);
|
||||
}
|
||||
|
||||
const tmpFile = `/tmp/script_${Date.now()}.py`;
|
||||
await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 });
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`${scriptArgs.join(" ")} exited with a non-zero code ${result.exitCode}:\n${
|
||||
result.stderr
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await runScript(tmpFile, [], options);
|
||||
} finally {
|
||||
try {
|
||||
await fs.promises.unlink(tmpFile);
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to clean up temporary file ${tmpFile}:`, {
|
||||
error: (error as Error).stack || (error as Error).message,
|
||||
return result;
|
||||
},
|
||||
{
|
||||
attributes: {
|
||||
pythonBin,
|
||||
args: scriptArgs.join(" "),
|
||||
[SemanticInternalAttributes.STYLE_ICON]: "brand-python",
|
||||
},
|
||||
}
|
||||
);
|
||||
},
|
||||
|
||||
async runScript(
|
||||
scriptPath: string,
|
||||
scriptArgs: string[] = [],
|
||||
options: PythonExecOptions = {}
|
||||
): Promise<Result> {
|
||||
assert(scriptPath, "Script path is required");
|
||||
assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`);
|
||||
|
||||
return await logger.trace(
|
||||
"python.runScript()",
|
||||
async (span) => {
|
||||
span.setAttribute("scriptPath", scriptPath);
|
||||
|
||||
const result = await x(
|
||||
process.env.PYTHON_BIN_PATH || "python",
|
||||
[scriptPath, ...scriptArgs],
|
||||
{
|
||||
...options,
|
||||
nodeOptions: {
|
||||
...(options.nodeOptions || {}),
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
},
|
||||
throwOnError: false,
|
||||
}
|
||||
);
|
||||
|
||||
if (result.exitCode) {
|
||||
span.setAttribute("exitCode", result.exitCode);
|
||||
}
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`${scriptPath} ${scriptArgs.join(" ")} exited with a non-zero code ${
|
||||
result.exitCode
|
||||
}:\n${result.stderr}`
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
attributes: {
|
||||
pythonBin: process.env.PYTHON_BIN_PATH || "python",
|
||||
scriptPath,
|
||||
args: scriptArgs.join(" "),
|
||||
[SemanticInternalAttributes.STYLE_ICON]: "brand-python",
|
||||
},
|
||||
}
|
||||
);
|
||||
},
|
||||
|
||||
async runInline(scriptContent: string, options: PythonExecOptions = {}): Promise<Result> {
|
||||
assert(scriptContent, "Script content is required");
|
||||
|
||||
return await logger.trace(
|
||||
"python.runInline()",
|
||||
async (span) => {
|
||||
span.setAttribute("contentLength", scriptContent.length);
|
||||
|
||||
// Using the withTempFile utility to handle the temporary file
|
||||
return await withTempFile(
|
||||
`script_${Date.now()}.py`,
|
||||
async (tempFilePath) => {
|
||||
span.setAttribute("tempFilePath", tempFilePath);
|
||||
|
||||
const pythonBin = process.env.PYTHON_BIN_PATH || "python";
|
||||
const result = await x(pythonBin, [tempFilePath], {
|
||||
...options,
|
||||
nodeOptions: {
|
||||
...(options.nodeOptions || {}),
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
},
|
||||
throwOnError: false,
|
||||
});
|
||||
|
||||
if (result.exitCode) {
|
||||
span.setAttribute("exitCode", result.exitCode);
|
||||
}
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`Inline script exited with a non-zero code ${result.exitCode}:\n${result.stderr}`
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
},
|
||||
scriptContent
|
||||
);
|
||||
},
|
||||
{
|
||||
attributes: {
|
||||
pythonBin: process.env.PYTHON_BIN_PATH || "python",
|
||||
contentPreview:
|
||||
scriptContent.substring(0, 100) + (scriptContent.length > 100 ? "..." : ""),
|
||||
[SemanticInternalAttributes.STYLE_ICON]: "brand-python",
|
||||
},
|
||||
}
|
||||
);
|
||||
},
|
||||
// Stream namespace for streaming functions
|
||||
stream: {
|
||||
run(scriptArgs: string[] = [], options: PythonExecOptions = {}): AsyncIterableStream<string> {
|
||||
const pythonBin = process.env.PYTHON_BIN_PATH || "python";
|
||||
|
||||
const pythonProcess = x(pythonBin, scriptArgs, {
|
||||
...options,
|
||||
nodeOptions: {
|
||||
...(options.nodeOptions || {}),
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
},
|
||||
throwOnError: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export default { run, runScript, runInline };
|
||||
const span = logger.startSpan("python.stream.run()", {
|
||||
attributes: {
|
||||
pythonBin,
|
||||
args: scriptArgs.join(" "),
|
||||
[SemanticInternalAttributes.STYLE_ICON]: "brand-python",
|
||||
},
|
||||
});
|
||||
|
||||
return createAsyncIterableStreamFromAsyncIterable(pythonProcess, {
|
||||
transform: (chunk, controller) => {
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
flush: () => {
|
||||
span.end();
|
||||
},
|
||||
});
|
||||
},
|
||||
runScript(
|
||||
scriptPath: string,
|
||||
scriptArgs: string[] = [],
|
||||
options: PythonExecOptions = {}
|
||||
): AsyncIterableStream<string> {
|
||||
assert(scriptPath, "Script path is required");
|
||||
assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`);
|
||||
|
||||
const pythonBin = process.env.PYTHON_BIN_PATH || "python";
|
||||
|
||||
const pythonProcess = x(pythonBin, [scriptPath, ...scriptArgs], {
|
||||
...options,
|
||||
nodeOptions: {
|
||||
...(options.nodeOptions || {}),
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
},
|
||||
throwOnError: false,
|
||||
});
|
||||
|
||||
const span = logger.startSpan("python.stream.runScript()", {
|
||||
attributes: {
|
||||
pythonBin,
|
||||
scriptPath,
|
||||
args: scriptArgs.join(" "),
|
||||
[SemanticInternalAttributes.STYLE_ICON]: "brand-python",
|
||||
},
|
||||
});
|
||||
|
||||
return createAsyncIterableStreamFromAsyncIterable(pythonProcess, {
|
||||
transform: (chunk, controller) => {
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
flush: () => {
|
||||
span.end();
|
||||
},
|
||||
});
|
||||
},
|
||||
runInline(scriptContent: string, options: PythonExecOptions = {}): AsyncIterableStream<string> {
|
||||
assert(scriptContent, "Script content is required");
|
||||
|
||||
const pythonBin = process.env.PYTHON_BIN_PATH || "python";
|
||||
|
||||
const pythonScriptPath = createTempFileSync(`script_${Date.now()}.py`, scriptContent);
|
||||
|
||||
const pythonProcess = x(pythonBin, [pythonScriptPath], {
|
||||
...options,
|
||||
nodeOptions: {
|
||||
...(options.nodeOptions || {}),
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
},
|
||||
throwOnError: false,
|
||||
});
|
||||
|
||||
const span = logger.startSpan("python.stream.runInline()", {
|
||||
attributes: {
|
||||
pythonBin,
|
||||
contentPreview:
|
||||
scriptContent.substring(0, 100) + (scriptContent.length > 100 ? "..." : ""),
|
||||
[SemanticInternalAttributes.STYLE_ICON]: "brand-python",
|
||||
},
|
||||
});
|
||||
|
||||
return createAsyncIterableStreamFromAsyncIterable(pythonProcess, {
|
||||
transform: (chunk, controller) => {
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
flush: () => {
|
||||
span.end();
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
/**
|
||||
* Creates a temporary file with a custom filename, passes it to the callback function, and ensures cleanup
|
||||
* @param filename The filename to use for the temporary file
|
||||
* @param callback Function that receives the path to the temporary file
|
||||
* @param content Optional content to write to the file
|
||||
* @returns Whatever the callback returns
|
||||
*/
|
||||
export async function withTempFile<T>(
|
||||
filename: string,
|
||||
callback: (filePath: string) => Promise<T>,
|
||||
content: string | Buffer = ""
|
||||
): Promise<T> {
|
||||
// Create temporary directory with random suffix
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "app-"));
|
||||
const tempFile = join(tempDir, filename);
|
||||
|
||||
try {
|
||||
// Write to the temporary file with appropriate permissions
|
||||
await writeFile(tempFile, content, { mode: 0o600 });
|
||||
// Use the file
|
||||
return await callback(tempFile);
|
||||
} finally {
|
||||
// Clean up
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export function createTempFileSync(filename: string, content: string | Buffer = ""): string {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "app-"));
|
||||
const tempFile = join(tempDir, filename);
|
||||
|
||||
writeFileSync(tempFile, content, { mode: 0o600 });
|
||||
return tempFile;
|
||||
}
|
||||
Generated
+28
-6
@@ -1419,15 +1419,9 @@ importers:
|
||||
|
||||
packages/python:
|
||||
dependencies:
|
||||
'@trigger.dev/build':
|
||||
specifier: workspace:3.3.16
|
||||
version: link:../build
|
||||
'@trigger.dev/core':
|
||||
specifier: workspace:3.3.16
|
||||
version: link:../core
|
||||
'@trigger.dev/sdk':
|
||||
specifier: workspace:3.3.16
|
||||
version: link:../trigger-sdk
|
||||
tinyexec:
|
||||
specifier: ^0.3.2
|
||||
version: 0.3.2
|
||||
@@ -1435,6 +1429,12 @@ importers:
|
||||
'@arethetypeswrong/cli':
|
||||
specifier: ^0.15.4
|
||||
version: 0.15.4
|
||||
'@trigger.dev/build':
|
||||
specifier: workspace:3.3.16
|
||||
version: link:../build
|
||||
'@trigger.dev/sdk':
|
||||
specifier: workspace:3.3.16
|
||||
version: link:../trigger-sdk
|
||||
'@types/node':
|
||||
specifier: 20.14.14
|
||||
version: 20.14.14
|
||||
@@ -1781,6 +1781,28 @@ importers:
|
||||
specifier: ^5
|
||||
version: 5.5.4
|
||||
|
||||
references/python-catalog:
|
||||
dependencies:
|
||||
'@trigger.dev/python':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/python
|
||||
'@trigger.dev/sdk':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/trigger-sdk
|
||||
zod:
|
||||
specifier: 3.23.8
|
||||
version: 3.23.8
|
||||
devDependencies:
|
||||
'@trigger.dev/build':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/build
|
||||
trigger.dev:
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/cli-v3
|
||||
typescript:
|
||||
specifier: ^5.5.4
|
||||
version: 5.5.4
|
||||
|
||||
references/test-tasks:
|
||||
dependencies:
|
||||
'@trigger.dev/sdk':
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "references-python-catalog",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "trigger dev",
|
||||
"deploy": "trigger deploy --self-hosted --load-image",
|
||||
"install-python-deps": "uv pip sync requirements.txt"
|
||||
},
|
||||
"dependencies": {
|
||||
"@trigger.dev/sdk": "workspace:*",
|
||||
"@trigger.dev/python": "workspace:*",
|
||||
"zod": "3.23.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@trigger.dev/build": "workspace:*",
|
||||
"trigger.dev": "workspace:*",
|
||||
"typescript": "^5.5.4"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
html2text==2024.2.26
|
||||
requests==2.32.3
|
||||
urllib3==2.3.0
|
||||
idna==3.10
|
||||
certifi==2025.1.31
|
||||
@@ -0,0 +1,37 @@
|
||||
import html2text
|
||||
import requests
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def fetch_html(url):
|
||||
"""Fetch HTML content from a URL."""
|
||||
try:
|
||||
response = requests.get(url)
|
||||
response.raise_for_status() # Raise an exception for HTTP errors
|
||||
return response.text
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Error fetching URL: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
def main():
|
||||
# Set up command line argument parsing
|
||||
parser = argparse.ArgumentParser(description='Convert HTML from a URL to plain text.')
|
||||
parser.add_argument('url', help='The URL to fetch HTML from')
|
||||
parser.add_argument('--ignore-links', action='store_true',
|
||||
help='Ignore converting links from HTML')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Fetch HTML from the URL
|
||||
html_content = fetch_html(args.url)
|
||||
|
||||
# Configure html2text
|
||||
h = html2text.HTML2Text()
|
||||
h.ignore_links = args.ignore_links
|
||||
|
||||
# Convert HTML to text and print
|
||||
text_content = h.handle(html_content)
|
||||
print(text_content)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
import { logger, schemaTask, task } from "@trigger.dev/sdk/v3";
|
||||
import { python } from "@trigger.dev/python";
|
||||
import { z } from "zod";
|
||||
|
||||
export const convertUrlToMarkdown = schemaTask({
|
||||
id: "convert-url-to-markdown",
|
||||
schema: z.object({
|
||||
url: z.string().url(),
|
||||
}),
|
||||
run: async (payload) => {
|
||||
const result = await python.runScript("./src/python/html2text_url.py", [payload.url]);
|
||||
|
||||
logger.debug("convert-url-to-markdown", {
|
||||
url: payload.url,
|
||||
output: result.stdout,
|
||||
});
|
||||
|
||||
const streamingResult = python.stream.runScript("./src/python/html2text_url.py", [payload.url]);
|
||||
|
||||
for await (const chunk of streamingResult) {
|
||||
logger.debug("convert-url-to-markdown", {
|
||||
url: payload.url,
|
||||
chunk,
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
export const pythonRunInlineTask = task({
|
||||
id: "python-run-inline",
|
||||
run: async () => {
|
||||
const result = await python.runInline(
|
||||
`
|
||||
import os
|
||||
import html2text as h2t
|
||||
|
||||
h = h2t.HTML2Text()
|
||||
|
||||
print(h.handle("<p>Hello, <a href='https://www.google.com/earth/'>world</a>!"))
|
||||
print(f"API Key: {os.environ['OPENAI_API_KEY']}")
|
||||
`,
|
||||
{
|
||||
env: {
|
||||
OPENAI_API_KEY: "sk-1234567890",
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
console.log(result.stdout);
|
||||
|
||||
const streamingResult = python.stream.runInline(`
|
||||
import html2text as h2t
|
||||
|
||||
h = h2t.HTML2Text()
|
||||
|
||||
print(h.handle("<p>Hello, <a href='https://www.google.com/earth/'>world</a>!"))
|
||||
print(h.handle("<p>Hello, <a href='https://www.google.com/earth/'>world</a>!"))
|
||||
`);
|
||||
|
||||
for await (const chunk of streamingResult) {
|
||||
logger.debug("python-run-inline", {
|
||||
chunk,
|
||||
});
|
||||
}
|
||||
|
||||
return result.stdout;
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { defineConfig } from "@trigger.dev/sdk/v3";
|
||||
import { pythonExtension } from "@trigger.dev/python/extension";
|
||||
|
||||
export default defineConfig({
|
||||
runtime: "node",
|
||||
project: "proj_hbsqkjxevkyuklehrgrp",
|
||||
machine: "small-1x",
|
||||
maxDuration: 3600,
|
||||
dirs: ["./src/trigger"],
|
||||
build: {
|
||||
extensions: [
|
||||
pythonExtension({
|
||||
requirementsFile: "./requirements.txt", // Optional: Path to your requirements file
|
||||
devPythonBinaryPath: `.venv/bin/python`, // Optional: Custom Python binary path
|
||||
scripts: ["src/python/**/*.py"], // List of Python scripts to include
|
||||
}),
|
||||
],
|
||||
},
|
||||
retries: {
|
||||
enabledInDev: true,
|
||||
default: {
|
||||
maxAttempts: 3,
|
||||
minTimeoutInMs: 1_000,
|
||||
maxTimeoutInMs: 5_000,
|
||||
factor: 1.6,
|
||||
randomize: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"strict": true,
|
||||
"outDir": "dist",
|
||||
"skipLibCheck": true,
|
||||
"customConditions": ["@triggerdotdev/source"],
|
||||
"jsx": "preserve",
|
||||
"emitDecoratorMetadata": true,
|
||||
"experimentalDecorators": true,
|
||||
"lib": ["DOM", "DOM.Iterable"],
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src/**/*.ts", "trigger.config.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user