diff --git a/.env.example b/.env.example
index 9d4333ee5..46be04c20 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,7 @@
# YOU MIGHT LIKE TO MODIFY THESE VARIABLES
SESSION_SECRET=abcdef1234
MAGIC_LINK_SECRET=abcdef1234
+ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal
LOGIN_ORIGIN=http://localhost:3030
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public
REMIX_APP_PORT=3030
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3d2d50ded..25fc75df8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -41,20 +41,15 @@ branch are tagged into a release monthly.
```
cp .env.example .env && cp packages/database/.env.example packages/database/.env
```
-5. Open the root `.env` file and fill in the required values Magic Link:
+5. Open the root `.env` file and generate a new value for `ENCRYPTION_KEY`:
- Both of these secrets should be random strings, which you can easily generate (and copy into your pasteboard) with the following command:
+ `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
```sh
- openssl rand -hex 16 | pbcopy
+ openssl rand -hex 16
```
-
Then set them here:
-
- ```
- SESSION_SECRET=
- MAGIC_LINK_SECRET=
- ```
+ Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
```
@@ -75,6 +70,7 @@ branch are tagged into a release monthly.
10. Run the app. See the section below.
## Running
+
1. You can run the app with:
```
@@ -85,14 +81,12 @@ branch are tagged into a release monthly.
2. Once the app is running click the magic link button and enter your email.
3. Check your terminal, the magic link email should have printed out as following:
-``
-webapp:dev: Log in to Trigger.dev
+ `webapp:dev: Log in to Trigger.dev
webapp:dev:
webapp:dev: Click here to log in with this magic link
webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
webapp:dev:
-webapp:dev: If you didn't try to log in, you can safely ignore this email.
-``
+webapp:dev: If you didn't try to log in, you can safely ignore this email.`
4. Paste the magic link shown in your terminal into your browser to login.
## Adding and running migrations
@@ -120,10 +114,11 @@ webapp:dev: If you didn't try to log in, you can safely ignore this email.
6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
## Testing CLI changes
+
To test CLI changes, follow the steps below:
1. Build the CLI and watch for changes
-
+
```
cd packages/cli
pnpm run dev
@@ -148,7 +143,7 @@ To test CLI changes, follow the steps below:
```
5. Open a new terminal window, navigate into the example, and initialize the CLI:
-
+
```
cd examples/your-newly-created-nextjs-project
pnpm i
@@ -158,6 +153,7 @@ To test CLI changes, follow the steps below:
6. When prompted, select `self-hosted` and enter `localhost:3030` for your local version of the webapp. When asked for an API key, use the key you copied earlier.
7. Run the CLI
+
```
pnpm exec trigger-cli dev
```
@@ -165,12 +161,13 @@ To test CLI changes, follow the steps below:
8. After running the CLI, start your newly created Next.js project. You should now be able to see the changes.
9. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+
## Add sample jobs
The [examples/jobs-starter](./examples/jobs-starter/) project defines simple jobs you can get started with.
1. `cd` into `examples/jobs-starter`
-2. Create a `.env.local` file with the following content,
+2. Create a `.env.local` file with the following content,
replacing `[TRIGGER_DEV_API_KEY]` with an actual key:
```
@@ -224,9 +221,7 @@ Most of the time the changes you'll make are likely to be categorized as patch r
### EADDRINUSE: address already in use :::3030
When receiving the following error message:
-``
-webapp:dev: Error: listen EADDRINUSE: address already in use :::3030
-``
+`webapp:dev: Error: listen EADDRINUSE: address already in use :::3030`
The process running on port `3030` should be destroyed.
diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts
index 4531669e4..062db56c4 100644
--- a/apps/webapp/app/env.server.ts
+++ b/apps/webapp/app/env.server.ts
@@ -10,6 +10,7 @@ const EnvironmentSchema = z.object({
DATABASE_URL: z.string(),
SESSION_SECRET: z.string(),
MAGIC_LINK_SECRET: z.string(),
+ ENCRYPTION_KEY: z.string(),
REMIX_APP_PORT: z.string().optional(),
LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
APP_ORIGIN: z.string().default("http://localhost:3030"),
diff --git a/apps/webapp/app/routes/api.v1.integrations.$integrationSlug.connections.$connectionId.ts b/apps/webapp/app/routes/api.v1.integrations.$integrationSlug.connections.$connectionId.ts
new file mode 100644
index 000000000..1d1dc933f
--- /dev/null
+++ b/apps/webapp/app/routes/api.v1.integrations.$integrationSlug.connections.$connectionId.ts
@@ -0,0 +1,89 @@
+import { LoaderArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { resolveApiConnection } from "~/models/runConnection.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { apiCors } from "~/utils/apiCors";
+
+const ParamsSchema = z.object({
+ integrationSlug: z.string(),
+ connectionId: z.string(),
+});
+
+export async function loader({ request, params }: LoaderArgs) {
+ const authenticationResult = await authenticateApiRequest(request);
+ if (!authenticationResult) {
+ return json({ error: "Invalid or Missing API key" }, { status: 401 });
+ }
+
+ if (authenticationResult.type !== "PRIVATE") {
+ return json(
+ { error: "Only private API keys can access this endpoint" },
+ { status: 403 }
+ );
+ }
+
+ const authenticatedEnv = authenticationResult.environment;
+
+ const parsedParams = ParamsSchema.safeParse(params);
+
+ if (!parsedParams.success) {
+ return apiCors(
+ request,
+ json({ error: parsedParams.error.message }, { status: 400 })
+ );
+ }
+
+ const connection = await prisma.integrationConnection.findFirst({
+ where: {
+ id: parsedParams.data.connectionId,
+ integration: {
+ slug: parsedParams.data.integrationSlug,
+ organization: authenticatedEnv.organization,
+ },
+ },
+ include: {
+ integration: {
+ include: {
+ authMethod: true,
+ },
+ },
+ dataReference: true,
+ },
+ });
+
+ if (!connection) {
+ return apiCors(
+ request,
+ json({ error: "Connection not found" }, { status: 404 })
+ );
+ }
+
+ const auth = await resolveApiConnection(connection);
+
+ return json({
+ id: connection.id,
+ type: connection.connectionType,
+ externalAccountId: connection.externalAccountId,
+ expiresAt: connection.expiresAt,
+ auth,
+ integration: {
+ id: connection.integration.id,
+ slug: connection.integration.slug,
+ title: connection.integration.title,
+ description: connection.integration.description,
+ authSource: connection.integration.authSource,
+ authMethod: connection.integration.authMethod
+ ? {
+ id: connection.integration.authMethod.id,
+ key: connection.integration.authMethod.key,
+ name: connection.integration.authMethod.name,
+ description: connection.integration.authMethod.description,
+ type: connection.integration.authMethod.type,
+ }
+ : null,
+ },
+ createdAt: connection.createdAt,
+ updatedAt: connection.updatedAt,
+ });
+}
diff --git a/apps/webapp/app/services/externalApis/integrationAuthRepository.server.ts b/apps/webapp/app/services/externalApis/integrationAuthRepository.server.ts
index d83a008e3..f24dc972e 100644
--- a/apps/webapp/app/services/externalApis/integrationAuthRepository.server.ts
+++ b/apps/webapp/app/services/externalApis/integrationAuthRepository.server.ts
@@ -44,8 +44,6 @@ export type ConnectionWithSecretReference = IntegrationConnection & {
dataReference: SecretReference;
};
-const randomGenerator = customAlphabet("1234567890abcdef", 3);
-
/** How many seconds before expiry we should refresh the token */
const tokenRefreshThreshold = 5 * 60;
diff --git a/apps/webapp/app/services/externalApis/oauth2.server.ts b/apps/webapp/app/services/externalApis/oauth2.server.ts
index 1c8f83e1c..09e07e7f7 100644
--- a/apps/webapp/app/services/externalApis/oauth2.server.ts
+++ b/apps/webapp/app/services/externalApis/oauth2.server.ts
@@ -273,7 +273,7 @@ function convertToken({
const scopesPtr = jsonpointer.compile(scopePointer);
const scopesValue = scopesPtr.get(token.token);
if (typeof scopesValue === "string") {
- actualScopes = (scopesValue as string).split(scopeSeparator);
+ actualScopes = scopesValue.split(scopeSeparator);
}
const refreshTokenPtr = jsonpointer.compile(refreshTokenPointer);
diff --git a/apps/webapp/app/services/secrets/secretStore.server.ts b/apps/webapp/app/services/secrets/secretStore.server.ts
index 101ce20b4..b34ea7830 100644
--- a/apps/webapp/app/services/secrets/secretStore.server.ts
+++ b/apps/webapp/app/services/secrets/secretStore.server.ts
@@ -1,5 +1,8 @@
import { PrismaClientOrTransaction, prisma } from "~/db.server";
import { z } from "zod";
+import { env } from "~/env.server";
+import nodeCrypto from "node:crypto";
+import { safeJsonParse } from "~/utils/json";
export const SecretStoreOptionsSchema = z.enum(["DATABASE", "AWS_PARAM_STORE"]);
export type SecretStoreOptions = z.infer;
@@ -41,11 +44,20 @@ export class SecretStore {
}
}
-/** This stores secrets in the Postgres Database, in plain text. NOT recommended outside of localhost. */
+const EncryptedSecretValueSchema = z.object({
+ nonce: z.string(),
+ ciphertext: z.string(),
+ tag: z.string(),
+});
+
+/** This stores secrets in the Postgres Database, encrypted using aes-256-gcm */
class PrismaSecretStore implements SecretStoreProvider {
#prismaClient: PrismaClientOrTransaction;
- constructor(private options?: ProviderInitializationOptions["DATABASE"]) {
+ constructor(
+ private readonly encryptionKey: string,
+ private options?: ProviderInitializationOptions["DATABASE"]
+ ) {
this.#prismaClient = options?.prismaClient ?? prisma;
}
@@ -60,23 +72,94 @@ class PrismaSecretStore implements SecretStoreProvider {
return undefined;
}
- return schema.parse(secret.value);
+ if (secret.version === "1") {
+ return schema.parse(secret.value);
+ }
+
+ const encryptedData = EncryptedSecretValueSchema.safeParse(secret.value);
+
+ if (!encryptedData.success) {
+ throw new Error(
+ `Unable to parse encrypted secret ${key}: ${encryptedData.error.message}`
+ );
+ }
+
+ const decrypted = await this.#decrypt(
+ encryptedData.data.nonce,
+ encryptedData.data.ciphertext,
+ encryptedData.data.tag
+ );
+
+ const parsedDecrypted = safeJsonParse(decrypted);
+
+ if (!parsedDecrypted) {
+ return;
+ }
+
+ return schema.parse(parsedDecrypted);
}
async setSecret(key: string, value: T): Promise {
+ const encrypted = await this.#encrypt(JSON.stringify(value));
+
await this.#prismaClient.secretStore.upsert({
create: {
key,
- value,
+ value: encrypted,
+ version: "2",
},
update: {
- value,
+ value: encrypted,
+ version: "2",
},
where: {
key,
},
});
}
+
+ async #decrypt(
+ nonce: string,
+ ciphertext: string,
+ tag: string
+ ): Promise {
+ const decipher = nodeCrypto.createDecipheriv(
+ "aes-256-gcm",
+ this.encryptionKey,
+ Buffer.from(nonce, "hex")
+ );
+
+ decipher.setAuthTag(Buffer.from(tag, "hex"));
+
+ let decrypted = decipher.update(ciphertext, "hex", "utf8");
+ decrypted += decipher.final("utf8");
+
+ return decrypted;
+ }
+
+ async #encrypt(value: string): Promise<{
+ nonce: string;
+ ciphertext: string;
+ tag: string;
+ }> {
+ const nonce = nodeCrypto.randomBytes(12);
+ const cipher = nodeCrypto.createCipheriv(
+ "aes-256-gcm",
+ this.encryptionKey,
+ nonce
+ );
+
+ let encrypted = cipher.update(value, "utf8", "hex");
+ encrypted += cipher.final("hex");
+
+ const tag = cipher.getAuthTag().toString("hex");
+
+ return {
+ nonce: nonce.toString("hex"),
+ ciphertext: encrypted,
+ tag,
+ };
+ }
}
export function getSecretStore<
@@ -85,7 +168,12 @@ export function getSecretStore<
>(provider: K, options?: TOptions): SecretStore {
switch (provider) {
case "DATABASE": {
- return new SecretStore(new PrismaSecretStore(options as any));
+ return new SecretStore(
+ new PrismaSecretStore(
+ env.ENCRYPTION_KEY,
+ options as ProviderInitializationOptions["DATABASE"]
+ )
+ );
}
default: {
throw new Error(`Unsupported secret store option ${provider}`);
diff --git a/packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql b/packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql
new file mode 100644
index 000000000..669c46311
--- /dev/null
+++ b/packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SecretStore" ADD COLUMN "version" TEXT NOT NULL DEFAULT '1';
diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma
index ce63d5cd5..a0958fc3a 100644
--- a/packages/database/prisma/schema.prisma
+++ b/packages/database/prisma/schema.prisma
@@ -850,10 +850,10 @@ enum SecretStoreProvider {
AWS_PARAM_STORE
}
-/// Used when the provider = "database". Not recommended outside of local development.
model SecretStore {
- key String @unique
- value Json
+ key String @unique
+ value Json
+ version String @default("1")
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt