feat: encrypt values in SecretStore using an encryption key (#240)

* feat: encrypt values in SecretStore using an encryption key and `aes-256-gcm` * Stopped using apiCors in the connectionId API endpoint
2023-08-01 10:11:47 +01:00
parent 92233f2e90
commit 36ec16babe
9 changed files with 205 additions and 31 deletions
@@ -1,6 +1,7 @@
 # YOU MIGHT LIKE TO MODIFY THESE VARIABLES
 SESSION_SECRET=abcdef1234
 MAGIC_LINK_SECRET=abcdef1234
+ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal
 LOGIN_ORIGIN=http://localhost:3030
 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public 
 REMIX_APP_PORT=3030
@@ -41,20 +41,15 @@ branch are tagged into a release monthly.
   ```
   cp .env.example .env && cp packages/database/.env.example packages/database/.env
   ```
-5. Open the root `.env` file and fill in the required values Magic Link:
+5. Open the root `.env` file and generate a new value for `ENCRYPTION_KEY`:

-   Both of these secrets should be random strings, which you can easily generate (and copy into your pasteboard) with the following command:
+   `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:

   ```sh
-   openssl rand -hex 16 | pbcopy
+   openssl rand -hex 16
   ```

-     <p>Then set them here:</p>
-
-   ```
-   SESSION_SECRET=<string>
-   MAGIC_LINK_SECRET=<string>
-   ```
+   Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.

 6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
   ```
@@ -75,6 +70,7 @@ branch are tagged into a release monthly.
 10. Run the app. See the section below.

 ## Running
+
 1. You can run the app with:

   ```
@@ -85,14 +81,12 @@ branch are tagged into a release monthly.

 2. Once the app is running click the magic link button and enter your email.
 3. Check your terminal, the magic link email should have printed out as following:
-``
-webapp:dev: Log in to Trigger.dev
+   `webapp:dev: Log in to Trigger.dev
 webapp:dev: 
 webapp:dev: Click here to log in with this magic link
 webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
 webapp:dev: 
-webapp:dev: If you didn't try to log in, you can safely ignore this email.
-``
+webapp:dev: If you didn't try to log in, you can safely ignore this email.`
 4. Paste the magic link shown in your terminal into your browser to login.

 ## Adding and running migrations
@@ -120,10 +114,11 @@ webapp:dev: If you didn't try to log in, you can safely ignore this email.
 6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.

 ## Testing CLI changes
+
 To test CLI changes, follow the steps below:

 1. Build the CLI and watch for changes
-   
+
   ```
   cd packages/cli
   pnpm run dev
@@ -148,7 +143,7 @@ To test CLI changes, follow the steps below:
   ```

 5. Open a new terminal window, navigate into the example, and initialize the CLI:
-   
+
   ```
   cd examples/your-newly-created-nextjs-project
   pnpm i
@@ -158,6 +153,7 @@ To test CLI changes, follow the steps below:
 6. When prompted, select `self-hosted` and enter `localhost:3030` for your local version of the webapp. When asked for an API key, use the key you copied earlier.

 7. Run the CLI
+
   ```
   pnpm exec trigger-cli dev
   ```
@@ -165,12 +161,13 @@ To test CLI changes, follow the steps below:
 8. After running the CLI, start your newly created Next.js project. You should now be able to see the changes.

 9. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+
 ## Add sample jobs

 The [examples/jobs-starter](./examples/jobs-starter/) project defines simple jobs you can get started with.

 1. `cd` into `examples/jobs-starter`
-2. Create a `.env.local` file with the following content, 
+2. Create a `.env.local` file with the following content,
   replacing `[TRIGGER_DEV_API_KEY]` with an actual key:

   ```
@@ -224,9 +221,7 @@ Most of the time the changes you'll make are likely to be categorized as patch r
 ### EADDRINUSE: address already in use :::3030

 When receiving the following error message:
-``
-webapp:dev: Error: listen EADDRINUSE: address already in use :::3030
-``
+`webapp:dev: Error: listen EADDRINUSE: address already in use :::3030`

 The process running on port `3030` should be destroyed.

@@ -10,6 +10,7 @@ const EnvironmentSchema = z.object({
  DATABASE_URL: z.string(),
  SESSION_SECRET: z.string(),
  MAGIC_LINK_SECRET: z.string(),
+  ENCRYPTION_KEY: z.string(),
  REMIX_APP_PORT: z.string().optional(),
  LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
  APP_ORIGIN: z.string().default("http://localhost:3030"),
@@ -0,0 +1,89 @@
+import { LoaderArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { resolveApiConnection } from "~/models/runConnection.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { apiCors } from "~/utils/apiCors";
+
+const ParamsSchema = z.object({
+  integrationSlug: z.string(),
+  connectionId: z.string(),
+});
+
+export async function loader({ request, params }: LoaderArgs) {
+  const authenticationResult = await authenticateApiRequest(request);
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (authenticationResult.type !== "PRIVATE") {
+    return json(
+      { error: "Only private API keys can access this endpoint" },
+      { status: 403 }
+    );
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return apiCors(
+      request,
+      json({ error: parsedParams.error.message }, { status: 400 })
+    );
+  }
+
+  const connection = await prisma.integrationConnection.findFirst({
+    where: {
+      id: parsedParams.data.connectionId,
+      integration: {
+        slug: parsedParams.data.integrationSlug,
+        organization: authenticatedEnv.organization,
+      },
+    },
+    include: {
+      integration: {
+        include: {
+          authMethod: true,
+        },
+      },
+      dataReference: true,
+    },
+  });
+
+  if (!connection) {
+    return apiCors(
+      request,
+      json({ error: "Connection not found" }, { status: 404 })
+    );
+  }
+
+  const auth = await resolveApiConnection(connection);
+
+  return json({
+    id: connection.id,
+    type: connection.connectionType,
+    externalAccountId: connection.externalAccountId,
+    expiresAt: connection.expiresAt,
+    auth,
+    integration: {
+      id: connection.integration.id,
+      slug: connection.integration.slug,
+      title: connection.integration.title,
+      description: connection.integration.description,
+      authSource: connection.integration.authSource,
+      authMethod: connection.integration.authMethod
+        ? {
+            id: connection.integration.authMethod.id,
+            key: connection.integration.authMethod.key,
+            name: connection.integration.authMethod.name,
+            description: connection.integration.authMethod.description,
+            type: connection.integration.authMethod.type,
+          }
+        : null,
+    },
+    createdAt: connection.createdAt,
+    updatedAt: connection.updatedAt,
+  });
+}
@@ -44,8 +44,6 @@ export type ConnectionWithSecretReference = IntegrationConnection & {
  dataReference: SecretReference;
 };

-const randomGenerator = customAlphabet("1234567890abcdef", 3);
-
 /** How many seconds before expiry we should refresh the token  */
 const tokenRefreshThreshold = 5 * 60;

@@ -273,7 +273,7 @@ function convertToken({
  const scopesPtr = jsonpointer.compile(scopePointer);
  const scopesValue = scopesPtr.get(token.token);
  if (typeof scopesValue === "string") {
-    actualScopes = (scopesValue as string).split(scopeSeparator);
+    actualScopes = scopesValue.split(scopeSeparator);
  }

  const refreshTokenPtr = jsonpointer.compile(refreshTokenPointer);
@@ -1,5 +1,8 @@
 import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { z } from "zod";
+import { env } from "~/env.server";
+import nodeCrypto from "node:crypto";
+import { safeJsonParse } from "~/utils/json";

 export const SecretStoreOptionsSchema = z.enum(["DATABASE", "AWS_PARAM_STORE"]);
 export type SecretStoreOptions = z.infer<typeof SecretStoreOptionsSchema>;
@@ -41,11 +44,20 @@ export class SecretStore {
  }
 }

-/** This stores secrets in the Postgres Database, in plain text. NOT recommended outside of localhost. */
+const EncryptedSecretValueSchema = z.object({
+  nonce: z.string(),
+  ciphertext: z.string(),
+  tag: z.string(),
+});
+
+/** This stores secrets in the Postgres Database, encrypted using aes-256-gcm */
 class PrismaSecretStore implements SecretStoreProvider {
  #prismaClient: PrismaClientOrTransaction;

-  constructor(private options?: ProviderInitializationOptions["DATABASE"]) {
+  constructor(
+    private readonly encryptionKey: string,
+    private options?: ProviderInitializationOptions["DATABASE"]
+  ) {
    this.#prismaClient = options?.prismaClient ?? prisma;
  }

@@ -60,23 +72,94 @@ class PrismaSecretStore implements SecretStoreProvider {
      return undefined;
    }

-    return schema.parse(secret.value);
+    if (secret.version === "1") {
+      return schema.parse(secret.value);
+    }
+
+    const encryptedData = EncryptedSecretValueSchema.safeParse(secret.value);
+
+    if (!encryptedData.success) {
+      throw new Error(
+        `Unable to parse encrypted secret ${key}: ${encryptedData.error.message}`
+      );
+    }
+
+    const decrypted = await this.#decrypt(
+      encryptedData.data.nonce,
+      encryptedData.data.ciphertext,
+      encryptedData.data.tag
+    );
+
+    const parsedDecrypted = safeJsonParse(decrypted);
+
+    if (!parsedDecrypted) {
+      return;
+    }
+
+    return schema.parse(parsedDecrypted);
  }

  async setSecret<T extends object>(key: string, value: T): Promise<void> {
+    const encrypted = await this.#encrypt(JSON.stringify(value));
+
    await this.#prismaClient.secretStore.upsert({
      create: {
        key,
-        value,
+        value: encrypted,
+        version: "2",
      },
      update: {
-        value,
+        value: encrypted,
+        version: "2",
      },
      where: {
        key,
      },
    });
  }
+
+  async #decrypt(
+    nonce: string,
+    ciphertext: string,
+    tag: string
+  ): Promise<string> {
+    const decipher = nodeCrypto.createDecipheriv(
+      "aes-256-gcm",
+      this.encryptionKey,
+      Buffer.from(nonce, "hex")
+    );
+
+    decipher.setAuthTag(Buffer.from(tag, "hex"));
+
+    let decrypted = decipher.update(ciphertext, "hex", "utf8");
+    decrypted += decipher.final("utf8");
+
+    return decrypted;
+  }
+
+  async #encrypt(value: string): Promise<{
+    nonce: string;
+    ciphertext: string;
+    tag: string;
+  }> {
+    const nonce = nodeCrypto.randomBytes(12);
+    const cipher = nodeCrypto.createCipheriv(
+      "aes-256-gcm",
+      this.encryptionKey,
+      nonce
+    );
+
+    let encrypted = cipher.update(value, "utf8", "hex");
+    encrypted += cipher.final("hex");
+
+    const tag = cipher.getAuthTag().toString("hex");
+
+    return {
+      nonce: nonce.toString("hex"),
+      ciphertext: encrypted,
+      tag,
+    };
+  }
 }

 export function getSecretStore<
@@ -85,7 +168,12 @@ export function getSecretStore<
 >(provider: K, options?: TOptions): SecretStore {
  switch (provider) {
    case "DATABASE": {
-      return new SecretStore(new PrismaSecretStore(options as any));
+      return new SecretStore(
+        new PrismaSecretStore(
+          env.ENCRYPTION_KEY,
+          options as ProviderInitializationOptions["DATABASE"]
+        )
+      );
    }
    default: {
      throw new Error(`Unsupported secret store option ${provider}`);
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SecretStore" ADD COLUMN     "version" TEXT NOT NULL DEFAULT '1';
@@ -850,10 +850,10 @@ enum SecretStoreProvider {
  AWS_PARAM_STORE
 }

-/// Used when the provider = "database". Not recommended outside of local development.
 model SecretStore {
-  key   String @unique
-  value Json
+  key     String @unique
+  value   Json
+  version String @default("1")

  createdAt DateTime @default(now())
  updatedAt DateTime @updatedAt