github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Implement the task output redacting to prevent redacted values from showing in the logs

Browse Source
This commit is contained in:
Eric Allam
2023-09-27 16:38:30 +01:00
parent 813ec74672
commit 35d0c2a06f
7 changed files with 267 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/webapp/app/presenters/TaskDetailsPresenter.server.ts
+25 -1
View File
@@ -1,6 +1,8 @@
import { RedactSchema } from "@trigger.dev/core";
import { StyleSchema } from "@trigger.dev/core";
import { PrismaClient, prisma } from "~/db.server";
import { mergeProperties } from "~/utils/mergeProperties.server";
import { Redactor } from "~/utils/redactor";
type DetailsProps = {
id: string;
@@ -61,6 +63,7 @@ export class TaskDetailsPresenter {
completedAt: true,
style: true,
parentId: true,
redact: true,
attempts: {
select: {
number: true,
@@ -85,11 +88,32 @@ export class TaskDetailsPresenter {
return {
...task,
output: task.output ? JSON.stringify(task.output, null, 2) : undefined,
redact: undefined,
output: task.output
? JSON.stringify(this.#stringifyOutputWithRedactions(task.output, task.redact), null, 2)
: undefined,
connection: task.runConnection,
params: task.params as Record<string, any>,
properties: mergeProperties(task.properties, task.outputProperties),
style: task.style ? StyleSchema.parse(task.style) : undefined,
};
}
#stringifyOutputWithRedactions(output: any, redact: unknown): any {
if (!output) {
return;
}
const parsedRedact = RedactSchema.safeParse(redact);
if (!parsedRedact.success) {
return output;
}
const paths = parsedRedact.data.paths;
const redactor = new Redactor(paths);
return redactor.redact(output);
}
}
apps/webapp/app/utils/redactor.ts
+71
View File
@@ -0,0 +1,71 @@
// Redacts the given object based on the given paths
// Example:
// const redactor = new Redactor(["data.object.balance_transaction"]);
// redactor.redact({
// data: {
// object: {
// balance_transaction: "txn_1NYWgTI0XSgju2urW3aXpinM",
// },
// },
// });
// Returns:
// {
// data: {
// object: {
// balance_transaction: "[REDACTED]",
// },
// },
// }
// Does not currenly support arrays
export class Redactor {
constructor(private paths: string[]) {}
public redact(subject: unknown): unknown {
if (!Array.isArray(this.paths)) {
return subject;
}
if (this.paths.length === 0) {
return subject;
}
const clonedSubject = JSON.parse(JSON.stringify(subject));
return this.redactPathsRecursive(clonedSubject, this.paths);
}
private redactPathsRecursive(subject: any, paths: string[]): any {
for (let path of paths) {
let parts = path.split(".");
let curSubject = subject;
// Make sure curSubject is an object
if (typeof curSubject !== "object") {
break;
}
for (let i = 0; i < parts.length; i++) {
const part = parts[i];
if (Object.prototype.hasOwnProperty.call(curSubject, part) === false) {
// Path is not found in object
break;
}
if (i === parts.length - 1) {
// We're at the end of our path and have a string, redact it
curSubject[part] = "[REDACTED]";
} else if (part in curSubject && typeof curSubject[part] === "object") {
// More paths to follow, continue down the path
curSubject = curSubject[part];
} else {
// Path is not found in object or doesn't point to a string
break;
}
}
}
return subject;
}
}
apps/webapp/package.json
-1
View File
@@ -73,7 +73,6 @@
"cuid": "^2.1.8",
"emails": "workspace:*",
"express": "^4.18.1",
"fast-redact": "^3.1.2",
"framer-motion": "^10.12.11",
"graphile-worker": "^0.13.0",
"highlight.run": "^7.3.4",
docs/sdk/io/runtask.mdx
+17 -3
View File
@@ -82,9 +82,12 @@ A Task is a resumable unit of a Run that can be retried, resumed and is logged.
</Expandable>
</ResponseField>
<ResponseField name="params" type="any">
The input params to the Task, will be displayed in the logs.
</ResponseField>
{" "}
<ResponseField name="params" type="any">
The input params to the Task, will be displayed in the logs.
</ResponseField>
<ResponseField name="style" type="object">
The style of the log entry.
@@ -98,6 +101,17 @@ A Task is a resumable unit of a Run that can be retried, resumed and is logged.
</Expandable>
</ResponseField>
<ResponseField name="redact" type="RedactOptions">
An optional object that specifies which fields to redact from the logs. This is useful for sensitive data like API keys.
<Expandable title="redact" defaultOpen>
<ResponseField name="paths" type="string[]">
An array of paths to redact. A path is a dot separated string, e.g. `user.email`. Currently does not support wildcards.
</ResponseField>
</Expandable>
</ResponseField>
</Expandable>
</ResponseField>
pnpm-lock.yaml
Generated
-7
View File
@@ -143,7 +143,6 @@ importers:
eslint: ^8.24.0
eslint-config-prettier: ^8.5.0
express: ^4.18.1
fast-redact: ^3.1.2
framer-motion: ^10.12.11
graphile-worker: ^0.13.0
highlight.run: ^7.3.4
@@ -242,7 +241,6 @@ importers:
cuid: 2.1.8
emails: link:../../packages/emails
express: 4.18.2
fast-redact: 3.1.2
framer-motion: 10.12.11_biqbaboplfbrettd7655fr4n2y
graphile-worker: 0.13.0
highlight.run: 7.3.4
@@ -18996,11 +18994,6 @@ packages:
resolution: {integrity: sha512-Ue0LwpDYErFbmNnZSF0UH6eImUwDmogUO1jyE+JbN2gsQz/jICm1Ve7t9QT0rNSsfJt+Hs4/S3GnsDVjL4HVrw==}
dev: false
/fast-redact/3.1.2:
resolution: {integrity: sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw==}
engines: {node: '>=6'}
dev: false
/fast-shallow-equal/1.0.0:
resolution: {integrity: sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw==}
dev: false
references/job-catalog/package.json
+1
View File
@@ -24,6 +24,7 @@
"linear": "nodemon --watch src/linear.ts -r tsconfig-paths/register -r dotenv/config src/linear.ts",
"status": "nodemon --watch src/status.ts -r tsconfig-paths/register -r dotenv/config src/status.ts",
"byo-auth": "nodemon --watch src/byo-auth.ts -r tsconfig-paths/register -r dotenv/config src/byo-auth.ts",
"redacted": "nodemon --watch src/redacted.ts -r tsconfig-paths/register -r dotenv/config src/redacted.ts",
"dev:trigger": "trigger-cli dev --port 8080"
},
"dependencies": {
references/job-catalog/src/redacted.ts
+153
View File
@@ -0,0 +1,153 @@
import { createExpressServer } from "@trigger.dev/express";
import { TriggerClient, eventTrigger } from "@trigger.dev/sdk";
import { z } from "zod";
export const client = new TriggerClient({
id: "job-catalog",
apiKey: process.env["TRIGGER_API_KEY"],
apiUrl: process.env["TRIGGER_API_URL"],
verbose: false,
ioLogLocalEnabled: true,
});
client.defineJob({
id: "redaction-example-1",
name: "Redaction Example 1",
version: "1.0.0",
enabled: true,
trigger: eventTrigger({
name: "redaction.example",
}),
run: async (payload, io, ctx) => {
const result = await io.runTask(
"task-example-1",
async () => {
return {
id: "evt_3NYWgVI0XSgju2ur0PN22Hsu",
object: "event",
api_version: "2022-11-15",
created: 1690473903,
data: {
object: {
id: "ch_3NYWgVI0XSgju2ur0C2UzeKC",
object: "charge",
amount: 1500,
amount_captured: 1500,
amount_refunded: 0,
application: null,
application_fee: null,
application_fee_amount: null,
balance_transaction: "txn_3NYWgVI0XSgju2ur0qujz4Kc",
billing_details: {
address: {
city: null,
country: null,
line1: null,
line2: null,
postal_code: null,
state: null,
},
email: null,
name: null,
phone: null,
},
calculated_statement_descriptor: "WWW.TRIGGER.DEV",
captured: true,
created: 1690473903,
currency: "usd",
customer: "cus_OLD6IR3D8CJasG",
description: "Subscription creation",
destination: null,
dispute: null,
disputed: false,
failure_balance_transaction: null,
failure_code: null,
failure_message: null,
fraud_details: {},
invoice: "in_1NYWgUI0XSgju2urV5ZTEyIn",
livemode: false,
metadata: {},
on_behalf_of: null,
order: null,
outcome: {
network_status: "approved_by_network",
reason: null,
risk_level: "normal",
risk_score: 61,
seller_message: "Payment complete.",
type: "authorized",
},
paid: true,
payment_intent: "pi_3NYWgVI0XSgju2ur0fWNLexG",
payment_method: "pm_1NYWgTI0XSgju2urW3aXpinM",
payment_method_details: {
card: {
brand: "visa",
checks: {
address_line1_check: null,
address_postal_code_check: null,
cvc_check: null,
},
country: "US",
exp_month: 7,
exp_year: 2024,
fingerprint: "w6qgKDLO5EbIJ5VZ",
funding: "credit",
installments: null,
last4: "4242",
mandate: null,
network: "visa",
network_token: {
used: false,
},
three_d_secure: null,
wallet: null,
},
type: "card",
},
receipt_email: null,
receipt_number: null,
receipt_url:
"https://pay.stripe.com/receipts/invoices/CAcaFwoVYWNjdF8xTVJtRzRJMFhTZ2p1MnVyKLCriqYGMga_ozxgMkA6LBbrKccthI_hGdug_gXtuu_piRAvzyNVaH_aMq9mUTOl3VdNbfcH7nhFjK08?s=ap",
refunded: false,
review: null,
shipping: null,
source: null,
source_transfer: null,
statement_descriptor: null,
statement_descriptor_suffix: null,
status: "succeeded",
transfer_data: null,
transfer_group: null,
},
},
livemode: false,
pending_webhooks: 2,
request: {
id: "req_vtwGrzB2O98Pnc",
idempotency_key: "215856c0-4f06-48eb-94c6-7ed4e839d7bc",
},
type: "charge.succeeded",
};
},
{
redact: {
paths: [
"data.object.balance_transaction",
"data.object.billing_details",
"data.object.this_does_not_exist",
"data.object.$$$$hello",
],
},
}
);
await io.logger.info("Log.1", { ctx, result });
await io.wait("wait-1", 1);
await io.logger.info("Log.2", { ctx, result });
},
});
createExpressServer(client);