Merge branch 'main' into v3/worker-attempt-creation

This commit is contained in:
nicktrn
2024-05-02 17:19:08 +01:00
112 changed files with 1644 additions and 389 deletions
+6
View File
@@ -0,0 +1,6 @@
---
"trigger.dev": patch
"@trigger.dev/core": patch
---
Fix issues with consecutive waits
+4
View File
@@ -67,6 +67,7 @@
"lemon-jobs-repair",
"light-bulldogs-press",
"light-dragons-complain",
"little-crabs-cross",
"loud-actors-remember",
"many-ligers-pump",
"mighty-camels-joke",
@@ -79,6 +80,7 @@
"polite-rockets-matter",
"poor-flowers-cross",
"purple-garlics-shop",
"rare-lamps-promise",
"rare-roses-float",
"real-planets-stare",
"rich-kangaroos-unite",
@@ -90,6 +92,7 @@
"sharp-zebras-serve",
"shiny-coats-cry",
"silly-suits-switch",
"six-ligers-exist",
"slow-buses-own",
"smart-needles-move",
"smart-olives-eat",
@@ -101,6 +104,7 @@
"swift-dragons-peel",
"tall-bees-wave",
"tame-guests-know",
"tender-moose-tell",
"tender-oranges-rhyme",
"tidy-balloons-suffer",
"tidy-dryers-sleep",
+6
View File
@@ -0,0 +1,6 @@
---
"trigger.dev": patch
"@trigger.dev/core": patch
---
Add a postInstall option to allow running scripts after dependencies have been installed in deployed images
+5
View File
@@ -0,0 +1,5 @@
---
"@trigger.dev/core": patch
---
Improve the display of non-object return types in the run trace viewer
+5
View File
@@ -0,0 +1,5 @@
---
"trigger.dev": patch
---
Use locked package versions when resolving dependencies in deployed workers
@@ -0,0 +1,29 @@
import { ShieldCheckIcon } from "@heroicons/react/20/solid";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from "~/components/primitives/Tooltip";
import { useIsImpersonating } from "~/hooks/useOrganizations";
import { useHasAdminAccess } from "~/hooks/useUser";
export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
const hasAdminAccess = useHasAdminAccess();
const isImpersonating = useIsImpersonating();
if (!hasAdminAccess && !isImpersonating) {
return null;
}
return (
<TooltipProvider>
<Tooltip>
<TooltipTrigger>
<ShieldCheckIcon className="h-5 w-5" />
</TooltipTrigger>
<TooltipContent className="flex items-center gap-1">{children}</TooltipContent>
</Tooltip>
</TooltipProvider>
);
}
@@ -53,3 +53,11 @@ export function useIsNewOrganizationPage(matches?: UIMatch[]): boolean {
export const useOrganizationChanged = (action: (org: MatchedOrganization | undefined) => void) => {
useChanged(useOptionalOrganization, action);
};
export function useIsImpersonating(matches?: UIMatch[]) {
const data = useTypedMatchesData<typeof orgLoader>({
id: "routes/_app.orgs.$organizationSlug",
matches,
});
return data?.isImpersonating === true;
}
+6
View File
@@ -26,3 +26,9 @@ export function useUser(matches?: UIMatch[]): User {
export function useUserChanged(callback: (user: User | undefined) => void) {
useChanged(useOptionalUser, callback);
}
export function useHasAdminAccess(matches?: UIMatch[]): boolean {
const user = useOptionalUser(matches);
return Boolean(user?.admin);
}
+6
View File
@@ -168,6 +168,12 @@ export async function adminGetOrganizations(userId: string, { page, search }: Se
mode: "insensitive",
},
},
{
id: {
contains: search,
mode: "insensitive",
},
},
],
}
: undefined,
+1 -1
View File
@@ -1,6 +1,6 @@
import { json, Session } from "@remix-run/node";
import { redirect } from "remix-typedjson";
import { createCookieSessionStorage } from "@remix-run/node";
import { redirect } from "remix-typedjson";
import { env } from "~/env.server";
export type ToastMessage = {
@@ -38,6 +38,7 @@ export class RunPresenter {
id: true,
type: true,
slug: true,
organizationId: true,
orgMember: {
select: {
user: {
@@ -66,9 +67,13 @@ export class RunPresenter {
if (!traceSummary) {
return {
run: {
id: run.id,
number: run.number,
friendlyId: run.friendlyId,
traceId: run.traceId,
environment: {
id: run.runtimeEnvironment.id,
organizationId: run.runtimeEnvironment.organizationId,
type: run.runtimeEnvironment.type,
slug: run.runtimeEnvironment.slug,
userId: run.runtimeEnvironment.orgMember?.user.id,
@@ -118,9 +123,13 @@ export class RunPresenter {
return {
run: {
id: run.id,
number: run.number,
friendlyId: run.friendlyId,
traceId: run.traceId,
environment: {
id: run.runtimeEnvironment.id,
organizationId: run.runtimeEnvironment.organizationId,
type: run.runtimeEnvironment.type,
slug: run.runtimeEnvironment.slug,
userId: run.runtimeEnvironment.orgMember?.user.id,
@@ -42,7 +42,7 @@ export class SpanPresenter {
const output =
span.outputType === "application/store"
? `/resources/packets/${span.environmentId}/${span.output}`
: typeof span.output !== "undefined" && span.output !== null
: typeof span.output !== "undefined"
? await prettyPrintPacket(span.output, span.outputType ?? undefined)
: undefined;
@@ -9,6 +9,7 @@ import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
import { Feedback } from "~/components/Feedback";
import { InitCommandV3, TriggerDevStepV3, TriggerLoginStepV3 } from "~/components/SetupCommands";
import { StepContentContainer } from "~/components/StepContentContainer";
import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { InlineCode } from "~/components/code/InlineCode";
import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -17,8 +18,9 @@ import { Callout } from "~/components/primitives/Callout";
import { formatDateTime } from "~/components/primitives/DateTime";
import { Header1, Header2, Header3 } from "~/components/primitives/Headers";
import { Input } from "~/components/primitives/Input";
import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
import { Spinner } from "~/components/primitives/Spinner";
import { StepNumber } from "~/components/primitives/StepNumber";
import {
@@ -125,6 +127,25 @@ export default function Page() {
<PageContainer>
<NavBar>
<PageTitle title="Tasks" />
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
{tasks.map((task) => (
<Property label={task.exportName} key={task.slug}>
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">
{task.environments
.map((e) =>
e.userName ? `${e.userName}/${e.id}` : `${e.type.slice(0, 3)}/${e.id}`
)
.join(", ")}
</Paragraph>
</div>
</Property>
))}
</PropertyTable>
</AdminDebugTooltip>
</PageAccessories>
</NavBar>
<PageBody>
<div className={cn("grid h-full grid-rows-1")}>
@@ -1,7 +1,8 @@
import { BookOpenIcon } from "@heroicons/react/20/solid";
import { BookOpenIcon, LightBulbIcon, ShieldCheckIcon } from "@heroicons/react/20/solid";
import { Form } from "@remix-run/react";
import { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
import { typedjson, useTypedLoaderData } from "remix-typedjson";
import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { EnvironmentLabel, environmentTitle } from "~/components/environments/EnvironmentLabel";
import { RegenerateApiKeyModal } from "~/components/environments/RegenerateApiKeyModal";
import { PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -10,8 +11,9 @@ import { Callout } from "~/components/primitives/Callout";
import { ClipboardField } from "~/components/primitives/ClipboardField";
import { DateTime } from "~/components/primitives/DateTime";
import { Header3 } from "~/components/primitives/Headers";
import { PageAccessories, NavBar, PageTitle } from "~/components/primitives/PageHeader";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
import {
Table,
TableBody,
@@ -24,7 +26,6 @@ import {
import { TextLink } from "~/components/primitives/TextLink";
import { prisma } from "~/db.server";
import { useFeatures } from "~/hooks/useFeatures";
import { useProject } from "~/hooks/useProject";
import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
import { createEnvironment } from "~/models/organization.server";
import { ApiKeysPresenter } from "~/presenters/v3/ApiKeysPresenter.server";
@@ -132,6 +133,18 @@ export default function Page() {
<NavBar>
<PageTitle title="API keys" />
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
{environments.map((environment) => (
<Property label={environment.slug} key={environment.id}>
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{environment.id}</Paragraph>
</div>
</Property>
))}
</PropertyTable>
</AdminDebugTooltip>
<LinkButton
variant={"minimal/small"}
LeadingIcon={BookOpenIcon}
@@ -67,6 +67,8 @@ import {
v3RunsPath,
} from "~/utils/pathBuilder";
import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const userId = await requireUserId(request);
@@ -116,6 +118,33 @@ export default function Page() {
title={`Run #${run.number}`}
/>
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
<Property label="ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
</div>
</Property>
<Property label="Trace ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
</div>
</Property>
<Property label="Env ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
</div>
</Property>
<Property label="Org ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">
{run.environment.organizationId}
</Paragraph>
</div>
</Property>
</PropertyTable>
</AdminDebugTooltip>
<EnvironmentLabel
size="large"
environment={run.environment}
@@ -165,6 +194,33 @@ export default function Page() {
title={`Run #${run.number}`}
/>
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
<Property label="ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
</div>
</Property>
<Property label="Trace ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
</div>
</Property>
<Property label="Env ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
</div>
</Property>
<Property label="Org ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">
{run.environment.organizationId}
</Paragraph>
</div>
</Property>
</PropertyTable>
</AdminDebugTooltip>
<EnvironmentLabel size="large" environment={run.environment} userName={usernameForEnv} />
</PageAccessories>
</NavBar>
@@ -4,6 +4,7 @@ import { Outlet, useLocation, useParams } from "@remix-run/react";
import { LoaderFunctionArgs } from "@remix-run/server-runtime";
import { typedjson, useTypedLoaderData } from "remix-typedjson";
import { BlankstateInstructions } from "~/components/BlankstateInstructions";
import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { InlineCode } from "~/components/code/InlineCode";
import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -12,6 +13,7 @@ import { DateTime } from "~/components/primitives/DateTime";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { PaginationControls } from "~/components/primitives/Pagination";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
import {
ResizableHandle,
ResizablePanel,
@@ -93,6 +95,18 @@ export default function Page() {
<NavBar>
<PageTitle title="Schedules" />
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
{schedules.map((schedule) => (
<Property label={schedule.friendlyId} key={schedule.id}>
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{schedule.id}</Paragraph>
</div>
</Property>
))}
</PropertyTable>
</AdminDebugTooltip>
<LinkButton
LeadingIcon={PlusIcon}
to={`${v3NewSchedulePath(organization, project)}${location.search}`}
@@ -3,6 +3,7 @@ import { parse } from "@conform-to/zod";
import { Form, useActionData, useNavigation } from "@remix-run/react";
import { ActionFunction, json } from "@remix-run/server-runtime";
import { z } from "zod";
import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { InlineCode } from "~/components/code/InlineCode";
import { PageBody, PageContainer } from "~/components/layout/AppLayout";
import { Button } from "~/components/primitives/Buttons";
@@ -14,7 +15,9 @@ import { Hint } from "~/components/primitives/Hint";
import { Input } from "~/components/primitives/Input";
import { InputGroup } from "~/components/primitives/InputGroup";
import { Label } from "~/components/primitives/Label";
import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
import { prisma } from "~/db.server";
import { useProject } from "~/hooks/useProject";
import { redirectWithSuccessMessage } from "~/models/message.server";
@@ -110,6 +113,23 @@ export default function Page() {
<PageContainer>
<NavBar>
<PageTitle title={`${project.name} project settings`} />
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
<Property label="ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{project.id}</Paragraph>
</div>
</Property>
<Property label="Org ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{project.organizationId}</Paragraph>
</div>
</Property>
</PropertyTable>
</AdminDebugTooltip>
</PageAccessories>
</NavBar>
<PageBody>
@@ -8,6 +8,7 @@ import { UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-type
import invariant from "tiny-invariant";
import { z } from "zod";
import { UserAvatar } from "~/components/UserProfilePhoto";
import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { PageBody, PageContainer } from "~/components/layout/AppLayout";
import {
Alert,
@@ -23,8 +24,9 @@ import { Button, ButtonContent, LinkButton } from "~/components/primitives/Butto
import { DateTime } from "~/components/primitives/DateTime";
import { Header2, Header3 } from "~/components/primitives/Headers";
import { NamedIcon } from "~/components/primitives/NamedIcon";
import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
import { SimpleTooltip } from "~/components/primitives/Tooltip";
import { useOrganization } from "~/hooks/useOrganizations";
import { useUser } from "~/hooks/useUser";
@@ -107,6 +109,28 @@ export default function Page() {
<PageContainer>
<NavBar>
<PageTitle title="Team" />
<PageAccessories>
<AdminDebugTooltip>
<PropertyTable>
<Property label="Org ID">
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">{organization.id}</Paragraph>
</div>
</Property>
{members.map((member) => (
<Property label={member.user.name} key={member.id}>
<div className="flex items-center gap-2">
<Paragraph variant="extra-small/bright/mono">
{member.user.email} - {member.user.id}
</Paragraph>
</div>
</Property>
))}
</PropertyTable>
</AdminDebugTooltip>
</PageAccessories>
</NavBar>
<PageBody>
<Header2>Members</Header2>
+6 -3
View File
@@ -15,9 +15,12 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
return redirect(confirmBasicDetailsPath());
}
return typedjson({
headers: [["Set-Cookie", await commitSession(await clearRedirectTo(request))]],
});
return typedjson(
{},
{
headers: { "Set-Cookie": await commitSession(await clearRedirectTo(request)) },
}
);
};
export default function App() {
+1 -5
View File
@@ -1,11 +1,9 @@
import { MagnifyingGlassIcon } from "@heroicons/react/20/solid";
import { Form } from "@remix-run/react";
import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
import { redirect } from "@remix-run/server-runtime";
import { typedjson, useTypedLoaderData } from "remix-typedjson";
import { z } from "zod";
import { Button, LinkButton } from "~/components/primitives/Buttons";
import { Header1 } from "~/components/primitives/Headers";
import { Input } from "~/components/primitives/Input";
import { PaginationControls } from "~/components/primitives/Pagination";
import { Paragraph } from "~/components/primitives/Paragraph";
@@ -18,10 +16,8 @@ import {
TableHeaderCell,
TableRow,
} from "~/components/primitives/Table";
import { useUser } from "~/hooks/useUser";
import { adminGetOrganizations, adminGetUsers, setV3Enabled } from "~/models/admin.server";
import { adminGetOrganizations, setV3Enabled } from "~/models/admin.server";
import { redirectWithSuccessMessage } from "~/models/message.server";
import { commitImpersonationSession, setImpersonationId } from "~/services/impersonation.server";
import { requireUserId } from "~/services/session.server";
import { createSearchParams } from "~/utils/searchParams";
@@ -63,6 +63,14 @@ export const action: ActionFunction = async ({ request, params }) => {
{ spanId: newRun.spanId }
);
logger.debug("Replayed run", {
taskRunId: taskRun.id,
taskRunFriendlyId: taskRun.friendlyId,
newRunId: newRun.id,
newRunFriendlyId: newRun.friendlyId,
runPath,
});
return redirectWithSuccessMessage(runPath, request, `Replaying run`);
} catch (error) {
if (error instanceof Error) {
+59 -15
View File
@@ -14,6 +14,7 @@ import {
correctErrorStackTrace,
createPacketAttributesAsJson,
flattenAttributes,
NULL_SENTINEL,
isExceptionSpanEvent,
omit,
unflattenAttributes,
@@ -441,21 +442,10 @@ export class EventRepository {
return;
}
const output = isEmptyJson(fullEvent.output)
? null
: unflattenAttributes(fullEvent.output as Attributes);
const output = rehydrateJson(fullEvent.output);
const payload = rehydrateJson(fullEvent.payload);
const payload = isEmptyJson(fullEvent.payload)
? null
: unflattenAttributes(fullEvent.payload as Attributes);
const show = unflattenAttributes(
filteredAttributes(fullEvent.properties as Attributes, SemanticInternalAttributes.SHOW)
)[SemanticInternalAttributes.SHOW] as
| {
actions?: boolean;
}
| undefined;
const show = rehydrateShow(fullEvent.properties);
const properties = sanitizedAttributes(fullEvent.properties);
@@ -1083,7 +1073,7 @@ function isEmptyJson(json: Prisma.JsonValue) {
return false;
}
function sanitizedAttributes(json: Prisma.JsonValue): Record<string, unknown> | undefined {
function sanitizedAttributes(json: Prisma.JsonValue) {
if (json === null || json === undefined) {
return;
}
@@ -1182,3 +1172,57 @@ function getNowInNanoseconds(): bigint {
function getDateFromNanoseconds(nanoseconds: bigint) {
return new Date(Number(nanoseconds) / 1_000_000);
}
function rehydrateJson(json: Prisma.JsonValue): any {
if (json === null) {
return undefined;
}
if (json === NULL_SENTINEL) {
return null;
}
if (typeof json === "string") {
return json;
}
if (typeof json === "number") {
return json;
}
if (typeof json === "boolean") {
return json;
}
if (Array.isArray(json)) {
return json.map((item) => rehydrateJson(item));
}
if (typeof json === "object") {
return unflattenAttributes(json as Attributes);
}
return null;
}
function rehydrateShow(properties: Prisma.JsonValue): { actions?: boolean } | undefined {
if (properties === null || properties === undefined) {
return;
}
if (typeof properties !== "object") {
return;
}
if (Array.isArray(properties)) {
return;
}
const actions = properties[SemanticInternalAttributes.SHOW_ACTIONS];
if (typeof actions === "boolean") {
return { actions };
}
return;
}
+26 -14
View File
@@ -19,6 +19,7 @@ import {
MarQSQueuePriorityStrategy,
MessagePayload,
QueueCapacities,
QueueRange,
} from "./types";
import { workerQueue } from "~/services/worker.server";
@@ -618,7 +619,7 @@ export class MarQS {
parentQueue
);
const queues = await this.#zrangeWithScores(parentQueue, range[0], range[1]);
const queues = await this.#getChildQueuesWithScores(parentQueue, range);
const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities);
@@ -629,21 +630,25 @@ export class MarQS {
selectionId
);
if (typeof choice !== "string") {
return;
}
span.setAttributes({
...flattenAttributes(queues, "marqs.queues"),
});
span.setAttributes({
...flattenAttributes(queuesWithScores, "marqs.queuesWithScores"),
});
span.setAttribute("marqs.nextRange", range);
span.setAttribute("marqs.queueCount", queues.length);
span.setAttribute("marqs.queueChoice", choice);
span.setAttribute("nextRange.offset", range.offset);
span.setAttribute("nextRange.count", range.count);
span.setAttribute("queueCount", queues.length);
return choice;
if (typeof choice !== "string") {
span.setAttribute("noQueueChoice", true);
return;
} else {
span.setAttribute("queueChoice", choice);
return choice;
}
},
{
kind: SpanKind.CONSUMER,
@@ -687,12 +692,19 @@ export class MarQS {
});
}
async #zrangeWithScores(
async #getChildQueuesWithScores(
key: string,
min: number,
max: number
range: QueueRange
): Promise<Array<{ value: string; score: number }>> {
const valuesWithScores = await this.redis.zrange(key, min, max, "WITHSCORES");
const valuesWithScores = await this.redis.zrangebyscore(
key,
"-inf",
Date.now(),
"WITHSCORES",
"LIMIT",
range.offset,
range.count
);
const result: Array<{ value: string; score: number }> = [];
for (let i = 0; i < valuesWithScores.length; i += 2) {
@@ -1591,7 +1603,7 @@ function getMarQSClient() {
return new MarQS({
keysProducer: new MarQSShortKeyProducer(KEY_PREFIX),
queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }),
queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }),
envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }),
workers: 1,
redis: redisOptions,
@@ -1,5 +1,10 @@
import { RedisOptions } from "ioredis";
import { MarQSQueuePriorityStrategy, PriorityStrategyChoice, QueueWithScores } from "./types";
import {
MarQSQueuePriorityStrategy,
PriorityStrategyChoice,
QueueRange,
QueueWithScores,
} from "./types";
import { nanoid } from "nanoid";
import seedrandom from "seedrandom";
@@ -26,9 +31,7 @@ export class DynamicWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
throw new Error("Method not implemented.");
}
nextCandidateSelection(
parentQueue: string
): Promise<{ range: [number, number]; selectionId: string }> {
nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }> {
throw new Error("Method not implemented.");
}
}
@@ -39,13 +42,18 @@ export type SimpleWeightedChoiceStrategyOptions = {
};
export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy {
private _nextRangesByParentQueue: Map<string, [number, number]> = new Map();
private _nextRangesByParentQueue: Map<string, QueueRange> = new Map();
private _randomGenerator = seedrandom(this.options.randomSeed);
constructor(private options: SimpleWeightedChoiceStrategyOptions) {}
private nextRangeForParentQueue(parentQueue: string) {
return this._nextRangesByParentQueue.get(parentQueue) ?? [0, this.options.queueSelectionCount];
private nextRangeForParentQueue(parentQueue: string): QueueRange {
return (
this._nextRangesByParentQueue.get(parentQueue) ?? {
offset: 0,
count: this.options.queueSelectionCount,
}
);
}
chooseQueue(
@@ -55,25 +63,23 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
): PriorityStrategyChoice {
const filteredQueues = filterQueuesAtCapacity(queues);
if (filteredQueues.length === 0) {
if (queues.length === this.options.queueSelectionCount) {
const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
const nextRange: [number, number] = nextRangeForParentQueue
? [
nextRangeForParentQueue[1],
nextRangeForParentQueue[1] + this.options.queueSelectionCount,
]
: [this.options.queueSelectionCount, this.options.queueSelectionCount * 2];
// If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
this._nextRangesByParentQueue.set(parentQueue, nextRange);
} else {
this._nextRangesByParentQueue.delete(parentQueue);
}
return { abort: true };
if (queues.length === this.options.queueSelectionCount) {
const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
const nextRange: QueueRange = nextRangeForParentQueue
? {
offset: nextRangeForParentQueue.offset + this.options.queueSelectionCount,
count: this.options.queueSelectionCount,
}
: { offset: this.options.queueSelectionCount, count: this.options.queueSelectionCount };
// If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
this._nextRangesByParentQueue.set(parentQueue, nextRange);
} else {
this._nextRangesByParentQueue.delete(parentQueue);
}
this._nextRangesByParentQueue.delete(parentQueue);
if (filteredQueues.length === 0) {
return { abort: true };
}
const queueWeights = this.#calculateQueueWeights(filteredQueues);
@@ -82,7 +88,7 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
async nextCandidateSelection(
parentQueue: string
): Promise<{ range: [number, number]; selectionId: string }> {
): Promise<{ range: QueueRange; selectionId: string }> {
return { range: this.nextRangeForParentQueue(parentQueue), selectionId: nanoid(24) };
}
@@ -28,7 +28,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
import { socketIo } from "../handleSocketIo.server";
import { findCurrentWorkerDeployment } from "../models/workerDeployment.server";
import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
import { tracer } from "../tracer.server";
import { SEMINTATTRS_FORCE_RECORDING, tracer } from "../tracer.server";
import { CrashTaskRunService } from "../services/crashTaskRun.server";
import { FailedTaskRunService } from "../failedTaskRun.server";
import { CreateTaskRunAttemptService } from "../services/createTaskRunAttempt.server";
@@ -71,7 +71,6 @@ export type SharedQueueConsumerOptions = {
traceTimeoutSeconds?: number;
nextTickInterval?: number;
interval?: number;
parentContext?: Context;
};
export class SharedQueueConsumer {
@@ -97,7 +96,6 @@ export class SharedQueueConsumer {
traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds
nextTickInterval: options.nextTickInterval ?? 1000, // 1 second
interval: options.interval ?? 100, // 100ms
parentContext: options.parentContext ?? ROOT_CONTEXT,
};
}
@@ -197,19 +195,17 @@ export class SharedQueueConsumer {
) {
this.#endCurrentSpan();
const parentContext = this._options.parentContext ?? ROOT_CONTEXT;
// Create a new trace
this._currentSpan = tracer.startSpan(
"SharedQueueConsumer.doWork()",
{
kind: SpanKind.CONSUMER,
},
parentContext
ROOT_CONTEXT
);
// Get the span trace context
this._currentSpanContext = trace.setSpan(parentContext, this._currentSpan);
this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan);
this._perTraceCountdown = this._options.maximumItemsPerTrace;
this._lastNewTrace = new Date();
+3 -3
View File
@@ -18,6 +18,8 @@ export type QueueWithScores = {
age: number;
};
export type QueueRange = { offset: number; count: number };
export interface MarQSKeyProducer {
queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string;
envConcurrencyLimitKey(env: AuthenticatedEnvironment): string;
@@ -69,9 +71,7 @@ export interface MarQSQueuePriorityStrategy {
*
* @returns The scores and the selectionId for the next candidate selection
*/
nextCandidateSelection(
parentQueue: string
): Promise<{ range: [number, number]; selectionId: string }>;
nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }>;
}
export const MessagePayload = z.object({
+6 -24
View File
@@ -1,19 +1,17 @@
import { trace } from "@opentelemetry/api";
import { clientWebsocketMessages, serverWebsocketMessages } from "@trigger.dev/core/v3";
import type { StructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
import {
MessageCatalogToSocketIoEvents,
ZodMessageHandler,
ZodMessageSender,
MessageCatalogToSocketIoEvents,
} from "@trigger.dev/core/v3/zodMessageHandler";
import type { StructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
import { Evt } from "evt";
import { randomUUID } from "node:crypto";
import type { DisconnectReason, Namespace, Socket } from "socket.io";
import { env } from "~/env.server";
import { logger } from "~/services/logger.server";
import { SharedQueueConsumer } from "./marqs/sharedQueueConsumer.server";
import type { DisconnectReason, Namespace, Socket } from "socket.io";
import { ROOT_CONTEXT, Span, SpanKind, trace } from "@opentelemetry/api";
import { env } from "~/env.server";
const tracer = trace.getTracer("sharedQueueConsumerPool");
interface SharedQueueConsumerPoolOptions {
sender: ZodMessageSender<typeof serverWebsocketMessages>;
@@ -22,22 +20,8 @@ interface SharedQueueConsumerPoolOptions {
class SharedQueueConsumerPool {
#consumers: SharedQueueConsumer[];
#span: Span;
constructor(opts: SharedQueueConsumerPoolOptions) {
this.#span = tracer.startSpan(
"SharedQueueConsumerPool()",
{
kind: SpanKind.CONSUMER,
attributes: {
"pool.size": opts.poolSize,
},
},
ROOT_CONTEXT
);
const spanContext = trace.setSpan(ROOT_CONTEXT, this.#span);
this.#consumers = Array(opts.poolSize)
.fill(null)
.map(
@@ -45,7 +29,6 @@ class SharedQueueConsumerPool {
new SharedQueueConsumer(opts.sender, {
interval: env.SHARED_QUEUE_CONSUMER_INTERVAL_MS,
nextTickInterval: env.SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS,
parentContext: spanContext,
})
);
}
@@ -56,7 +39,6 @@ class SharedQueueConsumerPool {
async stop() {
await Promise.allSettled(this.#consumers.map((consumer) => consumer.stop()));
this.#span.end();
}
}
@@ -100,7 +82,7 @@ export class SharedSocketConnection {
},
});
logger.log("Starting SharedQueueConsumer pool", {
logger.debug("Starting SharedQueueConsumer pool", {
poolSize: opts.poolSize ?? this._defaultPoolSize,
});
+1 -1
View File
@@ -125,7 +125,7 @@ function getTracer() {
);
} else {
if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
console.log(`🔦 Tracer: Logger exporter enabled`);
console.log(`🔦 Tracer: Logger exporter enabled (sampling = ${samplingRate})`);
const loggerExporter = new LoggerSpanExporter();
+64 -11
View File
@@ -88,7 +88,10 @@ describe("SimpleWeightedChoiceStrategy", () => {
const nextSelection = await stategy.nextCandidateSelection("parentQueue");
expect(nextSelection).toEqual({ range: [3, 6], selectionId: expect.any(String) });
expect(nextSelection).toEqual({
range: { offset: 3, count: 3 },
selectionId: expect.any(String),
});
// Now pass some queues that have some capacity
const chosenQueue2 = stategy.chooseQueue(
@@ -129,7 +132,10 @@ describe("SimpleWeightedChoiceStrategy", () => {
const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");
expect(nextSelection2).toEqual({ range: [0, 3], selectionId: expect.any(String) });
expect(nextSelection2).toEqual({
range: { offset: 6, count: 3 },
selectionId: expect.any(String),
});
});
it("should adjust the next filter range only if passed the maximum number of queues", async () => {
@@ -167,7 +173,10 @@ describe("SimpleWeightedChoiceStrategy", () => {
const nextSelection = await stategy.nextCandidateSelection("parentQueue");
expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
expect(nextSelection).toEqual({
range: { offset: 0, count: 3 },
selectionId: expect.any(String),
});
});
it("should adjust the next candidate range ONLY for the matching parent queue", async () => {
@@ -182,7 +191,7 @@ describe("SimpleWeightedChoiceStrategy", () => {
queue: "queue1",
age: 4497,
capacities: {
queue: { current: 10, limit: 10 },
queue: { current: 0, limit: 10 },
env: { current: 0, limit: 10 },
org: { current: 0, limit: 10 },
},
@@ -210,15 +219,21 @@ describe("SimpleWeightedChoiceStrategy", () => {
"selectionId"
);
expect(chosenQueue).toEqual({ abort: true });
expect(chosenQueue).toEqual("queue1");
const nextSelection = await stategy.nextCandidateSelection("parentQueue2");
expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
expect(nextSelection).toEqual({
range: { offset: 0, count: 3 },
selectionId: expect.any(String),
});
const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");
expect(nextSelection2).toEqual({ range: [3, 6], selectionId: expect.any(String) });
expect(nextSelection2).toEqual({
range: { offset: 3, count: 3 },
selectionId: expect.any(String),
});
const chosenQueue2 = stategy.chooseQueue(
[
@@ -250,14 +265,52 @@ describe("SimpleWeightedChoiceStrategy", () => {
},
},
],
"parentQueue2",
"parentQueue",
"selectionId"
);
expect(chosenQueue2).toEqual("queue3");
expect(chosenQueue2).toEqual("queue2");
const nextSelection3 = await stategy.nextCandidateSelection("parentQueue2");
const nextSelection3 = await stategy.nextCandidateSelection("parentQueue");
expect(nextSelection3).toEqual({ range: [0, 3], selectionId: expect.any(String) });
expect(nextSelection3).toEqual({
range: { offset: 6, count: 3 },
selectionId: expect.any(String),
});
// Not passed 3 queues, so the range should be reset (we've reached the end)
const chosenQueue3 = stategy.chooseQueue(
[
{
queue: "queue1",
age: 4497,
capacities: {
queue: { current: 0, limit: 10 },
env: { current: 0, limit: 10 },
org: { current: 0, limit: 10 },
},
},
{
queue: "queue2",
age: 19670,
capacities: {
queue: { current: 0, limit: 10 },
env: { current: 0, limit: 10 },
org: { current: 0, limit: 10 },
},
},
],
"parentQueue",
"selectionId"
);
expect(chosenQueue3).toEqual("queue2");
const nextSelection4 = await stategy.nextCandidateSelection("parentQueue");
expect(nextSelection4).toEqual({
range: { offset: 0, count: 3 },
selectionId: expect.any(String),
});
});
});
+7 -1
View File
@@ -29,6 +29,8 @@ This task will retry 10 times with exponential backoff.
One way to gain reliability is to break your work into smaller tasks and [trigger](/v3/triggering) them from each other. Each task can have its own retrying behavior:
```ts /trigger/multiple-tasks.ts
import { task } from "@trigger.dev/sdk/v3";
export const myTask = task({
id: "my-task",
retry: {
@@ -66,7 +68,7 @@ We provide some useful functions that you can use to retry smaller parts of a ta
You can retry a block of code that can throw an error, with the same retry settings as a task.
```ts /trigger/retry-on-throw.ts
import { task, logger, retry } from "@trigger.dev/sdk/v3"
import { task, logger, retry } from "@trigger.dev/sdk/v3";
export const retryOnThrow = task({
id: "retry-on-throw",
@@ -102,6 +104,8 @@ You can use `fetch`, `axios`, or any other library in your code.
But we do provide a convenient function to perform HTTP requests with conditional retrying based on the response:
```ts /trigger/retry-fetch.ts
import { task, logger, retry } from "@trigger.dev/sdk/v3";
export const taskWithFetchRetries = task({
id: "task-with-fetch-retries",
run: async ({ payload, ctx }) => {
@@ -195,6 +199,8 @@ In this complicated example:
- If we've run out of requests or tokens we retry at the time specified in the headers.
```ts
import { task } from "@trigger.dev/sdk/v3";
export const openaiTask = task({
id: "openai-task",
retry: {
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/airtable
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/airtable",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for airtable",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"airtable": "^0.12.1",
"zod": "3.22.3"
},
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/github
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/github",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The official GitHub integration for Trigger.dev",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -30,8 +30,8 @@
"@octokit/request-error": "^5.0.1",
"@octokit/webhooks": "^12.0.10",
"octokit": "^3.1.2",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"zod": "3.22.3"
},
"engines": {
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/linear
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/linear",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for @linear/sdk",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
},
"dependencies": {
"@linear/sdk": "^8.0.0",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"zod": "3.22.3"
},
"engines": {
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/slack
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/openai",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The official OpenAI integration for Trigger.dev",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -42,8 +42,8 @@
},
"dependencies": {
"openai": "^4.16.1",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25"
},
"engines": {
"node": ">=18.0.0"
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/plain
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/plain",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The official Plain.com integration for Trigger.dev",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
"build:tsup": "tsup"
},
"dependencies": {
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@team-plain/typescript-sdk": "^2.7.0"
},
"engines": {
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/replicate
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/replicate",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for replicate",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"replicate": "^0.18.1",
"zod": "3.22.3"
},
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/resend
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/resend",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The official Resend.com integration for Trigger.dev",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
"build:tsup": "tsup"
},
"dependencies": {
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"resend": "^2.1.0"
},
"engines": {
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/sendgrid
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/sendgrid",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for @sendgrid/mail",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
},
"dependencies": {
"@sendgrid/mail": "^7.7.0",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25"
},
"engines": {
"node": ">=16.8.0"
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/shopify
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/shopify",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for @shopify/shopify-api",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
},
"dependencies": {
"@shopify/shopify-api": "^8.0.2",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"zod": "3.22.3"
},
"engines": {
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/slack
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/slack",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The official Slack integration for Trigger.dev",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -25,7 +25,7 @@
},
"dependencies": {
"@slack/web-api": "^6.8.1",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"zod": "3.22.3"
},
"engines": {
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/stripe
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/stripe",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for stripe",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"stripe": "^12.14.0",
"zod": "3.22.3"
},
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/supabase
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/supabase",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev integration for @supabase/supabase-js",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
},
"dependencies": {
"@supabase/supabase-js": "^2.26.0",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"supabase-management-js": "^1.0.0",
"zod": "3.22.3"
},
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/typeform
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/integration-kit@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/typeform",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The official Typeform integration for Trigger.dev",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@typeform/api-client": "^1.8.0",
"zod": "3.22.3"
},
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/astro
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "@trigger.dev/astro",
"description": "An Astro-native integration for Trigger.dev background jobs platform",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"files": [
@@ -20,7 +20,7 @@
"build:tsup": "tsup"
},
"peerDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25"
},
"devDependencies": {
"astro": "^3.0.12",
+18
View File
@@ -1,5 +1,23 @@
# trigger.dev
## 3.0.0-beta.25
### Patch Changes
- e337b2165: Add a postInstall option to allow running scripts after dependencies have been installed in deployed images
- c37c82231: Use locked package versions when resolving dependencies in deployed workers
- Updated dependencies [e337b2165]
- Updated dependencies [9e5382951]
- @trigger.dev/core@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- 83dc87155: Fix issues with consecutive waits
- Updated dependencies [83dc87155]
- @trigger.dev/core@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "trigger.dev",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "A Command-Line Interface for Trigger.dev (v3) projects",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@@ -86,7 +86,7 @@
"@opentelemetry/sdk-trace-base": "^1.22.0",
"@opentelemetry/sdk-trace-node": "^1.22.0",
"@opentelemetry/semantic-conventions": "^1.22.0",
"@trigger.dev/core": "workspace:3.0.0-beta.23",
"@trigger.dev/core": "workspace:3.0.0-beta.25",
"@types/degit": "^2.8.3",
"chalk": "^5.2.0",
"chokidar": "^3.5.3",
+2
View File
@@ -19,6 +19,8 @@ COPY --chown=node:node . .
USER node
RUN npm ci --no-fund --no-audit && npm cache clean --force
__POST_INSTALL__
# Development or production stage builds upon the base stage
FROM base AS final
+31 -11
View File
@@ -1286,6 +1286,8 @@ async function compileProject(
const dependencies = await gatherRequiredDependencies(allImports, config, javascriptProject);
logger.debug("gatherRequiredDependencies()", { dependencies });
const packageJsonContents = {
name: "trigger-worker",
version: "0.0.0",
@@ -1328,8 +1330,19 @@ async function compileProject(
// Write the Containerfile to /tmp/dir/Containerfile
const containerFilePath = join(cliRootPath(), "Containerfile.prod");
// Copy the Containerfile to /tmp/dir/Containerfile
await copyFile(containerFilePath, join(tempDir, "Containerfile"));
let containerFileContents = readFileSync(containerFilePath, "utf-8");
if (config.postInstall) {
containerFileContents = containerFileContents.replace(
"__POST_INSTALL__",
`RUN ${config.postInstall}`
);
} else {
containerFileContents = containerFileContents.replace("__POST_INSTALL__", "");
}
await writeFile(join(tempDir, "Containerfile"), containerFileContents);
const contentHasher = createHash("sha256");
contentHasher.update(Buffer.from(entryPointOutputFile.text));
@@ -1539,6 +1552,7 @@ async function gatherRequiredDependencies(
project: JavascriptProject
) {
const dependencies: Record<string, string> = {};
const resolvablePackageNames = new Set<string>();
for (const file of imports) {
if ((file.kind !== "require-call" && file.kind !== "dynamic-import") || !file.external) {
@@ -1547,26 +1561,32 @@ async function gatherRequiredDependencies(
const packageName = detectPackageNameFromImportPath(file.path);
if (dependencies[packageName]) {
if (!packageName) {
continue;
}
const externalDependencyVersion = await project.resolve(packageName);
resolvablePackageNames.add(packageName);
}
if (externalDependencyVersion) {
dependencies[packageName] = stripWorkspaceFromVersion(externalDependencyVersion);
continue;
}
const resolvedPackageVersions = await project.resolveAll(Array.from(resolvablePackageNames));
const missingPackages = Array.from(resolvablePackageNames).filter(
(packageName) => !resolvedPackageVersions[packageName]
);
for (const missingPackage of missingPackages) {
const internalDependencyVersion =
(packageJson.dependencies as Record<string, string>)[packageName] ??
detectDependencyVersion(packageName);
(packageJson.dependencies as Record<string, string>)[missingPackage] ??
detectDependencyVersion(missingPackage);
if (internalDependencyVersion) {
dependencies[packageName] = stripWorkspaceFromVersion(internalDependencyVersion);
dependencies[missingPackage] = stripWorkspaceFromVersion(internalDependencyVersion);
}
}
for (const [packageName, version] of Object.entries(resolvedPackageVersions)) {
dependencies[packageName] = version;
}
if (config.additionalPackages) {
for (const packageName of config.additionalPackages) {
if (dependencies[packageName]) {
@@ -5,51 +5,10 @@ import { logger } from "./logger";
import { PackageManager, getUserPackageManager } from "./getUserPackageManager";
import { PackageJson } from "type-fest";
import { assertExhaustive } from "./assertExhaustive";
import { builtinModules } from "node:module";
export type ResolveOptions = { allowDev: boolean };
const BuiltInModules = new Set([
"assert",
"async_hooks",
"buffer",
"child_process",
"cluster",
"console",
"constants",
"crypto",
"dgram",
"dns",
"domain",
"events",
"fs",
"http",
"http2",
"https",
"inspector",
"module",
"net",
"os",
"path",
"perf_hooks",
"process",
"punycode",
"querystring",
"readline",
"repl",
"stream",
"string_decoder",
"timers",
"tls",
"trace_events",
"tty",
"url",
"util",
"v8",
"vm",
"worker_threads",
"zlib",
]);
export class JavascriptProject {
private _packageJson?: PackageJson;
private _packageManager?: PackageManager;
@@ -84,27 +43,73 @@ export class JavascriptProject {
}
}
async resolveAll(
packageNames: string[],
options?: ResolveOptions
): Promise<Record<string, string>> {
const externalPackages = packageNames.filter((packageName) => !isBuiltInModule(packageName));
const opts = { allowDev: false, ...options };
const command = await this.#getCommand();
try {
const versions = await command.resolveDependencyVersions(externalPackages, {
cwd: this.projectPath,
});
if (versions) {
logger.debug(`Resolved [${externalPackages.join(", ")}] version using ${command.name}`, {
versions,
});
}
// Merge the resolved versions with the package.json dependencies
const missingPackages = externalPackages.filter((packageName) => !versions[packageName]);
const missingPackageVersions: Record<string, string> = {};
for (const packageName of missingPackages) {
const packageJsonVersion = this.packageJson.dependencies?.[packageName];
if (typeof packageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using package.json`, {
packageJsonVersion,
});
missingPackageVersions[packageName] = packageJsonVersion;
}
if (opts.allowDev) {
const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
if (typeof devPackageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using devDependencies`, {
devPackageJsonVersion,
});
missingPackageVersions[packageName] = devPackageJsonVersion;
}
}
}
return { ...versions, ...missingPackageVersions };
} catch (error) {
logger.debug(`Failed to resolve dependency versions using ${command.name}`, {
packageNames,
error,
});
return {};
}
}
async resolve(packageName: string, options?: ResolveOptions): Promise<string | undefined> {
if (BuiltInModules.has(packageName)) {
if (isBuiltInModule(packageName)) {
return undefined;
}
const opts = { allowDev: false, ...options };
const packageJsonVersion = this.packageJson.dependencies?.[packageName];
if (typeof packageJsonVersion === "string") {
return packageJsonVersion;
}
if (opts.allowDev) {
const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
if (typeof devPackageJsonVersion === "string") {
return devPackageJsonVersion;
}
}
const command = await this.#getCommand();
try {
@@ -113,8 +118,30 @@ export class JavascriptProject {
});
if (version) {
logger.debug(`Resolved ${packageName} version using ${command.name}`, { version });
return version;
}
const packageJsonVersion = this.packageJson.dependencies?.[packageName];
if (typeof packageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using package.json`, { packageJsonVersion });
return packageJsonVersion;
}
if (opts.allowDev) {
const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
if (typeof devPackageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using devDependencies`, {
devPackageJsonVersion,
});
return devPackageJsonVersion;
}
}
} catch (error) {
logger.debug(`Failed to resolve dependency version using ${command.name}`, {
packageName,
@@ -176,6 +203,11 @@ interface PackageManagerCommands {
packageName: string,
options: PackageManagerOptions
): Promise<string | undefined>;
resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>>;
}
class PNPMCommands implements PackageManagerCommands {
@@ -197,7 +229,7 @@ class PNPMCommands implements PackageManagerCommands {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageName} -r --json`;
const result = JSON.parse(stdout) as PnpmList;
logger.debug(`Resolving ${packageName} version using ${this.name}`, { result });
logger.debug(`Resolving ${packageName} version using ${this.name}`);
// Return the first dependency version that matches the package name
for (const dep of result) {
@@ -208,6 +240,31 @@ class PNPMCommands implements PackageManagerCommands {
}
}
}
async resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>> {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageNames} -r --json`;
const result = JSON.parse(stdout) as PnpmList;
logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
const results: Record<string, string> = {};
// Return the first dependency version that matches the package name
for (const dep of result) {
for (const packageName of packageNames) {
const dependency = dep.dependencies?.[packageName];
if (dependency) {
results[packageName] = dependency.version;
}
}
}
return results;
}
}
type NpmDependency = {
@@ -246,6 +303,28 @@ class NPMCommands implements PackageManagerCommands {
return this.#recursivelySearchDependencies(output.dependencies, packageName);
}
async resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>> {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageNames} --json`;
const output = JSON.parse(stdout) as NpmListOutput;
logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`, { output });
const results: Record<string, string> = {};
for (const packageName of packageNames) {
const version = this.#recursivelySearchDependencies(output.dependencies, packageName);
if (version) {
results[packageName] = version;
}
}
return results;
}
#recursivelySearchDependencies(
dependencies: Record<string, NpmDependency>,
packageName: string
@@ -286,7 +365,7 @@ class YarnCommands implements PackageManagerCommands {
const lines = stdout.split("\n");
logger.debug(`Resolving ${packageName} version using ${this.name}`, { lines });
logger.debug(`Resolving ${packageName} version using ${this.name}`);
for (const line of lines) {
const json = JSON.parse(line);
@@ -296,4 +375,54 @@ class YarnCommands implements PackageManagerCommands {
}
}
}
async resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>> {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} info ${packageNames} --json`;
const lines = stdout.split("\n");
logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
const results: Record<string, string> = {};
for (const line of lines) {
const json = JSON.parse(line);
const packageName = this.#parseYarnValueIntoPackageName(json.value);
if (packageNames.includes(packageName)) {
results[packageName] = json.children.Version;
}
}
return results;
}
// The "value" when doing yarn info is formatted like this:
// "package-name@npm:version" or "package-name@workspace:version"
// This function will parse the value into just the package name.
// This correctly handles scoped packages as well e.g. @scope/package-name@npm:version
#parseYarnValueIntoPackageName(value: string): string {
const parts = value.split("@");
// If the value does not contain an "@" symbol, then it's just the package name
if (parts.length === 3) {
return parts[1] as string;
}
// If the value contains an "@" symbol, then the package name is the first part
return parts[0] as string;
}
}
function isBuiltInModule(module: string): boolean {
// if the module has node: prefix, it's a built-in module
if (module.startsWith("node:")) {
return true;
}
return builtinModules.includes(module);
}
@@ -91,7 +91,7 @@ class ProdWorker {
// Currently, this is only used for duration waits. Might need adjusting for other use cases.
this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
logger.log("onCancelCheckpoint()", { message });
logger.log("onCancelCheckpoint", { message });
const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
"CANCEL_CHECKPOINT",
+17
View File
@@ -1,5 +1,22 @@
# create-trigger
## 3.0.0-beta.25
### Patch Changes
- Updated dependencies [e337b2165]
- Updated dependencies [9e5382951]
- @trigger.dev/core@3.0.0-beta.25
- @trigger.dev/yalt@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- Updated dependencies [83dc87155]
- @trigger.dev/core@3.0.0-beta.24
- @trigger.dev/yalt@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/cli",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "The Trigger.dev CLI",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
+4
View File
@@ -1,5 +1,9 @@
# @trigger.dev/core-apps
## 3.0.0-beta.25
## 3.0.0-beta.24
## 3.0.0-beta.23
## 3.0.0-beta.22
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@trigger.dev/core-apps",
"description": "Backend core code used across apps",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"private": true,
"license": "MIT",
"main": "./dist/index.js",
+4
View File
@@ -1,5 +1,9 @@
# @trigger.dev/core-backend
## 3.0.0-beta.25
## 3.0.0-beta.24
## 3.0.0-beta.23
## 3.0.0-beta.22
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/core-backend",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Core code used across `@trigger.dev/sdk` and Trigger.dev server",
"license": "MIT",
"main": "./dist/index.js",
+13
View File
@@ -1,5 +1,18 @@
# internal-platform
## 3.0.0-beta.25
### Patch Changes
- e337b2165: Add a postInstall option to allow running scripts after dependencies have been installed in deployed images
- 9e5382951: Improve the display of non-object return types in the run trace viewer
## 3.0.0-beta.24
### Patch Changes
- 83dc87155: Fix issues with consecutive waits
## 3.0.0-beta.23
## 3.0.0-beta.22
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/core",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Core code used across the Trigger.dev SDK and platform",
"license": "MIT",
"main": "./dist/index.js",
+1
View File
@@ -28,6 +28,7 @@ export {
flattenAttributes,
primitiveValueOrflattenedAttributes,
unflattenAttributes,
NULL_SENTINEL,
} from "./utils/flattenAttributes";
export { omit } from "./utils/omit";
export {
+1
View File
@@ -212,6 +212,7 @@ export const Config = z.object({
dependenciesToBundle: z.array(z.union([z.string(), RegexSchema])).optional(),
logLevel: z.string().optional(),
enableConsoleLogging: z.boolean().optional(),
postInstall: z.string().optional(),
});
export type Config = z.infer<typeof Config>;
+7
View File
@@ -69,4 +69,11 @@ export interface ProjectConfig {
* onStart is called the first time a task is executed in a run (not before every retry)
*/
onStart?: (payload: unknown, params: StartFnParams) => Promise<void>;
/**
* postInstall will run during the deploy build step, after all the dependencies have been installed.
*
* @example "prisma generate"
*/
postInstall?: string;
}
+64 -29
View File
@@ -1,5 +1,7 @@
import { Attributes } from "@opentelemetry/api";
export const NULL_SENTINEL = "$@null((";
export function flattenAttributes(
obj: Record<string, unknown> | Array<unknown> | string | boolean | number | null | undefined,
prefix?: string
@@ -7,7 +9,12 @@ export function flattenAttributes(
const result: Attributes = {};
// Check if obj is null or undefined
if (!obj) {
if (obj === undefined) {
return result;
}
if (obj === null) {
result[prefix || ""] = NULL_SENTINEL;
return result;
}
@@ -27,14 +34,18 @@ export function flattenAttributes(
}
for (const [key, value] of Object.entries(obj)) {
const newPrefix = `${prefix ? `${prefix}.` : ""}${key}`;
const newPrefix = `${prefix ? `${prefix}.` : ""}${Array.isArray(obj) ? `[${key}]` : key}`;
if (Array.isArray(value)) {
for (let i = 0; i < value.length; i++) {
if (typeof value[i] === "object" && value[i] !== null) {
// update null check here as well
Object.assign(result, flattenAttributes(value[i], `${newPrefix}.[${i}]`));
} else {
result[`${newPrefix}.[${i}]`] = value[i];
if (value[i] === null) {
result[`${newPrefix}.[${i}]`] = NULL_SENTINEL;
} else {
result[`${newPrefix}.[${i}]`] = value[i];
}
}
}
} else if (isRecord(value)) {
@@ -43,6 +54,8 @@ export function flattenAttributes(
} else {
if (typeof value === "number" || typeof value === "string" || typeof value === "boolean") {
result[newPrefix] = value;
} else if (value === null) {
result[newPrefix] = NULL_SENTINEL;
}
}
}
@@ -54,55 +67,69 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
export function unflattenAttributes(obj: Attributes): Record<string, unknown> {
export function unflattenAttributes(
obj: Attributes
): Record<string, unknown> | string | number | boolean | null | undefined {
if (typeof obj !== "object" || obj === null || Array.isArray(obj)) {
return obj;
}
if (
typeof obj === "object" &&
obj !== null &&
Object.keys(obj).length === 1 &&
Object.keys(obj)[0] === ""
) {
return rehydrateNull(obj[""]) as any;
}
if (Object.keys(obj).length === 0) {
return;
}
const result: Record<string, unknown> = {};
for (const [key, value] of Object.entries(obj)) {
const parts = key.split(".").reduce((acc, part) => {
// Splitting array indices as separate parts
if (detectIsArrayIndex(part)) {
acc.push(part);
if (part.includes("[")) {
// Handling nested array indices
const subparts = part.split(/\[|\]/).filter((p) => p !== "");
acc.push(...subparts);
} else {
acc.push(...part.split(/\.\[(.*?)\]/).filter(Boolean));
acc.push(part);
}
return acc;
}, [] as string[]);
let current: Record<string, unknown> = result;
let current: any = result;
for (let i = 0; i < parts.length - 1; i++) {
const part = parts[i];
const isArray = detectIsArrayIndex(part);
const cleanPart = isArray ? part.substring(1, part.length - 1) : part;
const nextIsArray = detectIsArrayIndex(parts[i + 1]);
if (!current[cleanPart]) {
current[cleanPart] = nextIsArray ? [] : {};
const nextPart = parts[i + 1];
const isArray = /^\d+$/.test(nextPart);
if (isArray && !Array.isArray(current[part])) {
current[part] = [];
} else if (!isArray && current[part] === undefined) {
current[part] = {};
}
current = current[cleanPart] as Record<string, unknown>;
current = current[part];
}
const lastPart = parts[parts.length - 1];
const cleanLastPart = detectIsArrayIndex(lastPart)
? parseInt(lastPart.substring(1, lastPart.length - 1), 10)
: lastPart;
current[cleanLastPart] = value;
current[lastPart] = rehydrateNull(value);
}
// Convert the result to an array if all top-level keys are numeric indices
if (Object.keys(result).every((k) => /^\d+$/.test(k))) {
const maxIndex = Math.max(...Object.keys(result).map((k) => parseInt(k)));
const arrayResult = Array(maxIndex + 1);
for (const key in result) {
arrayResult[parseInt(key)] = result[key];
}
return arrayResult as any;
}
return result;
}
function detectIsArrayIndex(key: string): boolean {
const match = key.match(/^\[(\d+)\]$/);
if (match) {
return true;
}
return false;
}
export function primitiveValueOrflattenedAttributes(
obj: Record<string, unknown> | Array<unknown> | string | boolean | number | undefined,
prefix: string | undefined
@@ -129,3 +156,11 @@ export function primitiveValueOrflattenedAttributes(
return attributes;
}
function rehydrateNull(value: any): any {
if (value === NULL_SENTINEL) {
return null;
}
return value;
}
@@ -216,11 +216,13 @@ export async function createPacketAttributes(
const parsed = parse(packet.data) as any;
const jsonified = JSON.parse(JSON.stringify(parsed, safeReplacer));
return {
const result = {
...flattenAttributes(jsonified, dataKey),
[dataTypeKey]: "application/json",
};
} catch {
return result;
} catch (e) {
return;
}
+29 -3
View File
@@ -1,24 +1,50 @@
import { flattenAttributes, unflattenAttributes } from "../src/v3/utils/flattenAttributes";
describe("flattenAttributes", () => {
it("handles null and undefined gracefully", () => {
expect(flattenAttributes(null)).toEqual({});
expect(flattenAttributes(undefined)).toEqual({});
it("handles null correctly", () => {
expect(flattenAttributes(null)).toEqual({ "": "$@null((" });
expect(unflattenAttributes({ "": "$@null((" })).toEqual(null);
expect(flattenAttributes(null, "$output")).toEqual({ $output: "$@null((" });
expect(flattenAttributes({ foo: null })).toEqual({ foo: "$@null((" });
expect(unflattenAttributes({ foo: "$@null((" })).toEqual({ foo: null });
expect(flattenAttributes({ foo: [null] })).toEqual({ "foo.[0]": "$@null((" });
expect(unflattenAttributes({ "foo.[0]": "$@null((" })).toEqual({ foo: [null] });
expect(flattenAttributes([null])).toEqual({ "[0]": "$@null((" });
expect(unflattenAttributes({ "[0]": "$@null((" })).toEqual([null]);
});
it("flattens string attributes correctly", () => {
const result = flattenAttributes("testString");
expect(result).toEqual({ "": "testString" });
expect(unflattenAttributes(result)).toEqual("testString");
});
it("flattens number attributes correctly", () => {
const result = flattenAttributes(12345);
expect(result).toEqual({ "": 12345 });
expect(unflattenAttributes(result)).toEqual(12345);
});
it("flattens boolean attributes correctly", () => {
const result = flattenAttributes(true);
expect(result).toEqual({ "": true });
expect(unflattenAttributes(result)).toEqual(true);
});
it("flattens boolean attributes correctly", () => {
const result = flattenAttributes(true, "$output");
expect(result).toEqual({ $output: true });
expect(unflattenAttributes(result)).toEqual({ $output: true });
});
it("flattens array attributes correctly", () => {
const input = [1, 2, 3];
const result = flattenAttributes(input);
expect(result).toEqual({ "[0]": 1, "[1]": 2, "[2]": 3 });
expect(unflattenAttributes(result)).toEqual(input);
});
it("flattens complex objects correctly", () => {
+4
View File
@@ -1,5 +1,9 @@
# @trigger.dev/eslint-plugin
## 3.0.0-beta.25
## 3.0.0-beta.24
## 3.0.0-beta.23
## 3.0.0-beta.22
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/eslint-plugin",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "ESLint plugin with trigger.dev best practices",
"keywords": [
"eslint",
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/express
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/express",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Official Express adapter for Trigger.dev",
"license": "MIT",
"main": "./dist/index.js",
@@ -23,7 +23,7 @@
"./package.json": "./package.json"
},
"devDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@trigger.dev/tsconfig": "workspace:*",
"@types/debug": "^4.1.7",
"@types/express": "^4.17.13",
@@ -39,7 +39,7 @@
"build:tsup": "tsup"
},
"peerDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25"
},
"dependencies": {
"debug": "^4.3.4",
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/hono
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/hono",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "A Trigger.dev adapter for Hono.dev",
"license": "MIT",
"main": "./dist/index.js",
@@ -32,7 +32,7 @@
},
"peerDependencies": {
"hono": "3.x",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25"
},
"devDependencies": {
"@trigger.dev/tsconfig": "workspace:*",
+15
View File
@@ -1,5 +1,20 @@
# @trigger.dev/integration-kit
## 3.0.0-beta.25
### Patch Changes
- Updated dependencies [e337b2165]
- Updated dependencies [9e5382951]
- @trigger.dev/core@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- Updated dependencies [83dc87155]
- @trigger.dev/core@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/integration-kit",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev Integration Kit has helpers to make creating integrations easier",
"license": "MIT",
"main": "./dist/index.js",
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/nestjs
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/nestjs",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Official NestJS adapter for Trigger.dev",
"license": "MIT",
"main": "./dist/index.js",
@@ -23,7 +23,7 @@
"./package.json": "./package.json"
},
"devDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@trigger.dev/tsconfig": "workspace:*",
"@types/debug": "^4.1.7",
"@types/express": "^4.17.13",
@@ -41,7 +41,7 @@
},
"peerDependencies": {
"@nestjs/common": ">=10.0.0",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25"
},
"dependencies": {
"@nestjs/common": "^10.2.4",
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/nextjs
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/nextjs",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev Next.js integration",
"license": "MIT",
"main": "./dist/index.js",
@@ -41,7 +41,7 @@
"build:tsup": "tsup"
},
"peerDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"next": ">=12.0.0"
},
"dependencies": {
+4
View File
@@ -1,5 +1,9 @@
# @trigger.dev/otlp-importer
## 3.0.0-beta.25
## 3.0.0-beta.24
## 3.0.0-beta.23
## 3.0.0-beta.22
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/otlp-importer",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "OpenTelemetry OTLP Importer for Node.js written in TypeScript",
"license": "MIT",
"main": "./dist/index.js",
+15
View File
@@ -1,5 +1,20 @@
# @trigger.dev/react
## 3.0.0-beta.25
### Patch Changes
- Updated dependencies [e337b2165]
- Updated dependencies [9e5382951]
- @trigger.dev/core@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- Updated dependencies [83dc87155]
- @trigger.dev/core@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/react",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev React SDK",
"license": "MIT",
"types": "dist/index.d.ts",
@@ -27,7 +27,7 @@
},
"dependencies": {
"@tanstack/react-query": "5.0.0-beta.2",
"@trigger.dev/core": "workspace:^3.0.0-beta.23",
"@trigger.dev/core": "workspace:^3.0.0-beta.25",
"debug": "^4.3.4",
"zod": "3.22.3"
},
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/remix
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/remix",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev Remix integration",
"license": "MIT",
"main": "./dist/index.js",
@@ -39,7 +39,7 @@
"build:tsup": "tsup"
},
"peerDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
"@remix-run/server-runtime": ">1.19.0"
},
"dependencies": {
+12
View File
@@ -1,5 +1,17 @@
# @trigger.dev/sveltekit
## 3.0.0-beta.25
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/sveltekit",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"description": "Trigger.dev svelteKit integration",
"license": "MIT",
"main": "./dist/index.js",
@@ -39,7 +39,7 @@
"build:tsup": "tsup"
},
"peerDependencies": {
"@trigger.dev/sdk": "workspace:^3.0.0-beta.23"
"@trigger.dev/sdk": "workspace:^3.0.0-beta.25"
},
"dependencies": {
"debug": "^4.3.4"
+17
View File
@@ -1,5 +1,22 @@
# @trigger.dev/testing
## 3.0.0-beta.25
### Patch Changes
- Updated dependencies [e337b2165]
- Updated dependencies [9e5382951]
- @trigger.dev/core@3.0.0-beta.25
- @trigger.dev/sdk@3.0.0-beta.25
## 3.0.0-beta.24
### Patch Changes
- Updated dependencies [83dc87155]
- @trigger.dev/core@3.0.0-beta.24
- @trigger.dev/sdk@3.0.0-beta.24
## 3.0.0-beta.23
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@trigger.dev/testing",
"description": "A collection of useful tools to write tests for Trigger.dev.",
"version": "3.0.0-beta.23",
"version": "3.0.0-beta.25",
"license": "MIT",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",

Some files were not shown because too many files have changed in this diff Show More