From 50e3d9e43a139308ebe19635ea07c2d95098388a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 11 Oct 2023 17:31:19 +0100 Subject: [PATCH] Indexing errors don't get displayed anywhere (#605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added EndpointIndex status. Default is PENDING, existing rows are SUCCESS * Made it easier to create a migration SQL file * Created a job-catalog file for misconfigured Jobs that should error when running the CLI * EndpointIndex data and state are now optional * The Environments page now shows the status of the last refresh * Improved the UI about endpoints * Added EndpointIndex error column * WIP making the endpoint indexing more robust * Indexing errors are now surfaced * Improved the error show it shows the job id * Use “performEndpointIndexing” when you create your first endpoint from the UI * Use “performEndpointIndexing” for the recurring endpoint checker * Staging is now auto-indexed every 10 mins too * Use “performEndpointIndexing” for the webhook * Moved the throttling to a util * Removed instructional comments * Created a reusable retry system with exponential backoff * Use p-retry for retrying with backoff * The CLI gets indexing results and displays errors * Improved the indexing error messages and display in the console * Use a pre so the Indexing error is correctly split over multiple lines * Use a db transaction for webhook that triggers endpoint indexing * Tidied up imports * Support older versions of the server * Improved the comment on the misconfigured job * Changeset: When indexing user's jobs errors are now stored and displayed --- .changeset/seven-turkeys-worry.md | 6 + .../environments/EndpointIndexStatus.tsx | 74 ++++ .../app/components/primitives/Callout.tsx | 11 +- .../app/components/primitives/FormError.tsx | 13 +- .../webapp/app/models/indexEndpoint.server.ts | 14 - .../EnvironmentsPresenter.server.ts | 20 +- .../ConfigureEndpointSheet.tsx | 60 +++- .../route.tsx | 23 +- .../routes/api.v1.endpointindex.$indexId.ts | 68 ++++ ...endpointSlug.index.$indexHookIdentifier.ts | 80 +++-- ...nvironmentParam.endpoint.$endpointParam.ts | 6 - .../webapp/app/services/endpointApi.server.ts | 40 +-- .../endpoints/createEndpoint.server.ts | 15 +- .../endpoints/indexEndpoint.server.ts | 233 +----------- .../endpoints/performEndpointIndexService.ts | 338 ++++++++++++++++++ .../recurringEndpointIndex.server.ts | 18 +- .../validateCreateEndpoint.server.ts | 11 +- apps/webapp/app/services/worker.server.ts | 20 +- apps/webapp/package.json | 3 +- packages/cli/package.json | 3 + packages/cli/src/commands/dev.ts | 279 +++++++++------ packages/cli/src/commands/init.ts | 2 +- packages/cli/src/commands/whoami.ts | 2 +- packages/cli/src/utils/logger.ts | 3 + packages/cli/src/utils/throttle.ts | 18 + packages/cli/src/utils/triggerApi.ts | 36 +- packages/cli/src/utils/wait.ts | 5 + packages/cli/tsconfig.json | 1 - packages/core/src/schemas/api.ts | 47 +++ packages/database/package.json | 3 +- .../migration.sql | 11 + .../migration.sql | 3 + .../migration.sql | 2 + packages/database/prisma/schema.prisma | 17 +- pnpm-lock.yaml | 59 ++- references/job-catalog/package.json | 3 +- references/job-catalog/src/misconfigured.ts | 35 ++ 37 files changed, 1101 insertions(+), 481 deletions(-) create mode 100644 .changeset/seven-turkeys-worry.md create mode 100644 apps/webapp/app/components/environments/EndpointIndexStatus.tsx delete mode 100644 apps/webapp/app/models/indexEndpoint.server.ts create mode 100644 apps/webapp/app/routes/api.v1.endpointindex.$indexId.ts create mode 100644 apps/webapp/app/services/endpoints/performEndpointIndexService.ts create mode 100644 packages/cli/src/utils/throttle.ts create mode 100644 packages/cli/src/utils/wait.ts create mode 100644 packages/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql create mode 100644 packages/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql create mode 100644 packages/database/prisma/migrations/20231010135433_endpoint_index_added_error_column/migration.sql create mode 100644 references/job-catalog/src/misconfigured.ts diff --git a/.changeset/seven-turkeys-worry.md b/.changeset/seven-turkeys-worry.md new file mode 100644 index 000000000..9165ca4e2 --- /dev/null +++ b/.changeset/seven-turkeys-worry.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/cli": patch +--- + +When indexing user's jobs errors are now stored and displayed diff --git a/apps/webapp/app/components/environments/EndpointIndexStatus.tsx b/apps/webapp/app/components/environments/EndpointIndexStatus.tsx new file mode 100644 index 000000000..67398b0b7 --- /dev/null +++ b/apps/webapp/app/components/environments/EndpointIndexStatus.tsx @@ -0,0 +1,74 @@ +import { CheckCircleIcon, ClockIcon, XCircleIcon } from "@heroicons/react/20/solid"; +import { EndpointIndexStatus } from "@trigger.dev/database"; +import { cn } from "~/utils/cn"; +import { Spinner } from "../primitives/Spinner"; + +export function EndpointIndexStatusIcon({ status }: { status: EndpointIndexStatus }) { + switch (status) { + case "PENDING": + return ; + case "STARTED": + return ; + case "SUCCESS": + return ( + + ); + case "FAILURE": + return ; + } +} + +export function EndpointIndexStatusLabel({ status }: { status: EndpointIndexStatus }) { + switch (status) { + case "PENDING": + return ( + + {endpointIndexStatusTitle(status)} + + ); + case "STARTED": + return ( + + {endpointIndexStatusTitle(status)} + + ); + case "SUCCESS": + return ( + + {endpointIndexStatusTitle(status)} + + ); + case "FAILURE": + return ( + + {endpointIndexStatusTitle(status)} + + ); + } +} + +export function endpointIndexStatusTitle(status: EndpointIndexStatus): string { + switch (status) { + case "PENDING": + return "Pending"; + case "STARTED": + return "Started"; + case "SUCCESS": + return "Success"; + case "FAILURE": + return "Failure"; + } +} + +export function endpointIndexStatusClassNameColor(status: EndpointIndexStatus): string { + switch (status) { + case "PENDING": + return "text-dimmed"; + case "STARTED": + return "text-blue-500"; + case "SUCCESS": + return "text-green-500"; + case "FAILURE": + return "text-rose-500"; + } +} diff --git a/apps/webapp/app/components/primitives/Callout.tsx b/apps/webapp/app/components/primitives/Callout.tsx index eb920502e..a4bd20be6 100644 --- a/apps/webapp/app/components/primitives/Callout.tsx +++ b/apps/webapp/app/components/primitives/Callout.tsx @@ -13,6 +13,7 @@ import { import { Link } from "@remix-run/react"; import { cn } from "~/utils/cn"; import { Paragraph } from "./Paragraph"; +import { Spinner } from "./Spinner"; export const variantClasses = { info: { @@ -51,8 +52,16 @@ export const variantClasses = { textColor: "text-blue-200", linkClassName: "transition hover:bg-blue-400/40", }, + pending: { + className: "border-blue-400/20 bg-blue-800/30", + icon: , + textColor: "text-blue-300", + linkClassName: "transition hover:bg-blue-400/40", + }, } as const; +export type CalloutVariant = keyof typeof variantClasses; + export function Callout({ children, className, @@ -63,7 +72,7 @@ export function Callout({ children?: React.ReactNode; className?: string; icon?: React.ReactNode; - variant: keyof typeof variantClasses; + variant: CalloutVariant; to?: string; }) { const variantDefinition = variantClasses[variant]; diff --git a/apps/webapp/app/components/primitives/FormError.tsx b/apps/webapp/app/components/primitives/FormError.tsx index 79d537d74..ed90f7e99 100644 --- a/apps/webapp/app/components/primitives/FormError.tsx +++ b/apps/webapp/app/components/primitives/FormError.tsx @@ -2,8 +2,17 @@ import type { z } from "zod"; import { Paragraph } from "./Paragraph"; import { NamedIcon } from "./NamedIcon"; import { motion } from "framer-motion"; +import { cn } from "~/utils/cn"; -export function FormError({ children, id }: { children: React.ReactNode; id?: string }) { +export function FormError({ + children, + id, + className, +}: { + children: React.ReactNode; + id?: string; + className?: string; +}) { return ( <> {children && ( @@ -11,7 +20,7 @@ export function FormError({ children, id }: { children: React.ReactNode; id?: st initial={{ opacity: 0 }} animate={{ opacity: 1 }} transition={{ duration: 0.3 }} - className="flex items-start gap-0.5" + className={cn("flex items-start gap-0.5", className)} > diff --git a/apps/webapp/app/models/indexEndpoint.server.ts b/apps/webapp/app/models/indexEndpoint.server.ts deleted file mode 100644 index 6a58660de..000000000 --- a/apps/webapp/app/models/indexEndpoint.server.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { z } from "zod"; - -const IndexEndpointStatsSchema = z.object({ - jobs: z.number(), - sources: z.number(), - dynamicTriggers: z.number(), - dynamicSchedules: z.number(), -}); - -export type IndexEndpointStats = z.infer; - -export function parseEndpointIndexStats(stats: unknown): IndexEndpointStats { - return IndexEndpointStatsSchema.parse(stats); -} diff --git a/apps/webapp/app/presenters/EnvironmentsPresenter.server.ts b/apps/webapp/app/presenters/EnvironmentsPresenter.server.ts index 477691f62..bd5f4ef87 100644 --- a/apps/webapp/app/presenters/EnvironmentsPresenter.server.ts +++ b/apps/webapp/app/presenters/EnvironmentsPresenter.server.ts @@ -1,13 +1,19 @@ import { PrismaClient, prisma } from "~/db.server"; -import { IndexEndpointStats, parseEndpointIndexStats } from "~/models/indexEndpoint.server"; import { Project } from "~/models/project.server"; import { User } from "~/models/user.server"; import type { Endpoint, EndpointIndex, + EndpointIndexStatus, RuntimeEnvironment, RuntimeEnvironmentType, } from "@trigger.dev/database"; +import { + EndpointIndexError, + EndpointIndexErrorSchema, + IndexEndpointStats, + parseEndpointIndexStats, +} from "@trigger.dev/core"; export type Client = { slug: string; @@ -34,9 +40,11 @@ export type ClientEndpoint = url: string; indexWebhookPath: string; latestIndex?: { + status: EndpointIndexStatus; source: string; updatedAt: Date; - stats: IndexEndpointStats; + stats?: IndexEndpointStats; + error?: EndpointIndexError; }; environment: { id: string; @@ -81,9 +89,11 @@ export class EnvironmentsPresenter { indexingHookIdentifier: true, indexings: { select: { + status: true, source: true, updatedAt: true, stats: true, + error: true, }, take: 1, orderBy: { @@ -214,7 +224,7 @@ const environmentSortOrder: RuntimeEnvironmentType[] = [ function endpointClient( endpoint: Pick & { - indexings: Pick[]; + indexings: Pick[]; }, environment: Pick, baseUrl: string @@ -227,9 +237,13 @@ function endpointClient( indexWebhookPath: `${baseUrl}/api/v1/endpoints/${environment.id}/${endpoint.slug}/index/${endpoint.indexingHookIdentifier}`, latestIndex: endpoint.indexings[0] ? { + status: endpoint.indexings[0].status, source: endpoint.indexings[0].source, updatedAt: endpoint.indexings[0].updatedAt, stats: parseEndpointIndexStats(endpoint.indexings[0].stats), + error: endpoint.indexings[0].error + ? EndpointIndexErrorSchema.parse(endpoint.indexings[0].error) + : undefined, } : undefined, environment: environment, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx index 5db9273da..171761967 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx @@ -6,7 +6,7 @@ import { useEventSource } from "remix-utils"; import { InlineCode } from "~/components/code/InlineCode"; import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel"; import { Button } from "~/components/primitives/Buttons"; -import { Callout } from "~/components/primitives/Callout"; +import { Callout, CalloutVariant } from "~/components/primitives/Callout"; import { ClipboardField } from "~/components/primitives/ClipboardField"; import { DateTime } from "~/components/primitives/DateTime"; import { FormError } from "~/components/primitives/FormError"; @@ -18,8 +18,14 @@ import { Paragraph } from "~/components/primitives/Paragraph"; import { Sheet, SheetBody, SheetContent, SheetHeader } from "~/components/primitives/Sheet"; import { ClientEndpoint } from "~/presenters/EnvironmentsPresenter.server"; import { endpointStreamingPath } from "~/utils/pathBuilder"; -import { RuntimeEnvironmentType } from "../../../../../packages/database/src"; +import { EndpointIndexStatus, RuntimeEnvironmentType } from "../../../../../packages/database/src"; import { bodySchema } from "../resources.environments.$environmentParam.endpoint"; +import { + EndpointIndexStatusIcon, + EndpointIndexStatusLabel, + endpointIndexStatusTitle, +} from "~/components/environments/EndpointIndexStatus"; +import { CodeBlock } from "~/components/code/CodeBlock"; type ConfigureEndpointSheetProps = { slug: string; @@ -119,15 +125,29 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd method="post" action={`/resources/environments/${endpoint.environment.id}/endpoint/${endpoint.id}`} > - - - Endpoint configured. Last refreshed:{" "} - {endpoint.latestIndex ? ( - - ) : ( - "–" - )} - + + } + className="justiy-between items-center" + > +
+ + + Last refreshed:{" "} + {endpoint.latestIndex ? ( + <> + + + ) : ( + "–" + )} + +
+