From 9f076631ae076cb28b70cd616ea3e270461b0c21 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 1 Sep 2023 22:05:06 +0100 Subject: [PATCH] #346 Improve run dashboard perf when tasks have large outputs Performance degradation came from the syntax highlighting of large code blocks and from doing that on the server and the client, so fixed this in a couple of ways: 1. Stream the task details data using defer and Suspense/Await 2. Skipped syntax highlighting code blocks with more than 1k lines --- apps/webapp/app/components/code/CodeBlock.tsx | 190 ++++++++++-------- .../app/components/run/RunCompletedDetail.tsx | 2 +- apps/webapp/app/components/run/TaskDetail.tsx | 27 +-- .../presenters/TaskDetailsPresenter.server.ts | 5 +- .../route.tsx | 31 +-- .../route.tsx | 29 +-- examples/job-catalog/src/stressTest.ts | 48 +++++ 7 files changed, 199 insertions(+), 133 deletions(-) diff --git a/apps/webapp/app/components/code/CodeBlock.tsx b/apps/webapp/app/components/code/CodeBlock.tsx index c447bc646..5c7579be3 100644 --- a/apps/webapp/app/components/code/CodeBlock.tsx +++ b/apps/webapp/app/components/code/CodeBlock.tsx @@ -1,10 +1,9 @@ +import { Clipboard, ClipboardCheck } from "lucide-react"; import type { Language, PrismTheme } from "prism-react-renderer"; import Highlight, { defaultProps } from "prism-react-renderer"; import { forwardRef, useCallback, useState } from "react"; import { cn } from "~/utils/cn"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip"; -import { ClipboardDocumentCheckIcon, ClipboardIcon } from "@heroicons/react/24/solid"; -import { Clipboard, ClipboardCheck, ClipboardCheckIcon } from "lucide-react"; //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting @@ -192,6 +191,9 @@ export const CodeBlock = forwardRef( Array.from({ length: end - start + 1 }, (_, i) => start + i) ); + // if there are more than 1000 lines, don't highlight + const shouldHighlight = lineCount <= 1000; + return (
( )} - - {({ - className: inheritedClassName, - style: inheritedStyle, - tokens, - getLineProps, - getTokenProps, - }) => ( -
- 
+            {({
+              className: inheritedClassName,
+              style: inheritedStyle,
+              tokens,
+              getLineProps,
+              getTokenProps,
+            }) => (
+              

-                {tokens
-                  .map((line, index) => {
-                    if (
-                      index === tokens.length - 1 &&
-                      line.length === 1 &&
-                      line[0].content === "\n"
-                    ) {
-                      return null;
-                    }
+                
+                  {tokens
+                    .map((line, index) => {
+                      if (
+                        index === tokens.length - 1 &&
+                        line.length === 1 &&
+                        line[0].content === "\n"
+                      ) {
+                        return null;
+                      }
 
-                    const lineNumber = index + 1;
-                    const lineProps = getLineProps({ line, key: index });
+                      const lineNumber = index + 1;
+                      const lineProps = getLineProps({ line, key: index });
 
-                    let hasAnyHighlights = highlightLines ? highlightLines.length > 0 : false;
+                      let hasAnyHighlights = highlightLines ? highlightLines.length > 0 : false;
 
-                    let shouldDim = hasAnyHighlights;
-                    if (hasAnyHighlights && highlightLines?.includes(lineNumber)) {
-                      shouldDim = false;
-                    }
+                      let shouldDim = hasAnyHighlights;
+                      if (hasAnyHighlights && highlightLines?.includes(lineNumber)) {
+                        shouldDim = false;
+                      }
 
-                    return (
-                      

-                        {showLineNumbers && (
-                          

-                            {lineNumber}
+                      return (
+                        

+                          {showLineNumbers && (
+                            

+                              {lineNumber}
+                            

+                          )}
+
+                          

+                            {line.map((token, key) => {
+                              const tokenProps = getTokenProps({ token, key });
+                              return (
+                                
+                              );
+                            })}
                           

-                        )}
-
-                        

-                          {line.map((token, key) => {
-                            const tokenProps = getTokenProps({ token, key });
-                            return (
-                              
-                            );
-                          })}
+                          

                         

-                        

-                      

-                    );
-                  })
-                  .filter(Boolean)}
-              

-            

-          )}
-        
+                      );
+                    })
+                    .filter(Boolean)}
+
+
+ )} + + ) : ( +
+ 
+              {code}
+
+
+ )}
); } diff --git a/apps/webapp/app/components/run/RunCompletedDetail.tsx b/apps/webapp/app/components/run/RunCompletedDetail.tsx index 447dc6698..73cbceaa1 100644 --- a/apps/webapp/app/components/run/RunCompletedDetail.tsx +++ b/apps/webapp/app/components/run/RunCompletedDetail.tsx @@ -54,7 +54,7 @@ export function RunCompletedDetail({ run }: { run: MatchedRun }) { {run.error && } {run.output ? ( - + ) : ( run.output === null && This run returned nothing )} diff --git a/apps/webapp/app/components/run/TaskDetail.tsx b/apps/webapp/app/components/run/TaskDetail.tsx index efa68214d..9a85df317 100644 --- a/apps/webapp/app/components/run/TaskDetail.tsx +++ b/apps/webapp/app/components/run/TaskDetail.tsx @@ -1,4 +1,3 @@ -import { DetailedTask } from "~/presenters/TaskDetailsPresenter.server"; import { RunPanel, RunPanelBody, @@ -29,22 +28,16 @@ import { } from "../primitives/Table"; import { TaskAttemptStatusLabel } from "./TaskAttemptStatus"; import { TaskStatusIcon } from "./TaskStatus"; +import { ClientOnly } from "remix-utils"; +import { Spinner } from "../primitives/Spinner"; +import type { DetailedTask } from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route"; export function TaskDetail({ task }: { task: DetailedTask }) { - const { - name, - description, - icon, - startedAt, - completedAt, - status, - delayUntil, - params, - properties, - output, - style, - attempts, - } = task; + const { name, description, icon, status, params, properties, output, style, attempts } = task; + + const startedAt = task.startedAt ? new Date(task.startedAt) : undefined; + const completedAt = task.completedAt ? new Date(task.completedAt) : undefined; + const delayUntil = task.delayUntil ? new Date(task.delayUntil) : undefined; return ( @@ -150,7 +143,9 @@ export function TaskDetail({ task }: { task: DetailedTask }) {
Output {output ? ( - + }> + {() => } + ) : ( No output )} diff --git a/apps/webapp/app/presenters/TaskDetailsPresenter.server.ts b/apps/webapp/app/presenters/TaskDetailsPresenter.server.ts index 9061a090c..1898594bf 100644 --- a/apps/webapp/app/presenters/TaskDetailsPresenter.server.ts +++ b/apps/webapp/app/presenters/TaskDetailsPresenter.server.ts @@ -1,4 +1,4 @@ -import { DisplayPropertiesSchema, StyleSchema } from "@trigger.dev/core"; +import { StyleSchema } from "@trigger.dev/core"; import { PrismaClient, prisma } from "~/db.server"; import { mergeProperties } from "~/utils/mergeProperties.server"; @@ -7,8 +7,6 @@ type DetailsProps = { userId: string; }; -export type DetailedTask = NonNullable>>; - export class TaskDetailsPresenter { #prismaClient: PrismaClient; @@ -87,6 +85,7 @@ export class TaskDetailsPresenter { return { ...task, + output: task.output ? JSON.stringify(task.output, null, 2) : undefined, connection: task.runConnection, params: task.params as Record, properties: mergeProperties(task.properties, task.outputProperties), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route.tsx index 86ffe8131..a02bf3ace 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route.tsx @@ -1,5 +1,7 @@ -import { LoaderArgs } from "@remix-run/server-runtime"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Await, useLoaderData } from "@remix-run/react"; +import { LoaderArgs, SerializeFrom, defer } from "@remix-run/server-runtime"; +import { Suspense } from "react"; +import { Spinner } from "~/components/primitives/Spinner"; import { TaskDetail } from "~/components/run/TaskDetail"; import { TaskDetailsPresenter } from "~/presenters/TaskDetailsPresenter.server"; import { requireUserId } from "~/services/session.server"; @@ -10,23 +12,26 @@ export const loader = async ({ request, params }: LoaderArgs) => { const { taskParam } = TaskParamsSchema.parse(params); const presenter = new TaskDetailsPresenter(); - const task = await presenter.call({ + const taskPromise = presenter.call({ userId, id: taskParam, }); - if (!task) { - throw new Response(null, { - status: 404, - }); - } - - return typedjson({ - task, + return defer({ + taskPromise, }); }; +export type DetailedTask = NonNullable["taskPromise"]>>; + export default function Page() { - const { task } = useTypedLoaderData(); - return ; + const { taskPromise } = useLoaderData(); + + return ( + }> + Error loading task!

}> + {(resolvedTask) => resolvedTask && } + + + ); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam.tasks.$taskParam/route.tsx index d2a118146..bac7cb824 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam.tasks.$taskParam/route.tsx @@ -1,5 +1,7 @@ -import { LoaderArgs } from "@remix-run/server-runtime"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Await, useLoaderData } from "@remix-run/react"; +import { LoaderArgs, defer } from "@remix-run/server-runtime"; +import { Suspense } from "react"; +import { Spinner } from "~/components/primitives/Spinner"; import { TaskDetail } from "~/components/run/TaskDetail"; import { TaskDetailsPresenter } from "~/presenters/TaskDetailsPresenter.server"; import { requireUserId } from "~/services/session.server"; @@ -10,23 +12,24 @@ export const loader = async ({ request, params }: LoaderArgs) => { const { taskParam } = TriggerSourceRunTaskParamsSchema.parse(params); const presenter = new TaskDetailsPresenter(); - const task = await presenter.call({ + const taskPromise = presenter.call({ userId, id: taskParam, }); - if (!task) { - throw new Response(null, { - status: 404, - }); - } - - return typedjson({ - task, + return defer({ + taskPromise, }); }; export default function Page() { - const { task } = useTypedLoaderData(); - return ; + const { taskPromise } = useLoaderData(); + + return ( + }> + Error loading task!

}> + {(resolvedTask) => resolvedTask && } + + + ); } diff --git a/examples/job-catalog/src/stressTest.ts b/examples/job-catalog/src/stressTest.ts index 06949274e..ad7e50a86 100644 --- a/examples/job-catalog/src/stressTest.ts +++ b/examples/job-catalog/src/stressTest.ts @@ -38,4 +38,52 @@ client.defineJob({ }, }); +client.defineJob({ + id: "stress-test-2", + name: "Stress Test 2", + version: "1.0.0", + trigger: eventTrigger({ + name: "stress.test.2", + }), + run: async (payload, io, ctx) => { + await io.runTask(`task-1`, { name: `Task 1` }, async (task) => { + const response = await fetch("https://jsonplaceholder.typicode.com/photos"); + + return await response.json(); + }); + + await io.runTask(`task-2`, { name: `Task 2` }, async (task) => { + const response = await fetch("https://jsonplaceholder.typicode.com/comments"); + + return await response.json(); + }); + + await io.runTask(`task-3`, { name: `Task 3` }, async (task) => { + const response = await fetch("https://jsonplaceholder.typicode.com/photos"); + + return await response.json(); + }); + + await io.runTask(`task-4`, { name: `Task 4` }, async (task) => { + const response = await fetch("https://jsonplaceholder.typicode.com/comments"); + + return await response.json(); + }); + + const response = await io.runTask(`task-5`, { name: `Task 5` }, async (task) => { + const response = await fetch("https://jsonplaceholder.typicode.com/photos"); + + return await response.json(); + }); + + await io.runTask(`task-6`, { name: `Task 6` }, async (task) => { + const response = await fetch("https://jsonplaceholder.typicode.com/users"); + + return await response.json(); + }); + + return response; + }, +}); + createExpressServer(client);