Add your first endpoint in the dashboard (#269)

* Created the sheet

* Started work on the resource route

* Created ValidateCreateEndpointService and the EndpointValidateApi (name TBC)

* The TriggerClient responds with the id

* Tidied some stuff up

* Alternative: EndpointApi doesn’t take endpointSlug. Instead just Ping() does

* Create cool-snakes-deny.md

---------

Co-authored-by: Eric Allam <eallam@icloud.com>

.changeset/cool-snakes-deny.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+Adding the validate endpoint action to be able to add an endpoint first in the dashboard

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/FirstEndpointSheet.tsx

@@ -0,0 +1,121 @@
+import { conform, useForm, useInputEvent } from "@conform-to/react";
+import { parse } from "@conform-to/zod";
+import { useFetcher } from "@remix-run/react";
+import { InlineCode } from "~/components/code/InlineCode";
+import { Button, ButtonContent } from "~/components/primitives/Buttons";
+import { FormError } from "~/components/primitives/FormError";
+import { Header1, Header2 } from "~/components/primitives/Headers";
+import { Hint } from "~/components/primitives/Hint";
+import { Input } from "~/components/primitives/Input";
+import { InputGroup } from "~/components/primitives/InputGroup";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import {
+  Sheet,
+  SheetBody,
+  SheetContent,
+  SheetHeader,
+  SheetTrigger,
+} from "~/components/primitives/Sheet";
+import { TextLink } from "~/components/primitives/TextLink";
+import { docsPath } from "~/utils/pathBuilder";
+import { bodySchema } from "../resources.projects.$projectId.endpoint";
+import { RuntimeEnvironment, RuntimeEnvironmentType } from "@trigger.dev/database";
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "~/components/primitives/Select";
+import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { useRef, useState } from "react";
+
+type FirstEndpointSheetProps = {
+  projectId: string;
+  environments: { id: string; type: RuntimeEnvironmentType }[];
+};
+
+export function FirstEndpointSheet({ projectId, environments }: FirstEndpointSheetProps) {
+  const setEndpointUrlFetcher = useFetcher();
+  const [form, { url, environmentId }] = useForm({
+    id: "new-endpoint-url",
+    lastSubmission: setEndpointUrlFetcher.data,
+    onValidate({ formData }) {
+      return parse(formData, { schema: bodySchema });
+    },
+  });
+
+  const loadingEndpointUrl = setEndpointUrlFetcher.state !== "idle";
+
+  return (
+    <Sheet>
+      <SheetTrigger>
+        <ButtonContent variant={"primary/medium"}>Add your first endpoint</ButtonContent>
+      </SheetTrigger>
+      <SheetContent size="lg">
+        <SheetHeader>
+          <div>
+            <Header1>Add your first endpoint</Header1>
+            <Paragraph variant="small">
+              We recommend you use{" "}
+              <TextLink href={docsPath("documentation/guides/cli")}>the CLI</TextLink> when working
+              in development.
+            </Paragraph>
+          </div>
+        </SheetHeader>
+        <SheetBody>
+          <setEndpointUrlFetcher.Form
+            method="post"
+            action={`/resources/projects/${projectId}/endpoint`}
+            {...form.props}
+          >
+            <InputGroup className="mb-4 max-w-none">
+              <Header2>Environment type</Header2>
+              <SelectGroup>
+                <Select name={"environmentId"} defaultValue={environments[0].id}>
+                  <SelectTrigger size="secondary/small">
+                    <SelectValue placeholder="Select environment" className="m-0 p-0" /> Environment
+                  </SelectTrigger>
+                  <SelectContent>
+                    {environments.map((environment) => (
+                      <SelectItem key={environment.id} value={environment.id}>
+                        <EnvironmentLabel environment={environment} />
+                      </SelectItem>
+                    ))}
+                  </SelectContent>
+                </Select>
+              </SelectGroup>
+              <FormError id={environmentId.errorId}>{environmentId.error}</FormError>
+            </InputGroup>
+            <InputGroup className="max-w-none">
+              <Header2>Endpoint URL</Header2>
+              <div className="flex items-center">
+                <Input
+                  className="rounded-r-none"
+                  {...conform.input(url, { type: "url" })}
+                  placeholder="URL for your Trigger API route"
+                />
+                <Button
+                  type="submit"
+                  variant="primary/medium"
+                  className="rounded-l-none"
+                  disabled={loadingEndpointUrl}
+                  LeadingIcon={loadingEndpointUrl ? "spinner-white" : undefined}
+                >
+                  {loadingEndpointUrl ? "Saving" : "Save"}
+                </Button>
+              </div>
+              <FormError id={url.errorId}>{url.error}</FormError>
+              <FormError id={form.errorId}>{form.error}</FormError>
+              <Hint>
+                This is the URL of your Trigger API route, Typically this would be:{" "}
+                <InlineCode variant="extra-small">https://yourdomain.com/api/trigger</InlineCode>.
+              </Hint>
+            </InputGroup>
+          </setEndpointUrlFetcher.Form>
+        </SheetBody>
+      </SheetContent>
+    </Sheet>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx

@@ -7,7 +7,7 @@ import { EnvironmentLabel, environmentTitle } from "~/components/environments/En
 import { HowToUseApiKeysAndEndpoints } from "~/components/helpContent/HelpContentText";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { BreadcrumbLink } from "~/components/navigation/NavBar";
-import { ButtonContent } from "~/components/primitives/Buttons";
+import { Button, ButtonContent } from "~/components/primitives/Buttons";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
@@ -39,6 +39,7 @@ import { requestUrl } from "~/utils/requestUrl.server";
 import { RuntimeEnvironmentType } from "../../../../../packages/database/src";
 import { ConfigureEndpointSheet } from "./ConfigureEndpointSheet";
 import { Badge } from "~/components/primitives/Badge";
+import { FirstEndpointSheet } from "./FirstEndpointSheet";
 
 export const loader = async ({ request, params }: LoaderArgs) => {
   const userId = await requireUserId(request);
@@ -202,7 +203,12 @@ export default function Page() {
                       </div>
                     ))
                   ) : (
-                    <Paragraph>You have no clients yet</Paragraph>
+                    <>
+                      <Paragraph>Add your first endpoint</Paragraph>
+                      <Paragraph>
+                        <FirstEndpointSheet projectId={project.id} environments={environments} />
+                      </Paragraph>
+                    </>
                   )}
                 </div>
                 {selectedEndpoint && (

apps/webapp/app/routes/resources.projects.$projectId.endpoint.ts

@@ -0,0 +1,71 @@
+import { parse } from "@conform-to/zod";
+import { ActionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import {
+  CreateEndpointError,
+  CreateEndpointService,
+} from "~/services/endpoints/createEndpoint.server";
+import { requireUserId } from "~/services/session.server";
+import { RuntimeEnvironmentTypeSchema } from "@trigger.dev/core";
+import { env } from "process";
+import { ValidateCreateEndpointService } from "~/services/endpoints/validateCreateEndpoint.server";
+
+const ParamsSchema = z.object({
+  projectId: z.string(),
+});
+
+export const bodySchema = z.object({
+  environmentId: z.string(),
+  url: z.string().url("Must be a valid URL"),
+});
+
+export async function action({ request, params }: ActionArgs) {
+  const userId = await requireUserId(request);
+  const { projectId } = ParamsSchema.parse(params);
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: bodySchema });
+
+  if (!submission.value || submission.intent !== "submit") {
+    return json(submission);
+  }
+
+  try {
+    const environment = await prisma.runtimeEnvironment.findUnique({
+      include: {
+        organization: true,
+        project: true,
+      },
+      where: {
+        id: submission.value.environmentId,
+      },
+    });
+
+    if (!environment) {
+      submission.error.environmentId = "Environment not found";
+      return json(submission);
+    }
+
+    const service = new ValidateCreateEndpointService();
+    const result = await service.call({
+      url: submission.value.url,
+      environment,
+    });
+
+    return json(submission);
+  } catch (e) {
+    if (e instanceof CreateEndpointError) {
+      submission.error.url = e.message;
+      return json(submission);
+    }
+
+    if (e instanceof Error) {
+      submission.error.url = `${e.name}: ${e.message}`;
+    } else {
+      submission.error.url = "Unknown error";
+    }
+
+    return json(submission, { status: 400 });
+  }
+}

apps/webapp/app/services/endpointApi.server.ts

@@ -13,6 +13,8 @@ import {
   RegisterTriggerBodySchema,
   RunJobBody,
   RunJobResponseSchema,
+  ValidateResponse,
+  ValidateResponseSchema,
 } from "@trigger.dev/core";
 import { safeBodyFromResponse, safeParseBodyFromResponse } from "~/utils/json";
 import { logger } from "./logger.server";
@@ -25,16 +27,15 @@ export class EndpointApiError extends Error {
   }
 }
 
-// TODO: this should work with tunnelling
 export class EndpointApi {
-  constructor(private apiKey: string, private url: string, private id: string) {}
+  constructor(private apiKey: string, private url: string) {}
 
-  async ping(): Promise<PongResponse> {
+  async ping(endpointId: string): Promise<PongResponse> {
     const response = await safeFetch(this.url, {
       method: "POST",
       headers: {
         "x-trigger-api-key": this.apiKey,
-        "x-trigger-endpoint-id": this.id,
+        "x-trigger-endpoint-id": endpointId,
         "x-trigger-action": "PING",
       },
     });
@@ -271,6 +272,64 @@ export class EndpointApi {
 
     return HttpSourceResponseSchema.parse(anyBody);
   }
+
+  async validate(): Promise<ValidateResponse> {
+    const response = await safeFetch(this.url, {
+      method: "POST",
+      headers: {
+        "x-trigger-api-key": this.apiKey,
+        "x-trigger-action": "VALIDATE",
+      },
+    });
+
+    if (!response) {
+      return {
+        ok: false,
+        error: `Could not connect to endpoint ${this.url}`,
+      };
+    }
+
+    if (response.status === 401) {
+      const body = await safeBodyFromResponse(response, ErrorWithStackSchema);
+
+      if (body) {
+        return {
+          ok: false,
+          error: body.message,
+        } as const;
+      }
+
+      return {
+        ok: false,
+        error: `Trigger API key is invalid`,
+      } as const;
+    }
+
+    if (!response.ok) {
+      return {
+        ok: false,
+        error: `Could not connect to endpoint ${this.url}. Status code: ${response.status}`,
+      };
+    }
+
+    const validateResponse = await safeParseBodyFromResponse(response, ValidateResponseSchema);
+
+    if (!validateResponse) {
+      return {
+        ok: false,
+        error: `Could not parse response from endpoint. Make sure it points to the correct URL (you might be missing /api/trigger)`,
+      };
+    }
+
+    if (!validateResponse.success) {
+      return {
+        ok: false,
+        error: `Endpoint ${this.url} responded with error: ${validateResponse.error.message}`,
+      };
+    }
+
+    return validateResponse.data;
+  }
 }
 
 async function safeFetch(url: string, options: RequestInit) {

apps/webapp/app/services/endpoints/createEndpoint.server.ts

@@ -34,9 +34,9 @@ export class CreateEndpointService {
   }) {
     const endpointUrl = this.#normalizeEndpointUrl(url);
 
-    const client = new EndpointApi(environment.apiKey, endpointUrl, id);
+    const client = new EndpointApi(environment.apiKey, endpointUrl);
 
-    const pong = await client.ping();
+    const pong = await client.ping(id);
 
     if (!pong.ok) {
       throw new CreateEndpointError("FAILED_PING", pong.error);

apps/webapp/app/services/endpoints/indexEndpoint.server.ts

@@ -28,7 +28,7 @@ export class IndexEndpointService {
     const endpoint = await findEndpoint(id);
 
     // Make a request to the endpoint to fetch a list of jobs
-    const client = new EndpointApi(endpoint.environment.apiKey, endpoint.url, endpoint.slug);
+    const client = new EndpointApi(endpoint.environment.apiKey, endpoint.url);
 
     const indexResponse = await client.indexEndpoint();
 

apps/webapp/app/services/endpoints/validateCreateEndpoint.server.ts

@@ -0,0 +1,100 @@
+import { customAlphabet } from "nanoid";
+import { $transaction, prisma, PrismaClient } from "~/db.server";
+import { env } from "~/env.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { workerQueue } from "../worker.server";
+import { CreateEndpointError } from "./createEndpoint.server";
+import { EndpointApi } from "../endpointApi.server";
+
+const indexingHookIdentifier = customAlphabet("0123456789abcdefghijklmnopqrstuvxyz", 10);
+
+export class ValidateCreateEndpointService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call({ environment, url }: { environment: AuthenticatedEnvironment; url: string }) {
+    const endpointUrl = this.#normalizeEndpointUrl(url);
+
+    const client = new EndpointApi(environment.apiKey, endpointUrl);
+
+    const validationResult = await client.validate();
+
+    if (!validationResult.ok) {
+      throw new Error(validationResult.error);
+    }
+
+    try {
+      const result = await $transaction(this.#prismaClient, async (tx) => {
+        const endpoint = await tx.endpoint.upsert({
+          where: {
+            environmentId_slug: {
+              environmentId: environment.id,
+              slug: validationResult.endpointId,
+            },
+          },
+          create: {
+            environment: {
+              connect: {
+                id: environment.id,
+              },
+            },
+            organization: {
+              connect: {
+                id: environment.organizationId,
+              },
+            },
+            project: {
+              connect: {
+                id: environment.projectId,
+              },
+            },
+            slug: validationResult.endpointId,
+            url: endpointUrl,
+            indexingHookIdentifier: indexingHookIdentifier(),
+          },
+          update: {
+            url: endpointUrl,
+          },
+        });
+
+        // Kick off process to fetch the jobs for this endpoint
+        await workerQueue.enqueue(
+          "indexEndpoint",
+          {
+            id: endpoint.id,
+            source: "INTERNAL",
+          },
+          { tx }
+        );
+
+        return endpoint;
+      });
+
+      return result;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new CreateEndpointError("FAILED_UPSERT", error.message);
+      } else {
+        throw new CreateEndpointError("FAILED_UPSERT", "Something went wrong");
+      }
+    }
+  }
+
+  // If the endpoint URL points to localhost, and the RUNTIME_PLATFORM is docker-compose, then we need to rewrite the host to host.docker.internal
+  // otherwise we shouldn't change anything
+  #normalizeEndpointUrl(url: string) {
+    if (env.RUNTIME_PLATFORM === "docker-compose") {
+      const urlObj = new URL(url);
+
+      if (urlObj.hostname === "localhost") {
+        urlObj.hostname = "host.docker.internal";
+        return urlObj.toString();
+      }
+    }
+
+    return url;
+  }
+}

apps/webapp/app/services/runs/performRunExecution.server.ts

@@ -54,7 +54,7 @@ export class PerformRunExecutionService {
   async #executePreprocessing(execution: FoundRunExecution) {
     const { run } = execution;
 
-    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url, run.endpoint.slug);
+    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
     const event = ApiEventLogSchema.parse({ ...run.event, id: run.eventId });
     const startedAt = new Date();
 
@@ -189,7 +189,7 @@ export class PerformRunExecutionService {
       return;
     }
 
-    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url, run.endpoint.slug);
+    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
     const event = ApiEventLogSchema.parse({ ...run.event, id: run.eventId });
 
     const startedAt = new Date();

apps/webapp/app/services/sources/deliverHttpSourceRequest.server.ts

@@ -55,8 +55,7 @@ export class DeliverHttpSourceRequestService {
 
     const clientApi = new EndpointApi(
       httpSourceRequest.environment.apiKey,
-      httpSourceRequest.endpoint.url,
-      httpSourceRequest.endpoint.slug
+      httpSourceRequest.endpoint.url
     );
 
     const { response, events } = await clientApi.deliverHttpSourceRequest({

apps/webapp/app/services/triggers/initializeTrigger.server.ts

@@ -45,7 +45,7 @@ export class InitializeTriggerService {
       },
     });
 
-    const clientApi = new EndpointApi(environment.apiKey, endpoint.url, endpoint.slug);
+    const clientApi = new EndpointApi(environment.apiKey, endpoint.url);
 
     const registerMetadata = await clientApi.initializeTrigger(dynamicTrigger.slug, payload.params);
 

packages/core/src/schemas/api.ts

@@ -128,6 +128,23 @@ export const PongResponseSchema = z.discriminatedUnion("ok", [
 
 export type PongResponse = z.infer<typeof PongResponseSchema>;
 
+export const ValidateSuccessResponseSchema = z.object({
+  ok: z.literal(true),
+  endpointId: z.string(),
+});
+
+export const ValidateErrorResponseSchema = z.object({
+  ok: z.literal(false),
+  error: z.string(),
+});
+
+export const ValidateResponseSchema = z.discriminatedUnion("ok", [
+  ValidateSuccessResponseSchema,
+  ValidateErrorResponseSchema,
+]);
+
+export type ValidateResponse = z.infer<typeof ValidateResponseSchema>;
+
 export const QueueOptionsSchema = z.object({
   name: z.string(),
   maxConcurrent: z.number().optional(),

packages/trigger-sdk/src/triggerClient.ts

@@ -389,6 +389,15 @@ export class TriggerClient {
           },
         };
       }
+      case "VALIDATE": {
+        return {
+          status: 200,
+          body: {
+            ok: true,
+            endpointId: this.id,
+          },
+        };
+      }
     }
 
     return {