From 6961004a7392713bbfc0142e4dff0373523c3fab Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Tue, 2 Jun 2026 16:19:51 +0100
Subject: [PATCH] fix(webapp): restore Postgres fallback for non-ClickHouse
 OTLP spans (#3803)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Problem

On environments where runs carry a Postgres-backed `taskEventStore`
value (`taskEvent` or `taskEventPartitioned`), OTLP ingest endpoints
(`POST /otel/v1/traces` and `/otel/v1/logs`) were returning HTTP 500.

**Root cause:** The org-scoped ClickHouse factory introduced in a recent
PR routes all OTLP spans through `getEventRepositoryForOrganizationSync`
→ `buildEventRepository`. That function only handles `"clickhouse"` and
`"clickhouse_v2"` store values and throws `Unknown ClickHouse event
repository store: <value>` for anything else. The throw occurred inside
the grouping loop of `#exportEvents`, unwinding the entire method and
returning 500 for the whole batch.

The OpenTelemetry collector's `otlphttp` exporter treats HTTP 500 as
non-retryable and drops the batch — causing real span loss.

**Fix:** Guard the `getEventRepositoryForOrganizationSync` call in
`#exportEvents` so it is only invoked for `clickhouse` / `clickhouse_v2`
store values. All other values are routed directly to the Postgres
`eventRepository`, matching the guard pattern already present in
`resolveEventRepositoryForStore` and `getEventRepositoryForStore` in
`eventRepository/index.server.ts`.

The ClickHouse factory call is also wrapped in a try/catch that falls
back to Postgres so any unexpected store value in a future OTLP batch
degrades gracefully instead of failing the whole request.

## Changes

- `apps/webapp/app/v3/otlpExporter.server.ts` — add Postgres routing
guard and try/catch fallback in `#exportEvents`

## Testing

The `eventRepository/index.server.ts` module already has the same guard
pattern thoroughly covered. The fix brings `#exportEvents` into
alignment with that existing, tested pattern. Manual verification:
confirm OTLP batches containing Postgres-store spans return 200 and
route to the correct repository.
---
 .server-changes/otlp-postgres-store-fallback.md |  6 ++++++
 apps/webapp/app/v3/otlpExporter.server.ts       | 17 +++++++++++++----
 2 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 .server-changes/otlp-postgres-store-fallback.md

diff --git a/.server-changes/otlp-postgres-store-fallback.md b/.server-changes/otlp-postgres-store-fallback.md
new file mode 100644
index 000000000..ca0e5c9f7
--- /dev/null
+++ b/.server-changes/otlp-postgres-store-fallback.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Fixes OTLP ingest endpoints returning HTTP 500 for runs on environments that use a Postgres-backed task event store. This caused the OpenTelemetry collector to drop entire span batches as non-retryable, resulting in real span loss.
diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts
index 788e73398..975e4aed4 100644
--- a/apps/webapp/app/v3/otlpExporter.server.ts
+++ b/apps/webapp/app/v3/otlpExporter.server.ts
@@ -24,6 +24,7 @@ import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.
 import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
 
 import { generateSpanId } from "./eventRepository/common.server";
+import { eventRepository } from "./eventRepository/eventRepository.server";
 import type {
   CreatableEventKind,
   CreatableEventStatus,
@@ -120,10 +121,18 @@ class OTLPExporter {
         const routeKey = `${event.organizationId}\0${taskEventStore}`;
         let resolved = routeCache.get(routeKey);
         if (!resolved) {
-          resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
-            taskEventStore,
-            event.organizationId
-          );
+          // Non-ClickHouse stores (taskEvent / taskEventPartitioned) are Postgres-backed.
+          // The ClickHouse factory only handles clickhouse/clickhouse_v2 and throws otherwise.
+          if (taskEventStore !== "clickhouse" && taskEventStore !== "clickhouse_v2") {
+            // Non-ClickHouse stores (taskEvent / taskEventPartitioned) are Postgres-backed.
+            // The ClickHouse factory only handles clickhouse/clickhouse_v2 and throws otherwise.
+            resolved = { key: "postgres:default", repository: eventRepository };
+          } else {
+            resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
+              taskEventStore,
+              event.organizationId
+            );
+          }
           routeCache.set(routeKey, resolved);
         }