From dba4313c5c373bb44abbd42eeebd64ad628eff38 Mon Sep 17 00:00:00 2001
From: Eric Allam <eallam@icloud.com>
Date: Wed, 19 Jun 2024 14:22:06 +0100
Subject: [PATCH] v3: more visibility into flushing worker otel data in prod

---
 .../cli-v3/src/workers/prod/worker-facade.ts  | 40 ++++++++++++++++++-
 .../cli-v3/src/workers/prod/worker-setup.ts   |  4 +-
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/packages/cli-v3/src/workers/prod/worker-facade.ts b/packages/cli-v3/src/workers/prod/worker-facade.ts
index c8876237c..aa014ef8a 100644
--- a/packages/cli-v3/src/workers/prod/worker-facade.ts
+++ b/packages/cli-v3/src/workers/prod/worker-facade.ts
@@ -202,18 +202,54 @@ const zodIpc = new ZodIpcConnection({
     },
     CLEANUP: async ({ flush, kill }, sender) => {
       if (kill) {
-        await Promise.all([prodUsageManager.flush(), tracingSDK.flush()]);
+        await flushAll();
         // Now we need to exit the process
         await sender.send("READY_TO_DISPOSE", undefined);
       } else {
         if (flush) {
-          await Promise.all([prodUsageManager.flush(), tracingSDK.flush()]);
+          await flushAll();
         }
       }
     },
   },
 });
 
+async function flushAll(timeoutInMs: number = 10_000) {
+  const now = performance.now();
+
+  console.log(`Flushing at ${now}`);
+
+  await Promise.all([flushUsage(), flushTracingSDK()]);
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed in ${duration}ms`);
+}
+
+async function flushUsage() {
+  const now = performance.now();
+
+  console.log(`Flushing usage at ${now}`);
+
+  await prodUsageManager.flush();
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed usage in ${duration}ms`);
+}
+
+async function flushTracingSDK() {
+  const now = performance.now();
+
+  console.log(`Flushing tracingSDK at ${now}`);
+
+  await tracingSDK.flush();
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed tracingSDK in ${duration}ms`);
+}
+
 // Ignore SIGTERM, handled by entry point
 process.on("SIGTERM", async () => {});
 
diff --git a/packages/cli-v3/src/workers/prod/worker-setup.ts b/packages/cli-v3/src/workers/prod/worker-setup.ts
index 528b37c86..47d71ff9e 100644
--- a/packages/cli-v3/src/workers/prod/worker-setup.ts
+++ b/packages/cli-v3/src/workers/prod/worker-setup.ts
@@ -16,7 +16,9 @@ export const tracingSDK = new TracingSDK({
   url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
   instrumentations: setupImportedConfig?.instrumentations ?? [],
   diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
-  forceFlushTimeoutMillis: 1_000,
+  forceFlushTimeoutMillis: process.env.OTEL_FORCE_FLUSH_TIMEOUT
+    ? parseInt(process.env.OTEL_FORCE_FLUSH_TIMEOUT, 10)
+    : 1_000,
 });
 
 export const otelTracer: Tracer = tracingSDK.getTracer("trigger-prod-worker", packageJson.version);