fix(webapp): recover from stale /build assets via a bounded reload (#4282)

## Problem

The webapp's HTML references content-hashed `/build` assets, and each
running
instance contains exactly one build and returns 404 for asset hashes it
doesn't
have. During a rolling deploy a client can hold HTML from one build
while a
request for one of its assets is served by an instance on a different
build →
missing styles or a failed chunk load.

## What this does

On a `/build` stylesheet/script/chunk load failure, the client does a
**bounded
full-document reload** (at most 2 per 5 minutes, tracked in
`sessionStorage`) so
the page reloads onto a single consistent build. That's the whole
mechanism — no
polling, no `fetch` interception, no blocking overlay, no form
snapshotting.

- `apps/webapp/app/components/StaleAssetRecovery.tsx` — authored as a
typed,
lint-checked function and serialized to an inline script via
`.toString()` (so
the logic is real, reviewable code, not an opaque string), injected
before
  `<Links />`, production only.
- Detection: capture-phase `error` listener for
`<link>`/`<script>`/modulepreload
failures under `/build/`, plus an `unhandledrejection` guard for
dynamic-import
  failures.
- Guards: once-per-page re-entrancy guard, the bounded reload budget,
and a
`navigator.onLine` check so it never reloads into an offline error page.
- Unit tests in `StaleAssetRecovery.test.ts`.

## Relationship to #4260

Replaces the recovery introduced in #4260 (reverted in #4280) with a
much
smaller, reload-only approach — the previous version intercepted `fetch`
and
could turn a data request into a navigation, and showed a full-screen
overlay on
any asset error; this drops both.

## `/build-version` compatibility shim

`apps/webapp/server.ts` adds a tiny `GET /build-version` endpoint (build
id only,
`no-store`). A previously-deployed client build polls it after an asset
failure
and reloads once it sees a newer build, so those older tabs recover in
one reload
instead of getting stuck. Temporary — safe to remove once older clients
have
cycled out. It deliberately does **not** re-add an `X-Build-Id` response
header.

## Also

Restores the `.server-changes` writing guidance in
`.claude/rules/server-apps.md`
(reverted alongside #4260).

## Self-hosting note

Recovery is most reliable when your load balancer keeps a client on one
instance
for the duration of a deploy (short session stickiness) — the reload
then lands
on a consistent build in one hop.

.claude/rules/server-apps.md

@@ -14,10 +14,12 @@ area: webapp
 type: fix
 ---
 
-Brief description of what changed and why.
+Fix pages occasionally loading unstyled during deploys. The dashboard now recovers automatically.
 EOF
 ```
 
 - **area**: `webapp` | `supervisor`
 - **type**: `feature` | `fix` | `improvement` | `breaking`
 - If the PR also touches `packages/`, just the changeset is sufficient (no `.server-changes/` needed).
+
+The body ships **verbatim in user-facing release notes**. Keep it to 1–2 short sentences, non-technical, written for a dashboard user: describe what changed for them, never the implementation (no header names, endpoints, middleware, storage mechanisms, internal tools). See `.server-changes/README.md` for full guidance.

.server-changes/stale-deploy-asset-recovery.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Fix pages occasionally loading unstyled or failing to load during a deploy. The dashboard now reloads automatically to recover.

apps/webapp/app/components/StaleAssetRecovery.test.ts

@@ -0,0 +1,56 @@
+// @vitest-environment jsdom
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { staleAssetRecoveryScript } from "./StaleAssetRecovery";
+
+// Each staleAssetRecoveryScript() call models a fresh page load: it reads the shared
+// sessionStorage budget and returns its own `recover`. We drive recover() directly rather
+// than dispatching resource-error events, so accumulated window listeners never fire.
+describe("staleAssetRecoveryScript", () => {
+  let reload: ReturnType<typeof vi.fn>;
+
+  beforeEach(() => {
+    sessionStorage.clear();
+    reload = vi.fn();
+    vi.stubGlobal("location", { reload });
+    vi.stubGlobal("navigator", { onLine: true });
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it("reloads on a recovery", () => {
+    staleAssetRecoveryScript().recover();
+    expect(reload).toHaveBeenCalledTimes(1);
+  });
+
+  it("reloads only once per page even if several assets fail (re-entrancy guard)", () => {
+    const { recover } = staleAssetRecoveryScript();
+    recover();
+    recover();
+    recover();
+    expect(reload).toHaveBeenCalledTimes(1);
+  });
+
+  it("stops reloading once the budget is spent across reloads", () => {
+    staleAssetRecoveryScript().recover(); // reload 1
+    staleAssetRecoveryScript().recover(); // reload 2
+    staleAssetRecoveryScript().recover(); // budget spent -> no reload
+    expect(reload).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not reload when offline", () => {
+    vi.stubGlobal("navigator", { onLine: false });
+    staleAssetRecoveryScript().recover();
+    expect(reload).not.toHaveBeenCalled();
+  });
+
+  it("does not reload when sessionStorage is unavailable", () => {
+    vi.spyOn(Storage.prototype, "setItem").mockImplementation(() => {
+      throw new Error("blocked");
+    });
+    staleAssetRecoveryScript().recover();
+    expect(reload).not.toHaveBeenCalled();
+  });
+});

apps/webapp/app/components/StaleAssetRecovery.tsx

@@ -0,0 +1,94 @@
+// Recovers from a rolling deploy rotating the content-hashed /build assets out from
+// under a page. Each image serves only its own build and hard-404s unknown hashes, so
+// a client can request a hash the serving replica doesn't have and get missing styles
+// or a failed asset load. On such a /build load failure we do a bounded full document
+// reload: the fresh document (and, under sticky routing, all of its assets) lands on a
+// single live build, so the asset resolves. Bounded via sessionStorage so it can never
+// loop; when the budget is spent it stops rather than reloading forever.
+//
+// Deliberately minimal — no fetch interception, no build-version polling, no server
+// build-id contract, no form snapshot, no blocking overlay.
+
+// The recovery logic runs as an inline <script> injected before <Links /> (see the
+// component below), so it must execute before the app bundle and before the stylesheet
+// can fail to load. It is authored as a normal, type-checked and lint-checked function
+// and serialized with .toString() at render time — NOT hand-written into a string — so
+// the logic is real code the compiler and linter can see. Because it is serialized, it
+// must stay fully self-contained: no imports, no references to module scope, and plain
+// ES that the bundler won't rewrite to reach a hoisted helper. It returns its `recover`
+// closure purely so the unit test can drive the logic directly (the inline IIFE that
+// runs in the browser ignores the return value).
+export function staleAssetRecoveryScript() {
+  var KEY = "trigger:assetReload";
+  var MAX_RELOADS = 2;
+  var WINDOW_MS = 300000;
+  var recovering = false;
+
+  function budgetAllows() {
+    try {
+      var raw = sessionStorage.getItem(KEY);
+      var state = raw ? (JSON.parse(raw) as { n: number; t: number }) : { n: 0, t: 0 };
+      if (Date.now() - state.t > WINDOW_MS) state = { n: 0, t: 0 };
+      if (state.n >= MAX_RELOADS) return false;
+      sessionStorage.setItem(KEY, JSON.stringify({ n: state.n + 1, t: Date.now() }));
+      return true;
+    } catch {
+      // Storage blocked (private mode / quota): can't bound reloads, so don't auto-reload.
+      return false;
+    }
+  }
+
+  function recover() {
+    // One recovery per page: a broken load fails several /build assets at once and each
+    // fires its own error event before location.reload() commits — without this guard a
+    // single incident would burn the entire reload budget.
+    if (recovering) return;
+    recovering = true;
+    // Don't reload into the browser's offline error page.
+    if (navigator.onLine === false) return;
+    if (budgetAllows()) location.reload();
+  }
+
+  // Non-bubbling resource load failures (stylesheet, modulepreload, entry <script>) at
+  // document load — the failure class nothing else covers. Capture phase is required.
+  window.addEventListener(
+    "error",
+    function (event) {
+      var el = event.target as Element | null;
+      if (!el || typeof el.tagName !== "string") return; // window/global errors have no tagName
+      var url =
+        el.tagName === "LINK"
+          ? (el as HTMLLinkElement).href
+          : el.tagName === "SCRIPT"
+            ? (el as HTMLScriptElement).src
+            : null;
+      if (url && url.indexOf("/build/") !== -1) recover();
+    },
+    true
+  );
+
+  // Raw dynamic import() failures in app code. (Remix reloads its own route chunks, so
+  // that path rarely reaches here.) The message URL isn't reliable cross-browser, so
+  // match the chunk-load error shape; the once-guard + bounded budget make a rare stray
+  // reload harmless.
+  window.addEventListener("unhandledrejection", function (event) {
+    var message = (event.reason && event.reason.message) || "";
+    if (
+      /dynamically imported module|Importing a module script failed|ChunkLoadError/i.test(message)
+    ) {
+      recover();
+    }
+  });
+
+  return { recover };
+}
+
+export function StaleAssetRecovery({ isProduction }: { isProduction: boolean }) {
+  if (!isProduction) {
+    return null;
+  }
+
+  return (
+    <script dangerouslySetInnerHTML={{ __html: `(${staleAssetRecoveryScript.toString()})()` }} />
+  );
+}

apps/webapp/app/entry.server.tsx

@@ -54,6 +54,12 @@ export default function handleRequest(
 ) {
   const url = new URL(request.url);
 
+  // Stale documents reference /build asset hashes that 404 after a deploy —
+  // always revalidate HTML. Route-set headers win.
+  if (!responseHeaders.has("Cache-Control")) {
+    responseHeaders.set("Cache-Control", "no-cache");
+  }
+
   if (url.pathname.startsWith("/login")) {
     responseHeaders.set("X-Frame-Options", "SAMEORIGIN");
     responseHeaders.set("Content-Security-Policy", "frame-ancestors 'self'");

apps/webapp/app/root.tsx

@@ -7,6 +7,7 @@ import type { ToastMessage } from "~/models/message.server";
 import { commitSession, getSession } from "~/models/message.server";
 import tailwindStylesheetUrl from "~/tailwind.css";
 import { RouteErrorDisplay } from "./components/ErrorDisplay";
+import { StaleAssetRecovery } from "./components/StaleAssetRecovery";
 import { AppContainer, MainCenteredContainer } from "./components/layout/AppLayout";
 import { ShortcutsProvider } from "./components/primitives/ShortcutsProvider";
 import { Toast } from "./components/primitives/Toast";
@@ -18,6 +19,11 @@ import { getUser } from "./services/session.server";
 import { getTimezonePreference } from "./services/preferences/uiPreferences.server";
 import { appEnvTitleTag } from "./utils";
 
+// Derived here (not inside StaleAssetRecovery) so the shared component takes
+// the flag as a prop. NODE_ENV is statically replaced in browser bundles, and
+// the ErrorBoundary can't rely on loader data.
+const isProduction = process.env.NODE_ENV === "production";
+
 export const links: LinksFunction = () => {
   return [{ rel: "stylesheet", href: tailwindStylesheetUrl }];
 };
@@ -99,6 +105,7 @@ export function ErrorBoundary() {
         <head>
           <meta charSet="utf-8" />
 
+          <StaleAssetRecovery isProduction={isProduction} />
           <Meta />
           <Links />
         </head>
@@ -125,6 +132,7 @@ export default function App() {
     <>
       <html lang="en" className="h-full" data-theme="dark">
         <head>
+          <StaleAssetRecovery isProduction={isProduction} />
           <Meta />
           <Links />
         </head>

apps/webapp/server.ts

@@ -134,6 +134,15 @@ if (ENABLE_CLUSTER && cluster.isPrimary) {
   const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
 
   if (process.env.HTTP_SERVER_DISABLED !== "true") {
+    // Back-compat shim: a previously-deployed client build polls this endpoint after a
+    // /build asset 404 and reloads once it reports a newer build id, letting those older
+    // tabs recover in a single reload. Temporary — safe to remove once older clients have
+    // churned out. Deliberately does NOT set an X-Build-Id response header.
+    app.get("/build-version", (_req, res) => {
+      res.set("Cache-Control", "no-store");
+      res.json({ version: build.assets.version });
+    });
+
     const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
     const wss: WebSocketServer | undefined = build.entry.module.wss;
     const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter;