Make backend-connectivity failures resilient and observable

Frontend degraded badly and silently whenever the backend was slow, still
booting, or briefly down: tool operations dropped a dead-end "Network error"
toast, the auth screens showed a branded "Backend not found" page after a
single probe, and none of these failures were ever captured as exceptions, so
their true frequency was invisible in error tracking.

- Tool operation POSTs now auto-retry a few times on no-status network
  failures before surfacing the toast (new postWithNetworkRetry helper);
  intermediate attempts are silent, only the final failure reaches the user.
- useBackendProbe auto-polls with backoff and distinguishes a still-"starting"
  backend (reassuring "starting up" screen that auto-refreshes) from a genuinely
  "down" one (error + Retry), replacing the per-screen 5s interval polling.
- Backend-connectivity failures (no-response network errors and 5xx) are
  forwarded to error tracking via posthog.captureException so they stop being
  toast-only and invisible.

Generated-By: PostHog Code
Task-Id: 808d576d-2e8f-4be2-b2e9-7f17a6ab1d8b

frontend/editor/public/locales/en-US/translation.toml

@@ -2016,6 +2016,8 @@ wait = "Please wait for the backend to finish launching and try again."
 [backendStartup]
 notFoundTitle = "Backend not found"
 retry = "Retry"
+startingMessage = "The backend is still starting up. This can take a moment on first launch — this screen will refresh automatically."
+startingTitle = "Starting up…"
 unreachable = "The application cannot currently connect to the backend. Verify the backend status and network connectivity, then try again."
 
 [billing]

frontend/editor/src/core/hooks/tools/shared/useToolApiCalls.ts

@@ -1,6 +1,6 @@
 import { useCallback, useRef } from "react";
 import axios, { type CancelTokenSource } from "axios"; // Real axios for static methods (CancelToken, isCancel)
-import apiClient from "@app/services/apiClient"; // Our configured instance
+import { postWithNetworkRetry } from "@app/services/networkRetry";
 import {
   processResponse,
   ResponseHandler,
@@ -65,7 +65,7 @@ export const useToolApiCalls = <TParams = void>() => {
         try {
           const formData = config.buildFormData(params, file);
           console.debug("[processFiles] POST", { endpoint, name: file.name });
-          const response = await apiClient.post(endpoint, formData, {
+          const response = await postWithNetworkRetry(endpoint, formData, {
             responseType: "blob",
             cancelToken: cancelTokenRef.current?.token,
           });

frontend/editor/src/core/hooks/tools/shared/useToolOperation.ts

@@ -1,5 +1,5 @@
 import { useCallback, useRef, useEffect, useContext } from "react";
-import apiClient from "@app/services/apiClient";
+import { postWithNetworkRetry } from "@app/services/networkRetry";
 import { useTranslation } from "react-i18next";
 import { useFileContext } from "@app/contexts/FileContext";
 import { useNavigationActions } from "@app/contexts/NavigationContext";
@@ -280,7 +280,7 @@ export const useToolOperation = <TParams>(
               );
             }
 
-            const response = await apiClient.post(endpoint, formData, {
+            const response = await postWithNetworkRetry(endpoint, formData, {
               responseType: "blob",
             });
 

frontend/editor/src/core/hooks/useBackendProbe.ts

@@ -1,5 +1,6 @@
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { BASE_PATH } from "@app/constants/app";
+import { captureNetworkError } from "@app/services/analytics";
 
 type BackendStatus = "up" | "starting" | "down";
 
@@ -9,9 +10,29 @@ interface BackendProbeState {
   loading: boolean;
 }
 
+// A single probe's raw verdict, before the startup grace window is applied.
+// "unreachable" means the request never completed (backend slow/booting/down);
+// "starting" means it answered but isn't ready yet.
+type ProbeVerdict = "up" | "starting" | "unreachable";
+
+// Keep treating an unreachable backend as "starting" (a reassuring "still
+// starting up" screen) for this long before declaring it genuinely "down".
+const STARTUP_GRACE_MS = 20_000;
+// Auto-poll backoff bounds while the backend isn't up yet. Fast at first so the
+// screen advances quickly when the backend finishes booting, then slower so we
+// keep polling for recovery without hammering the network.
+const MIN_POLL_DELAY_MS = 1_000;
+const MAX_POLL_DELAY_MS = 8_000;
+
 /**
  * Lightweight backend probe that avoids global axios interceptors.
- * Used on auth screens to decide whether to show login, anonymous mode, or a backend-starting message.
+ * Used on auth screens to decide whether to show login, anonymous mode, a
+ * "backend starting up" message, or a "backend unreachable" error.
+ *
+ * The probe auto-polls with backoff: an unreachable backend stays "starting"
+ * during a grace window (and while still booting) before it is reported as
+ * "down", so a slow or restarting backend no longer lands users on a dead-end
+ * error screen. Polling continues after "down" so the screen auto-recovers.
  */
 export function useBackendProbe() {
   const [state, setState] = useState<BackendProbeState>({
@@ -20,63 +41,128 @@ export function useBackendProbe() {
     loading: true,
   });
 
-  const probe = useCallback(async () => {
+  // Start of the current not-up streak, used to apply the startup grace window.
+  const streakStartRef = useRef<number | null>(null);
+  // Ensure we only forward a genuine "down" to error tracking once per streak.
+  const capturedDownRef = useRef(false);
+
+  const probeOnce = useCallback(async (): Promise<{
+    verdict: ProbeVerdict;
+    loginDisabled: boolean;
+  }> => {
     const statusUrl = `${BASE_PATH || ""}/api/v1/info/status`;
     const loginUrl = `${BASE_PATH || ""}/api/v1/proprietary/ui-data/login`;
 
-    const next: BackendProbeState = {
-      status: "starting",
-      loginDisabled: false,
-      loading: false,
-    };
+    let verdict: ProbeVerdict;
+    let loginDisabled = false;
 
     try {
       const res = await fetch(statusUrl, { method: "GET", cache: "no-store" });
       if (res.ok) {
         const data = await res.json().catch(() => null);
-        if (data && data.status === "UP") {
-          next.status = "up";
-          setState(next);
-          return next;
-        }
-        next.status = "starting";
+        verdict = data && data.status === "UP" ? "up" : "starting";
       } else if (res.status === 404 || res.status === 503) {
-        next.status = "starting";
+        verdict = "starting";
       } else {
-        next.status = "down";
+        verdict = "unreachable";
       }
     } catch {
-      next.status = "down";
+      verdict = "unreachable";
+    }
+
+    if (verdict === "up") {
+      return { verdict, loginDisabled };
     }
 
     // Fallback: proprietary login endpoint to detect disabled login and backend availability
     try {
       const res = await fetch(loginUrl, { method: "GET", cache: "no-store" });
       if (res.ok) {
-        next.status = "up";
+        verdict = "up";
         const data = await res.json().catch(() => null);
         if (data && data.enableLogin === false) {
-          next.loginDisabled = true;
+          loginDisabled = true;
         }
       } else if (res.status === 404) {
         // Endpoint missing usually means login disabled
-        next.status = "up";
-        next.loginDisabled = true;
+        verdict = "up";
+        loginDisabled = true;
       } else if (res.status === 503) {
-        next.status = "starting";
+        verdict = "starting";
       } else {
-        next.status = "down";
+        verdict = "unreachable";
       }
     } catch {
-      // keep previous inferred state (down/starting)
+      // keep previous inferred verdict (unreachable/starting)
     }
 
-    setState(next);
-    return next;
+    return { verdict, loginDisabled };
   }, []);
 
+  const probe = useCallback(async (): Promise<BackendProbeState> => {
+    const { verdict, loginDisabled } = await probeOnce();
+
+    if (verdict === "up") {
+      streakStartRef.current = null;
+      capturedDownRef.current = false;
+      const next: BackendProbeState = {
+        status: "up",
+        loginDisabled,
+        loading: false,
+      };
+      setState(next);
+      return next;
+    }
+
+    // Not up: start (or continue) the not-up streak.
+    const now = Date.now();
+    if (streakStartRef.current == null) {
+      streakStartRef.current = now;
+    }
+    const elapsed = now - streakStartRef.current;
+
+    // A reachable-but-booting backend is always "starting". An unreachable one
+    // is optimistically "starting" during the grace window, then "down".
+    let status: BackendStatus = "starting";
+    if (verdict === "unreachable" && elapsed >= STARTUP_GRACE_MS) {
+      status = "down";
+      if (!capturedDownRef.current) {
+        capturedDownRef.current = true;
+        captureNetworkError(new Error("Backend unreachable"), {
+          endpoint: `${BASE_PATH || ""}/api/v1/info/status`,
+          status: null,
+          context: "backend_probe",
+        });
+      }
+    }
+
+    const next: BackendProbeState = { status, loginDisabled, loading: false };
+    setState(next);
+    return next;
+  }, [probeOnce]);
+
+  // Auto-poll with backoff until the backend is up (or login is disabled, which
+  // means we can proceed anonymously). Keep polling after "down" so the screen
+  // recovers on its own once the backend comes back.
   useEffect(() => {
-    void probe();
+    let cancelled = false;
+    let timer: number | undefined;
+    let delay = MIN_POLL_DELAY_MS;
+
+    const loop = async () => {
+      const result = await probe();
+      if (cancelled) return;
+      if (result.status === "up" || result.loginDisabled) return;
+      delay = Math.min(delay * 2, MAX_POLL_DELAY_MS);
+      timer = window.setTimeout(() => void loop(), delay);
+    };
+
+    void loop();
+
+    return () => {
+      cancelled = true;
+      if (timer) window.clearTimeout(timer);
+    };
   }, [probe]);
 
   return {

frontend/editor/src/core/services/analytics.ts

@@ -38,3 +38,24 @@ export function trackEditorOperation(toolId: string, fileCount: number): void {
     if (DEV) console.warn("[analytics] trackEditorOperation failed", error);
   }
 }
+
+/**
+ * Forward a backend-connectivity failure (no-response network error or 5xx) to
+ * error tracking. These otherwise only ever surface as a toast, so error
+ * tracking never sees them and their true frequency stays invisible.
+ */
+export function captureNetworkError(
+  error: unknown,
+  context: Record<string, unknown> = {},
+): void {
+  try {
+    if (!canCapture()) return;
+    const err = error instanceof Error ? error : new Error(String(error));
+    posthog.captureException(err, {
+      $exception_source: "backend_connectivity",
+      ...context,
+    });
+  } catch (e) {
+    if (DEV) console.warn("[analytics] captureNetworkError failed", error, e);
+  }
+}

frontend/editor/src/core/services/httpErrorHandler.ts

@@ -12,6 +12,7 @@ import {
   extractAxiosErrorMessage,
 } from "@app/services/httpErrorUtils";
 import { withBasePath } from "@app/constants/app";
+import { captureNetworkError } from "@app/services/analytics";
 
 // Module-scoped state to reduce global variable usage
 const recentSpecialByEndpoint: Record<string, number> = {};
@@ -147,6 +148,17 @@ export async function handleHttpError(error: any): Promise<boolean> {
 
   if (handleSaaSError(error)) return true;
 
+  // Forward the network-failure class — a request that never got a response
+  // (no status) or a 5xx — to error tracking. These otherwise only surface as a
+  // toast, so error tracking never sees them and their real frequency is hidden.
+  if (!status || status >= 500) {
+    captureNetworkError(error, {
+      endpoint: error?.config?.url,
+      method: error?.config?.method,
+      status: status ?? null,
+    });
+  }
+
   // Compute title/body (friendly) from the error object
   const { title, body } = extractAxiosErrorMessage(error);
 

frontend/editor/src/core/services/networkRetry.ts

@@ -0,0 +1,58 @@
+import axios from "axios";
+import apiClient from "@app/services/apiClient";
+
+// Follow whatever HTTP client this build resolves @app/services/apiClient to
+// (axios on web, the Tauri client on desktop) so the helper is build-agnostic.
+// The response type is left to inference from the apiClient.post call itself —
+// ReturnType<> can't recover it because axios's post generics stay uninstantiated.
+type PostConfig = NonNullable<Parameters<typeof apiClient.post>[2]>;
+
+// Bounded auto-retry for tool operation requests that fail with no HTTP status
+// (the request never got a response — backend slow, still booting, or briefly
+// down). We retry transparently a few times before letting the failure surface
+// to the user, so a momentary blip no longer becomes a dead-end "Network error".
+const MAX_ATTEMPTS = 3; // 1 initial attempt + 2 retries
+const BASE_RETRY_DELAY_MS = 800;
+
+function wait(ms: number): Promise<void> {
+  return new Promise((resolve) => window.setTimeout(resolve, ms));
+}
+
+/**
+ * A no-status network failure: an axios error with no response at all. HTTP
+ * errors (4xx/5xx) carry a response and a meaningful body, so retrying them is
+ * pointless (422) or the server's job to fix (5xx) — only genuine
+ * never-got-a-response failures are retried here.
+ */
+function isNoStatusNetworkError(error: unknown): boolean {
+  return axios.isAxiosError(error) && !error.response && !axios.isCancel(error);
+}
+
+/**
+ * POST with a bounded auto-retry on no-status network failures. Intermediate
+ * attempts suppress the global error toast/capture (via suppressErrorToast) so
+ * only the final failure surfaces to the user and error tracking.
+ */
+export async function postWithNetworkRetry(
+  url: string,
+  data: unknown,
+  config: PostConfig = {},
+) {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    const isFinalAttempt = attempt === MAX_ATTEMPTS;
+    try {
+      return await apiClient.post(url, data, {
+        ...config,
+        ...(isFinalAttempt ? {} : { suppressErrorToast: true }),
+      });
+    } catch (error) {
+      lastError = error;
+      if (isFinalAttempt || !isNoStatusNetworkError(error)) {
+        throw error;
+      }
+      await wait(BASE_RETRY_DELAY_MS * attempt);
+    }
+  }
+  throw lastError;
+}

frontend/editor/src/proprietary/routes/Landing.tsx

@@ -49,37 +49,16 @@ export default function Landing() {
     session,
   ]);
 
-  // Periodically probe while backend isn't up so the screen can auto-advance when it comes online
-  useEffect(() => {
-    if (backendProbe.status === "up" || backendProbe.loginDisabled) {
-      return;
-    }
-    const tick = async () => {
-      const result = await backendProbe.probe();
-      if (result.status === "up") {
-        await refetch();
-        if (result.loginDisabled) {
-          navigate("/", { replace: true });
-        }
-      }
-    };
-    const intervalId = window.setInterval(() => {
-      void tick();
-    }, 5000);
-    return () => window.clearInterval(intervalId);
-  }, [
-    backendProbe.status,
-    backendProbe.loginDisabled,
-    backendProbe.probe,
-    navigate,
-    refetch,
-  ]);
-
+  // useBackendProbe auto-polls with backoff, so the screen advances on its own
+  // when the backend comes online; just refetch config once it's up.
   useEffect(() => {
     if (backendProbe.status === "up") {
       void refetch();
+      if (backendProbe.loginDisabled) {
+        navigate("/", { replace: true });
+      }
     }
-  }, [backendProbe.status, refetch]);
+  }, [backendProbe.status, backendProbe.loginDisabled, refetch, navigate]);
 
   console.log("[Landing] ════════════════════════════════════");
   console.log("[Landing] Render state:", {
@@ -123,9 +102,23 @@ export default function Landing() {
     return <HomePage />;
   }
 
-  // If backend is not up yet and user is not authenticated, show a branded status screen
+  // If backend is not up yet and user is not authenticated, show a branded status screen.
+  // A backend that is still booting shows a reassuring "starting up" state (the probe
+  // auto-refreshes); a genuinely unreachable backend shows the error with a Retry button.
   if (!session && backendProbe.status !== "up") {
-    const backendTitle = t("backendStartup.notFoundTitle", "Backend not found");
+    const isStarting = backendProbe.status === "starting";
+    const backendTitle = isStarting
+      ? t("backendStartup.startingTitle", "Starting up…")
+      : t("backendStartup.notFoundTitle", "Backend not found");
+    const message = isStarting
+      ? t(
+          "backendStartup.startingMessage",
+          "The backend is still starting up. This can take a moment on first launch — this screen will refresh automatically.",
+        )
+      : t(
+          "backendStartup.unreachable",
+          "The application cannot currently connect to the backend. Verify the backend status and network connectivity, then try again.",
+        );
     const handleRetry = async () => {
       const result = await backendProbe.probe();
       if (result.status === "up") {
@@ -147,19 +140,23 @@ export default function Landing() {
           }}
         >
           <p style={{ margin: "0 0 0.75rem 0", color: "var(--text-primary)" }}>
-            {t(
-              "backendStartup.unreachable",
-              "The application cannot currently connect to the backend. Verify the backend status and network connectivity, then try again.",
-            )}
+            {message}
           </p>
-          <Button
-            type="button"
-            onClick={handleRetry}
-            className="auth-cta-button px-4 py-[0.75rem] rounded-[0.625rem] text-base font-semibold mt-5 border-0 cursor-pointer"
-            style={{ width: "fit-content" }}
-          >
-            {t("backendStartup.retry", "Retry")}
-          </Button>
+          {isStarting ? (
+            <div
+              className="animate-spin rounded-full h-6 w-6 border-b-2 border-blue-600 mt-4"
+              aria-hidden="true"
+            />
+          ) : (
+            <Button
+              type="button"
+              onClick={handleRetry}
+              className="auth-cta-button px-4 py-[0.75rem] rounded-[0.625rem] text-base font-semibold mt-5 border-0 cursor-pointer"
+              style={{ width: "fit-content" }}
+            >
+              {t("backendStartup.retry", "Retry")}
+            </Button>
+          )}
         </div>
       </AuthLayout>
     );

frontend/editor/src/proprietary/routes/Login.tsx

@@ -172,32 +172,8 @@ export default function Login() {
   const isUserPassAllowed = login.isUserPassAllowed;
   const isSsoOnlyMode = !login.isUserPassAllowed;
 
-  // Periodically probe while backend isn't up so the screen can auto-advance when it comes online
-  useEffect(() => {
-    if (backendProbe.status === "up" || backendProbe.loginDisabled) {
-      return;
-    }
-    const tick = async () => {
-      const result = await backendProbe.probe();
-      if (result.status === "up") {
-        await refetch();
-        if (loginDisabled) {
-          navigate("/", { replace: true });
-        }
-      }
-    };
-    const intervalId = window.setInterval(() => {
-      void tick();
-    }, 5000);
-    return () => window.clearInterval(intervalId);
-  }, [
-    backendProbe.status,
-    backendProbe.loginDisabled,
-    backendProbe.probe,
-    refetch,
-    navigate,
-    loginDisabled,
-  ]);
+  // useBackendProbe auto-polls with backoff, so the screen advances on its own
+  // when the backend comes online; config refetch on "up" is handled below.
 
   // Redirect immediately if user has valid session (JWT already validated by AuthProvider)
   useEffect(() => {
@@ -410,8 +386,20 @@ export default function Login() {
     return <LoggedInState />;
   }
 
-  // If backend isn't ready yet, show a lightweight status screen instead of the form
+  // If backend isn't ready yet, show a lightweight status screen instead of the form.
+  // A booting backend shows a reassuring "starting up" state (the probe auto-refreshes);
+  // a genuinely unreachable backend shows the error with a Retry button.
   if (backendProbe.status !== "up" && !loginDisabled) {
+    const isStarting = backendProbe.status === "starting";
+    const message = isStarting
+      ? t(
+          "backendStartup.startingMessage",
+          "The backend is still starting up. This can take a moment on first launch — this screen will refresh automatically.",
+        )
+      : t(
+          "backendStartup.unreachable",
+          "The application cannot currently connect to the backend. Verify the backend status and network connectivity, then try again.",
+        );
     const handleRetry = async () => {
       const result = await backendProbe.probe();
       if (result.status === "up") {
@@ -444,19 +432,23 @@ export default function Login() {
           }}
         >
           <p style={{ margin: "0 0 0.75rem 0", color: "var(--text-primary)" }}>
-            {t(
-              "backendStartup.unreachable",
-              "The application cannot currently connect to the backend. Verify the backend status and network connectivity, then try again.",
-            )}
+            {message}
           </p>
-          <Button
-            type="button"
-            onClick={handleRetry}
-            className="auth-cta-button px-4 py-[0.75rem] rounded-[0.625rem] text-base font-semibold mt-5 border-0 cursor-pointer"
-            style={{ width: "fit-content" }}
-          >
-            {t("backendStartup.retry", "Retry")}
-          </Button>
+          {isStarting ? (
+            <div
+              className="animate-spin rounded-full h-6 w-6 border-b-2 border-blue-600 mt-4"
+              aria-hidden="true"
+            />
+          ) : (
+            <Button
+              type="button"
+              onClick={handleRetry}
+              className="auth-cta-button px-4 py-[0.75rem] rounded-[0.625rem] text-base font-semibold mt-5 border-0 cursor-pointer"
+              style={{ width: "fit-content" }}
+            >
+              {t("backendStartup.retry", "Retry")}
+            </Button>
+          )}
         </div>
       </AuthLayout>
     );