Infinite loading guard: heartbeats + stall watchdogs on the model runs

A user got stuck on the wizard's loading screen: a hung model meant
silence, and silence was indistinguishable from work - the server's own
hard deadline is 2x OLLAMA_TIMEOUT_MS (ten minutes by default), and the
client waited on a dead stream indefinitely.

Both generate routes (deck, homework) now send an NDJSON heartbeat
every 15s, and the client guards the two quiet phases: no response at
all for 45s, and no bytes for 90s mid-stream, both surfacing the
regular error state with a Retry. The caller's abort still maps to the
friendly "cancelled" state.

Verified with a stalled route and Playwright's fake clock in the wizard
suite (93 checks), plus contrast 0 findings, mobile 124/124, chat
101/101, regression 44/44, i18n 13/13, native 33/33.
Version 0.4.0, SW -> freezmo-v69.

README.md

@@ -1411,6 +1411,15 @@ Things I chose while building, with the reasoning:
     pre-blurred wallpaper — the scene is clearly visible through the glass, and the
     pixel-contrast scan still passes.
 
+75. **A quiet run can no longer load forever.** A user got stuck on the wizard's loading screen:
+    a hung model meant silence, and silence was indistinguishable from work — the server's own hard
+    deadline is 2x `OLLAMA_TIMEOUT_MS` (ten minutes by default), and the client waited on a dead
+    stream indefinitely. Both generate routes (deck, homework) now send an NDJSON heartbeat every
+    15s, and the client guards the two quiet phases: no response at all for 45s, and no bytes for
+    90s mid-stream, both surfacing the regular error state with a Retry — verified with a stalled
+    route and Playwright's fake clock in the wizard suite (93 checks). The heartbeat also keeps
+    proxies from closing an idle connection.
+
 74. **Phones get flat glass, not live blur.** The touch fallback from 73 (a real backdrop blur
     over the live scene) turned out to be the wrong trade: every pane re-blurs on every frame
     while the bubbles animate, and phones choke — input lag, "buttons take ages to respond". Touch

app/api/generate/route.ts

@@ -39,7 +39,8 @@ const requestSchema = z.object({
 type StreamEvent =
   | { type: "stage"; stage: PipelineStage; message: string }
   | { type: "deck"; deck: unknown; meta: Record<string, unknown> }
-  | { type: "error"; error: string; kind?: string };
+  | { type: "error"; error: string; kind?: string }
+  | { type: "ping" };
 
 function encoder() {
   const enc = new TextEncoder();
@@ -112,6 +113,10 @@ export async function POST(request: Request) {
   const abort = new AbortController();
   // A single hard deadline for the whole pipeline (two sequential model calls).
   const globalTimer = setTimeout(() => abort.abort(), config.timeoutMs * 2);
+  // Heartbeats: one quiet model call can run for minutes, and the client's
+  // stall guard needs to tell "slow" from "dead" — a silent stream is how the
+  // wizard ended up loading forever. Cleared with the stream either way.
+  let heartbeat: ReturnType<typeof setInterval> | null = null;
 
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
@@ -123,6 +128,7 @@ export async function POST(request: Request) {
         }
       };
 
+      heartbeat = setInterval(() => send({ type: "ping" }), 15_000);
       try {
         let sourceText: string;
         let deckTitle: string | undefined;
@@ -206,6 +212,7 @@ export async function POST(request: Request) {
           kind: error instanceof OllamaError ? error.kind : "unknown",
         });
       } finally {
+        if (heartbeat) clearInterval(heartbeat);
         clearTimeout(globalTimer);
         gate.gate.release();
         try {
@@ -216,6 +223,7 @@ export async function POST(request: Request) {
       }
     },
     cancel() {
+      if (heartbeat) clearInterval(heartbeat);
       clearTimeout(globalTimer);
       gate.gate.release();
       abort.abort();

app/api/homework/route.ts

@@ -34,7 +34,8 @@ const requestSchema = z.object({
 type StreamEvent =
   | { type: "stage"; stage: PipelineStage; message: string }
   | { type: "homework"; homework: unknown; meta: Record<string, unknown> }
-  | { type: "error"; error: string; kind?: string };
+  | { type: "error"; error: string; kind?: string }
+  | { type: "ping" };
 
 function encoder() {
   const enc = new TextEncoder();
@@ -105,6 +106,8 @@ export async function POST(request: Request) {
   const abort = new AbortController();
   // A single hard deadline for the whole pipeline (two sequential model calls).
   const globalTimer = setTimeout(() => abort.abort(), config.timeoutMs * 2);
+  // Heartbeats so the client's stall guard can tell "slow" from "dead".
+  let heartbeat: ReturnType<typeof setInterval> | null = null;
 
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
@@ -115,6 +118,7 @@ export async function POST(request: Request) {
           /* client went away */
         }
       };
+      heartbeat = setInterval(() => send({ type: "ping" }), 15_000);
 
       try {
         let assignmentText: string;
@@ -262,6 +266,7 @@ export async function POST(request: Request) {
           kind: error instanceof OllamaError ? error.kind : "unknown",
         });
       } finally {
+        if (heartbeat) clearInterval(heartbeat);
         clearTimeout(globalTimer);
         gate.gate.release();
         try {
@@ -272,6 +277,7 @@ export async function POST(request: Request) {
       }
     },
     cancel() {
+      if (heartbeat) clearInterval(heartbeat);
       clearTimeout(globalTimer);
       gate.gate.release();
       abort.abort();

lib/client-api.ts

@@ -30,10 +30,17 @@ export type GenerateProgress = {
   message: string;
 };
 
+/** No response at all for this long: the run is stuck, not slow. */
+const RESPONSE_GUARD_MS = 45_000;
+/** No bytes for this long while streaming: the model stopped talking. The
+ *  routes heartbeat every 15s, so 90s of silence really is silence. */
+const STREAM_GUARD_MS = 90_000;
+
 type StreamEvent =
   | { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
   | { type: "deck"; deck: StreamedDeck; meta: Record<string, unknown> }
-  | { type: "error"; error: string; kind?: string };
+  | { type: "error"; error: string; kind?: string }
+  | { type: "ping" };
 
 export class ApiError extends Error {
   kind: string;
@@ -52,11 +59,13 @@ export class ApiError extends Error {
 export async function consumeNdjson(
   response: Response,
   handleEvent: (event: unknown) => void,
+  options: { stallMs?: number } = {},
 ): Promise<void> {
   if (!response.body) throw new ApiError(t("api.empty"), "bad_response");
   const reader = response.body.getReader();
   const decoder = new TextDecoder();
   let buffer = "";
+  const stallMs = options.stallMs ?? 0;
   const flush = (line: string) => {
     if (!line.trim()) return;
     try {
@@ -66,9 +75,25 @@ export async function consumeNdjson(
     }
   };
   while (true) {
-    const { done, value } = await reader.read();
+    let chunk: ReadableStreamReadResult<Uint8Array>;
-    if (done) break;
+    if (stallMs > 0) {
-    buffer += decoder.decode(value, { stream: true });
+      let timer: ReturnType<typeof setTimeout> | null = null;
+      const stalled = new Promise<never>((_, reject) => {
+        timer = setTimeout(() => reject(new ApiError(t("api.stalled"), "timeout")), stallMs);
+      });
+      try {
+        chunk = await Promise.race([reader.read(), stalled]);
+      } catch (error) {
+        void reader.cancel().catch(() => {});
+        throw error;
+      } finally {
+        if (timer) clearTimeout(timer);
+      }
+    } else {
+      chunk = await reader.read();
+    }
+    if (chunk.done) break;
+    buffer += decoder.decode(chunk.value, { stream: true });
     let newline = buffer.indexOf("\n");
     while (newline !== -1) {
       flush(buffer.slice(0, newline));
@@ -87,19 +112,35 @@ export async function generateDeck(
   input: GenerateInput,
   options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
 ): Promise<{ deck: StreamedDeck; meta: Record<string, unknown> }> {
+  // A run that goes quiet is worse than one that fails: the wizard used to sit
+  // on its loading screen until the server's own 2x deadline (ten minutes by
+  // default) when the model hung. The caller's signal is forwarded, and a
+  // no-response guard covers the "nothing at all" phase.
+  const controller = new AbortController();
+  const forwarded = options.signal;
+  let headersTimedOut = false;
+  if (forwarded) {
+    if (forwarded.aborted) controller.abort();
+    else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
+  }
+  const headersTimer = setTimeout(() => {
+    headersTimedOut = true;
+    controller.abort();
+  }, RESPONSE_GUARD_MS);
   let response: Response;
   try {
     response = await fetch("/api/generate", {
       method: "POST",
       headers: { "Content-Type": "application/json", ...authHeaders(), ...localeHeader() },
       body: JSON.stringify(input),
-      signal: options.signal,
+      signal: controller.signal,
     });
   } catch (error) {
-    if (error instanceof Error && error.name === "AbortError") {
+    if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
-      throw new ApiError(t("api.cancelled"), "aborted");
+    if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
-    }
     throw new ApiError(t("api.unreachable"), "network");
+  } finally {
+    clearTimeout(headersTimer);
   }
 
   if (!response.ok) {
@@ -124,7 +165,7 @@ export async function generateDeck(
     } else if (event.type === "error") {
       streamError = new ApiError(event.error, event.kind);
     }
-  });
+  }, { stallMs: STREAM_GUARD_MS });
 
   if (streamError) throw streamError;
   if (!deck) {
@@ -183,7 +224,8 @@ export type StreamedHomework =
 type HomeworkStreamEvent =
   | { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
   | { type: "homework"; homework: StreamedHomework; meta: Record<string, unknown> }
-  | { type: "error"; error: string; kind?: string };
+  | { type: "error"; error: string; kind?: string }
+  | { type: "ping" };
 
 /**
  * Calls POST /api/homework and consumes the NDJSON progress stream. Same shape
@@ -193,19 +235,32 @@ export async function generateHomework(
   input: HomeworkInput,
   options: { onProgress?: (progress: HomeworkProgress) => void; signal?: AbortSignal } = {},
 ): Promise<{ homework: StreamedHomework; meta: Record<string, unknown> }> {
+  // Same guards as generateDeck: no infinite loading screen on a hung model.
+  const controller = new AbortController();
+  const forwarded = options.signal;
+  let headersTimedOut = false;
+  if (forwarded) {
+    if (forwarded.aborted) controller.abort();
+    else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
+  }
+  const headersTimer = setTimeout(() => {
+    headersTimedOut = true;
+    controller.abort();
+  }, RESPONSE_GUARD_MS);
   let response: Response;
   try {
     response = await fetch("/api/homework", {
       method: "POST",
       headers: { "Content-Type": "application/json", ...authHeaders(), ...localeHeader() },
       body: JSON.stringify(input),
-      signal: options.signal,
+      signal: controller.signal,
     });
   } catch (error) {
-    if (error instanceof Error && error.name === "AbortError") {
+    if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
-      throw new ApiError(t("api.cancelled"), "aborted");
+    if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
-    }
     throw new ApiError(t("api.unreachable"), "network");
+  } finally {
+    clearTimeout(headersTimer);
   }
 
   if (!response.ok) {
@@ -228,7 +283,7 @@ export async function generateHomework(
     } else if (event.type === "error") {
       streamError = new ApiError(event.error, event.kind);
     }
-  });
+  }, { stallMs: STREAM_GUARD_MS });
 
   if (streamError) throw streamError;
   if (!homework) {

lib/i18n-dict.ts

@@ -720,6 +720,7 @@ const en: Record<string, string> = {
   "api.unreachableShort": "Could not reach the Freezmo server.",
   "api.httpError": "The server returned HTTP {status}.",
   "api.empty": "The server sent an empty response.",
+  "api.stalled": "The model stopped responding. Check it and try again.",
   "api.almost": "Almost there…",
   "api.noDeck": "No deck came back from the server. Check your Ollama settings and try again.",
   "api.incomplete": "The server finished without sending a deck.",
@@ -1675,6 +1676,7 @@ const cs: Record<string, string> = {
   "api.unreachableShort": "Nepodařilo se připojit k serveru Freezmo.",
   "api.httpError": "Server vrátil HTTP {status}.",
   "api.empty": "Server poslal prázdnou odpověď.",
+  "api.stalled": "Model přestal odpovídat. Zkontrolujte ho a zkuste to znovu.",
   "api.almost": "Už to bude…",
   "api.noDeck": "Ze serveru nepřišla žádná sada. Zkontroluj nastavení Ollamy a zkus to znovu.",
   "api.incomplete": "Server skončil, aniž by poslal sadu.",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freezmo",
-  "version": "0.3.9",
+  "version": "0.4.0",
   "private": true,
   "license": "GPL-3.0-only",
   "scripts": {

public/sw.js

@@ -9,7 +9,7 @@
  *
  * Bumping CACHE_VERSION is how a new deploy invalidates old assets.
  */
-const CACHE_VERSION = "freezmo-v68";
+const CACHE_VERSION = "freezmo-v69";
 const SHELL_CACHE = `${CACHE_VERSION}-shell`;
 const ASSET_CACHE = `${CACHE_VERSION}-assets`;
 

tests/wizard-test.mjs

@@ -199,6 +199,32 @@ for (const viewport of [{ width: 390, height: 844 }, { width: 360, height: 740 }
   await context.close();
 }
 
+/* 4b. a stalled run gives up on its own (no infinite loading) -------- */
+{
+  const { context, page } = await makePage(browser);
+  await seed(page, { decks: [] });
+  await page.clock.install();
+  await page.route("**/api/generate", () => new Promise(() => {}));
+
+  await page.goto(`${BASE}/create`, { waitUntil: "networkidle" });
+  await page.waitForTimeout(300);
+  await page.getByRole("button", { name: "Paste text instead" }).click();
+  await page.locator("textarea").first().fill("Photosynthesis needs light and water.");
+  await page.getByRole("button", { name: "Next" }).click();
+  await page.getByRole("button", { name: "Next" }).click();
+  await page.getByRole("button", { name: "Freeze my deck" }).click();
+  await page.waitForTimeout(400);
+  check("stall: the loading screen shows", await page.getByRole("button", { name: "Cancel" }).isVisible());
+  await page.clock.fastForward(60_000);
+  await page.waitForTimeout(600);
+  check(
+    "stall: the run gives up with a retry instead of loading forever",
+    await page.getByRole("button", { name: "Retry" }).isVisible(),
+  );
+  check("stall: the wizard is usable again", await page.getByRole("heading", { name: "Ready to freeze" }).isVisible());
+  await context.close();
+}
+
 /* 5. leaving mid-run aborts: no surprise deck, no navigation --------- */
 {
   const { context, page } = await makePage(browser);