Infinite loading guard: heartbeats + stall watchdogs on the model runs
A user got stuck on the wizard's loading screen: a hung model meant silence, and silence was indistinguishable from work - the server's own hard deadline is 2x OLLAMA_TIMEOUT_MS (ten minutes by default), and the client waited on a dead stream indefinitely. Both generate routes (deck, homework) now send an NDJSON heartbeat every 15s, and the client guards the two quiet phases: no response at all for 45s, and no bytes for 90s mid-stream, both surfacing the regular error state with a Retry. The caller's abort still maps to the friendly "cancelled" state. Verified with a stalled route and Playwright's fake clock in the wizard suite (93 checks), plus contrast 0 findings, mobile 124/124, chat 101/101, regression 44/44, i18n 13/13, native 33/33. Version 0.4.0, SW -> freezmo-v69.
This commit is contained in:
parent
2b78dcf0d1
commit
20bb08b0e7
8 changed files with 125 additions and 19 deletions
|
|
@ -1411,6 +1411,15 @@ Things I chose while building, with the reasoning:
|
|||
pre-blurred wallpaper — the scene is clearly visible through the glass, and the
|
||||
pixel-contrast scan still passes.
|
||||
|
||||
75. **A quiet run can no longer load forever.** A user got stuck on the wizard's loading screen:
|
||||
a hung model meant silence, and silence was indistinguishable from work — the server's own hard
|
||||
deadline is 2x `OLLAMA_TIMEOUT_MS` (ten minutes by default), and the client waited on a dead
|
||||
stream indefinitely. Both generate routes (deck, homework) now send an NDJSON heartbeat every
|
||||
15s, and the client guards the two quiet phases: no response at all for 45s, and no bytes for
|
||||
90s mid-stream, both surfacing the regular error state with a Retry — verified with a stalled
|
||||
route and Playwright's fake clock in the wizard suite (93 checks). The heartbeat also keeps
|
||||
proxies from closing an idle connection.
|
||||
|
||||
74. **Phones get flat glass, not live blur.** The touch fallback from 73 (a real backdrop blur
|
||||
over the live scene) turned out to be the wrong trade: every pane re-blurs on every frame
|
||||
while the bubbles animate, and phones choke — input lag, "buttons take ages to respond". Touch
|
||||
|
|
|
|||
|
|
@ -39,7 +39,8 @@ const requestSchema = z.object({
|
|||
type StreamEvent =
|
||||
| { type: "stage"; stage: PipelineStage; message: string }
|
||||
| { type: "deck"; deck: unknown; meta: Record<string, unknown> }
|
||||
| { type: "error"; error: string; kind?: string };
|
||||
| { type: "error"; error: string; kind?: string }
|
||||
| { type: "ping" };
|
||||
|
||||
function encoder() {
|
||||
const enc = new TextEncoder();
|
||||
|
|
@ -112,6 +113,10 @@ export async function POST(request: Request) {
|
|||
const abort = new AbortController();
|
||||
// A single hard deadline for the whole pipeline (two sequential model calls).
|
||||
const globalTimer = setTimeout(() => abort.abort(), config.timeoutMs * 2);
|
||||
// Heartbeats: one quiet model call can run for minutes, and the client's
|
||||
// stall guard needs to tell "slow" from "dead" — a silent stream is how the
|
||||
// wizard ended up loading forever. Cleared with the stream either way.
|
||||
let heartbeat: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
|
|
@ -123,6 +128,7 @@ export async function POST(request: Request) {
|
|||
}
|
||||
};
|
||||
|
||||
heartbeat = setInterval(() => send({ type: "ping" }), 15_000);
|
||||
try {
|
||||
let sourceText: string;
|
||||
let deckTitle: string | undefined;
|
||||
|
|
@ -206,6 +212,7 @@ export async function POST(request: Request) {
|
|||
kind: error instanceof OllamaError ? error.kind : "unknown",
|
||||
});
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
clearTimeout(globalTimer);
|
||||
gate.gate.release();
|
||||
try {
|
||||
|
|
@ -216,6 +223,7 @@ export async function POST(request: Request) {
|
|||
}
|
||||
},
|
||||
cancel() {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
clearTimeout(globalTimer);
|
||||
gate.gate.release();
|
||||
abort.abort();
|
||||
|
|
|
|||
|
|
@ -34,7 +34,8 @@ const requestSchema = z.object({
|
|||
type StreamEvent =
|
||||
| { type: "stage"; stage: PipelineStage; message: string }
|
||||
| { type: "homework"; homework: unknown; meta: Record<string, unknown> }
|
||||
| { type: "error"; error: string; kind?: string };
|
||||
| { type: "error"; error: string; kind?: string }
|
||||
| { type: "ping" };
|
||||
|
||||
function encoder() {
|
||||
const enc = new TextEncoder();
|
||||
|
|
@ -105,6 +106,8 @@ export async function POST(request: Request) {
|
|||
const abort = new AbortController();
|
||||
// A single hard deadline for the whole pipeline (two sequential model calls).
|
||||
const globalTimer = setTimeout(() => abort.abort(), config.timeoutMs * 2);
|
||||
// Heartbeats so the client's stall guard can tell "slow" from "dead".
|
||||
let heartbeat: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
|
|
@ -115,6 +118,7 @@ export async function POST(request: Request) {
|
|||
/* client went away */
|
||||
}
|
||||
};
|
||||
heartbeat = setInterval(() => send({ type: "ping" }), 15_000);
|
||||
|
||||
try {
|
||||
let assignmentText: string;
|
||||
|
|
@ -262,6 +266,7 @@ export async function POST(request: Request) {
|
|||
kind: error instanceof OllamaError ? error.kind : "unknown",
|
||||
});
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
clearTimeout(globalTimer);
|
||||
gate.gate.release();
|
||||
try {
|
||||
|
|
@ -272,6 +277,7 @@ export async function POST(request: Request) {
|
|||
}
|
||||
},
|
||||
cancel() {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
clearTimeout(globalTimer);
|
||||
gate.gate.release();
|
||||
abort.abort();
|
||||
|
|
|
|||
|
|
@ -30,10 +30,17 @@ export type GenerateProgress = {
|
|||
message: string;
|
||||
};
|
||||
|
||||
/** No response at all for this long: the run is stuck, not slow. */
|
||||
const RESPONSE_GUARD_MS = 45_000;
|
||||
/** No bytes for this long while streaming: the model stopped talking. The
|
||||
* routes heartbeat every 15s, so 90s of silence really is silence. */
|
||||
const STREAM_GUARD_MS = 90_000;
|
||||
|
||||
type StreamEvent =
|
||||
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
|
||||
| { type: "deck"; deck: StreamedDeck; meta: Record<string, unknown> }
|
||||
| { type: "error"; error: string; kind?: string };
|
||||
| { type: "error"; error: string; kind?: string }
|
||||
| { type: "ping" };
|
||||
|
||||
export class ApiError extends Error {
|
||||
kind: string;
|
||||
|
|
@ -52,11 +59,13 @@ export class ApiError extends Error {
|
|||
export async function consumeNdjson(
|
||||
response: Response,
|
||||
handleEvent: (event: unknown) => void,
|
||||
options: { stallMs?: number } = {},
|
||||
): Promise<void> {
|
||||
if (!response.body) throw new ApiError(t("api.empty"), "bad_response");
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const stallMs = options.stallMs ?? 0;
|
||||
const flush = (line: string) => {
|
||||
if (!line.trim()) return;
|
||||
try {
|
||||
|
|
@ -66,9 +75,25 @@ export async function consumeNdjson(
|
|||
}
|
||||
};
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
let chunk: ReadableStreamReadResult<Uint8Array>;
|
||||
if (stallMs > 0) {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const stalled = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new ApiError(t("api.stalled"), "timeout")), stallMs);
|
||||
});
|
||||
try {
|
||||
chunk = await Promise.race([reader.read(), stalled]);
|
||||
} catch (error) {
|
||||
void reader.cancel().catch(() => {});
|
||||
throw error;
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
} else {
|
||||
chunk = await reader.read();
|
||||
}
|
||||
if (chunk.done) break;
|
||||
buffer += decoder.decode(chunk.value, { stream: true });
|
||||
let newline = buffer.indexOf("\n");
|
||||
while (newline !== -1) {
|
||||
flush(buffer.slice(0, newline));
|
||||
|
|
@ -87,19 +112,35 @@ export async function generateDeck(
|
|||
input: GenerateInput,
|
||||
options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
|
||||
): Promise<{ deck: StreamedDeck; meta: Record<string, unknown> }> {
|
||||
// A run that goes quiet is worse than one that fails: the wizard used to sit
|
||||
// on its loading screen until the server's own 2x deadline (ten minutes by
|
||||
// default) when the model hung. The caller's signal is forwarded, and a
|
||||
// no-response guard covers the "nothing at all" phase.
|
||||
const controller = new AbortController();
|
||||
const forwarded = options.signal;
|
||||
let headersTimedOut = false;
|
||||
if (forwarded) {
|
||||
if (forwarded.aborted) controller.abort();
|
||||
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
|
||||
}
|
||||
const headersTimer = setTimeout(() => {
|
||||
headersTimedOut = true;
|
||||
controller.abort();
|
||||
}, RESPONSE_GUARD_MS);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch("/api/generate", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...authHeaders(), ...localeHeader() },
|
||||
body: JSON.stringify(input),
|
||||
signal: options.signal,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
throw new ApiError(t("api.cancelled"), "aborted");
|
||||
}
|
||||
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
|
||||
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
|
||||
throw new ApiError(t("api.unreachable"), "network");
|
||||
} finally {
|
||||
clearTimeout(headersTimer);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -124,7 +165,7 @@ export async function generateDeck(
|
|||
} else if (event.type === "error") {
|
||||
streamError = new ApiError(event.error, event.kind);
|
||||
}
|
||||
});
|
||||
}, { stallMs: STREAM_GUARD_MS });
|
||||
|
||||
if (streamError) throw streamError;
|
||||
if (!deck) {
|
||||
|
|
@ -183,7 +224,8 @@ export type StreamedHomework =
|
|||
type HomeworkStreamEvent =
|
||||
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
|
||||
| { type: "homework"; homework: StreamedHomework; meta: Record<string, unknown> }
|
||||
| { type: "error"; error: string; kind?: string };
|
||||
| { type: "error"; error: string; kind?: string }
|
||||
| { type: "ping" };
|
||||
|
||||
/**
|
||||
* Calls POST /api/homework and consumes the NDJSON progress stream. Same shape
|
||||
|
|
@ -193,19 +235,32 @@ export async function generateHomework(
|
|||
input: HomeworkInput,
|
||||
options: { onProgress?: (progress: HomeworkProgress) => void; signal?: AbortSignal } = {},
|
||||
): Promise<{ homework: StreamedHomework; meta: Record<string, unknown> }> {
|
||||
// Same guards as generateDeck: no infinite loading screen on a hung model.
|
||||
const controller = new AbortController();
|
||||
const forwarded = options.signal;
|
||||
let headersTimedOut = false;
|
||||
if (forwarded) {
|
||||
if (forwarded.aborted) controller.abort();
|
||||
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
|
||||
}
|
||||
const headersTimer = setTimeout(() => {
|
||||
headersTimedOut = true;
|
||||
controller.abort();
|
||||
}, RESPONSE_GUARD_MS);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch("/api/homework", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...authHeaders(), ...localeHeader() },
|
||||
body: JSON.stringify(input),
|
||||
signal: options.signal,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
throw new ApiError(t("api.cancelled"), "aborted");
|
||||
}
|
||||
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
|
||||
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
|
||||
throw new ApiError(t("api.unreachable"), "network");
|
||||
} finally {
|
||||
clearTimeout(headersTimer);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -228,7 +283,7 @@ export async function generateHomework(
|
|||
} else if (event.type === "error") {
|
||||
streamError = new ApiError(event.error, event.kind);
|
||||
}
|
||||
});
|
||||
}, { stallMs: STREAM_GUARD_MS });
|
||||
|
||||
if (streamError) throw streamError;
|
||||
if (!homework) {
|
||||
|
|
|
|||
|
|
@ -720,6 +720,7 @@ const en: Record<string, string> = {
|
|||
"api.unreachableShort": "Could not reach the Freezmo server.",
|
||||
"api.httpError": "The server returned HTTP {status}.",
|
||||
"api.empty": "The server sent an empty response.",
|
||||
"api.stalled": "The model stopped responding. Check it and try again.",
|
||||
"api.almost": "Almost there…",
|
||||
"api.noDeck": "No deck came back from the server. Check your Ollama settings and try again.",
|
||||
"api.incomplete": "The server finished without sending a deck.",
|
||||
|
|
@ -1675,6 +1676,7 @@ const cs: Record<string, string> = {
|
|||
"api.unreachableShort": "Nepodařilo se připojit k serveru Freezmo.",
|
||||
"api.httpError": "Server vrátil HTTP {status}.",
|
||||
"api.empty": "Server poslal prázdnou odpověď.",
|
||||
"api.stalled": "Model přestal odpovídat. Zkontrolujte ho a zkuste to znovu.",
|
||||
"api.almost": "Už to bude…",
|
||||
"api.noDeck": "Ze serveru nepřišla žádná sada. Zkontroluj nastavení Ollamy a zkus to znovu.",
|
||||
"api.incomplete": "Server skončil, aniž by poslal sadu.",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "freezmo",
|
||||
"version": "0.3.9",
|
||||
"version": "0.4.0",
|
||||
"private": true,
|
||||
"license": "GPL-3.0-only",
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
*
|
||||
* Bumping CACHE_VERSION is how a new deploy invalidates old assets.
|
||||
*/
|
||||
const CACHE_VERSION = "freezmo-v68";
|
||||
const CACHE_VERSION = "freezmo-v69";
|
||||
const SHELL_CACHE = `${CACHE_VERSION}-shell`;
|
||||
const ASSET_CACHE = `${CACHE_VERSION}-assets`;
|
||||
|
||||
|
|
|
|||
|
|
@ -199,6 +199,32 @@ for (const viewport of [{ width: 390, height: 844 }, { width: 360, height: 740 }
|
|||
await context.close();
|
||||
}
|
||||
|
||||
/* 4b. a stalled run gives up on its own (no infinite loading) -------- */
|
||||
{
|
||||
const { context, page } = await makePage(browser);
|
||||
await seed(page, { decks: [] });
|
||||
await page.clock.install();
|
||||
await page.route("**/api/generate", () => new Promise(() => {}));
|
||||
|
||||
await page.goto(`${BASE}/create`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(300);
|
||||
await page.getByRole("button", { name: "Paste text instead" }).click();
|
||||
await page.locator("textarea").first().fill("Photosynthesis needs light and water.");
|
||||
await page.getByRole("button", { name: "Next" }).click();
|
||||
await page.getByRole("button", { name: "Next" }).click();
|
||||
await page.getByRole("button", { name: "Freeze my deck" }).click();
|
||||
await page.waitForTimeout(400);
|
||||
check("stall: the loading screen shows", await page.getByRole("button", { name: "Cancel" }).isVisible());
|
||||
await page.clock.fastForward(60_000);
|
||||
await page.waitForTimeout(600);
|
||||
check(
|
||||
"stall: the run gives up with a retry instead of loading forever",
|
||||
await page.getByRole("button", { name: "Retry" }).isVisible(),
|
||||
);
|
||||
check("stall: the wizard is usable again", await page.getByRole("heading", { name: "Ready to freeze" }).isVisible());
|
||||
await context.close();
|
||||
}
|
||||
|
||||
/* 5. leaving mid-run aborts: no surprise deck, no navigation --------- */
|
||||
{
|
||||
const { context, page } = await makePage(browser);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue