freezmo/lib/client-api.ts
Martin Dvorak 8e1cd0855d Podcast voices: natural provider TTS, and system-voice hygiene
Two reports, one root: the platform's speech engine.

"Loud cracking at the end of every sentence" is the browser/OS TTS popping
between its internal sentence chunks (an idle cancel makes it worse). Hygiene
for that path: never cancel when nothing is pending, normalise the spoken text
(control characters, ellipses, dashes), and a 140ms beat between sections so
speech never runs into the engine's tail.

The real fix is a second voice source that does not use the platform engine:
- lib/providers carries TTS profiles (OpenAI gpt-4o-mini-tts, Groq
  playai-tts, any OpenAI-compatible /audio/speech behind the custom provider),
  male voices first.
- /api/tts proxies them with the same guards and the key still in headers.
- the picker gains System / Natural, with the provider's voice list.
- lib/podcast-audio plays one clip per section through a hidden <audio>
  element, measures the clip's real duration and gives each word a slice of it
  proportional to its spoken length - the highlight follows the audio exactly,
  and pause/seek/chapters become real audio operations. Clips cache per
  session, the next section generates while the current plays, and a voice
  change disposes the element and its blobs. Speed applies live (setRate)
  without dropping the cache.

podcast-test stubs /api/tts with a real silent WAV and pins the path: the
voice list, the request body, exactly one playing element, and the highlight
riding currentTime. 43 checks. tests/lib.mjs launches Chromium with the
autoplay policy relaxed (a real device has the listener's gesture).
Version 0.7.6, SW v98.
2026-10-16 16:34:46 +00:00

644 lines
22 KiB
TypeScript

"use client";
import { localeHeader, t } from "./i18n";
import type { Activity, Difficulty, HomeworkMode, HomeworkStep } from "./schemas";
import { API_KEY_HEADER, authHeaders } from "./settings";
import { providerHeaders, providersStore } from "./providers-client";
export type GenerateInput = {
images?: string[];
text?: string;
difficulty: Difficulty;
count?: number;
focus?: string;
language?: string;
mode?: "create" | "more";
existing?: { title: string; types: string[]; count: number };
};
export type StreamedDeck = {
title: string;
subject?: string;
summary?: string;
language?: string;
sourceText: string;
difficulty: Difficulty;
activities: Activity[];
};
export type GenerateProgress = {
stage: "vision" | "generation" | "done";
message: string;
};
/** No response at all for this long: the run is stuck, not slow. */
const RESPONSE_GUARD_MS = 45_000;
/** No bytes for this long while streaming: the model stopped talking. The
* routes heartbeat every 15s, so 90s of silence really is silence. */
const STREAM_GUARD_MS = 90_000;
type StreamEvent = {
type: "podcast";
podcast: StreamedPodcast;
meta?: Record<string, unknown>;
} | StreamEventBase;
type StreamEventBase =
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
| { type: "deck"; deck: StreamedDeck; meta: Record<string, unknown> }
| { type: "error"; error: string; kind?: string }
| { type: "ping" };
export class ApiError extends Error {
kind: string;
constructor(message: string, kind = "unknown") {
super(message);
this.name = "ApiError";
this.kind = kind;
}
}
/**
* Reads an NDJSON response line by line and hands each parsed event to the
* callback. Malformed lines are skipped (a half-written line must never break
* the stream), and the trailing buffer is flushed at the end.
*/
export async function consumeNdjson(
response: Response,
handleEvent: (event: unknown) => void,
options: { stallMs?: number } = {},
): Promise<void> {
if (!response.body) throw new ApiError(t("api.empty"), "bad_response");
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
const stallMs = options.stallMs ?? 0;
const flush = (line: string) => {
if (!line.trim()) return;
try {
handleEvent(JSON.parse(line));
} catch {
/* ignore malformed lines */
}
};
while (true) {
let chunk: ReadableStreamReadResult<Uint8Array>;
let timer: ReturnType<typeof setTimeout> | null = null;
try {
if (stallMs > 0) {
const stalled = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new ApiError(t("api.stalled"), "timeout")), stallMs);
});
chunk = await Promise.race([reader.read(), stalled]);
} else {
chunk = await reader.read();
}
} catch (error) {
// A connection that dropped mid-run used to leak a raw DOMException, and
// the wizard answered with its generic "check your Ollama settings"
// card for what was really a network blip. Classify everything here.
void reader.cancel().catch(() => {});
if (error instanceof ApiError) throw error;
if (error instanceof Error && error.name === "AbortError") {
throw new ApiError(t("api.cancelled"), "aborted");
}
throw new ApiError(t("api.streamLost"), "network");
} finally {
if (timer) clearTimeout(timer);
}
if (chunk.done) break;
buffer += decoder.decode(chunk.value, { stream: true });
let newline = buffer.indexOf("\n");
while (newline !== -1) {
flush(buffer.slice(0, newline));
buffer = buffer.slice(newline + 1);
newline = buffer.indexOf("\n");
}
}
flush(buffer);
}
/**
* Calls POST /api/generate and consumes the NDJSON progress stream.
* Progress events arrive between the vision and generation steps.
*/
export async function generateDeck(
input: GenerateInput,
options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
): Promise<{ deck: StreamedDeck; meta: Record<string, unknown> }> {
// A provider picked in Settings → Providers runs the generation instead of
// the server's Ollama; its key travels with the request only.
const generation = providersStore.getSnapshot().generation;
// A run that goes quiet is worse than one that fails: the wizard used to sit
// on its loading screen until the server's own 2x deadline (ten minutes by
// default) when the model hung. The caller's signal is forwarded, and a
// no-response guard covers the "nothing at all" phase.
const controller = new AbortController();
const forwarded = options.signal;
let headersTimedOut = false;
if (forwarded) {
if (forwarded.aborted) controller.abort();
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
}
const headersTimer = setTimeout(() => {
headersTimedOut = true;
controller.abort();
}, RESPONSE_GUARD_MS);
let response: Response;
try {
response = await fetch("/api/generate", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(
generation ? { ...input, textModel: generation.textModel, visionModel: generation.visionModel } : input,
),
signal: controller.signal,
});
} catch {
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
throw new ApiError(t("api.unreachable"), "network");
} finally {
clearTimeout(headersTimer);
}
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
let deck: StreamedDeck | null = null;
let meta: Record<string, unknown> = {};
let streamError: ApiError | null = null;
let sawDeck = false;
await consumeNdjson(response, (raw) => {
const event = raw as StreamEvent;
if (event.type === "stage") {
options.onProgress?.({ stage: event.stage, message: event.message });
} else if (event.type === "deck") {
deck = event.deck;
meta = event.meta ?? {};
sawDeck = true;
options.onProgress?.({ stage: "done", message: t("api.almost") });
} else if (event.type === "error") {
streamError = new ApiError(event.error, event.kind);
}
}, { stallMs: STREAM_GUARD_MS });
if (streamError) throw streamError;
if (!deck) {
throw new ApiError(
sawDeck
? t("api.incomplete")
: t("api.noDeck"),
"bad_response",
);
}
const valid = (deck as StreamedDeck).activities?.filter(
(activity) => activity && typeof activity.type === "string" && typeof activity.id === "string",
);
if (!valid || valid.length === 0) {
throw new ApiError(t("api.noActivities"), "bad_response");
}
return { deck: { ...(deck as StreamedDeck), activities: valid }, meta };
}
/* ------------------------------------------------------------------ */
/* Podcast mode */
/* ------------------------------------------------------------------ */
export type PodcastInput = {
images?: string[];
text?: string;
/** Target episode length in minutes. */
minutes: number;
focus?: string;
language?: string;
};
export type StreamedPodcast = {
title: string;
language?: string;
summary?: string;
sourceText: string;
sections: { heading: string; text: string }[];
};
/** Notes → a spoken episode. Same stream shape as the deck generator. */
export async function generatePodcast(
input: PodcastInput,
options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
): Promise<{ podcast: StreamedPodcast; meta: Record<string, unknown> }> {
const generation = providersStore.getSnapshot().generation;
const controller = new AbortController();
const forwarded = options.signal;
let headersTimedOut = false;
if (forwarded) {
if (forwarded.aborted) controller.abort();
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
}
const headersTimer = setTimeout(() => {
headersTimedOut = true;
controller.abort();
}, RESPONSE_GUARD_MS);
let response: Response;
try {
response = await fetch("/api/podcast", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(
generation ? { ...input, textModel: generation.textModel, visionModel: generation.visionModel } : input,
),
signal: controller.signal,
});
} catch {
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
throw new ApiError(t("api.unreachable"), "network");
} finally {
clearTimeout(headersTimer);
}
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
let podcast: StreamedPodcast | null = null;
let streamError: ApiError | null = null;
await consumeNdjson(response, (raw) => {
const event = raw as StreamEvent;
if (event.type === "stage") {
options.onProgress?.({ stage: event.stage, message: event.message });
} else if (event.type === "podcast") {
podcast = event.podcast as StreamedPodcast;
options.onProgress?.({ stage: "done", message: t("api.almost") });
} else if (event.type === "error") {
streamError = new ApiError(event.error, event.kind);
}
}, { stallMs: STREAM_GUARD_MS });
if (streamError) throw streamError;
const ready = podcast as StreamedPodcast | null;
if (!ready || !Array.isArray(ready.sections) || ready.sections.length === 0) {
throw new ApiError(t("api.noPodcast"), "bad_response");
}
return { podcast: ready, meta: {} };
}
/* ------------------------------------------------------------------ */
/* Natural podcast voices (provider text-to-speech) */
/* ------------------------------------------------------------------ */
/**
* One section of the script, spoken by the picked provider. The key rides in
* the headers exactly like every other model call; the audio comes back as a
* blob URL the player hands to an <audio> element.
*/
export async function synthesizeSpeech(
input: { text: string; voice?: string; model?: string; providerId: string },
options: { signal?: AbortSignal } = {},
): Promise<{ url: string; type: string }> {
const response = await fetch("/api/tts", {
method: "POST",
headers: {
"Content-Type": "application/json",
...authHeaders(),
...providerHeaders(input.providerId),
...localeHeader(),
},
body: JSON.stringify({ text: input.text, voice: input.voice, model: input.model }),
signal: options.signal,
}).catch(() => {
throw new ApiError(t("api.unreachable"), "network");
});
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
const blob = await response.blob();
if (blob.size === 0) throw new ApiError(t("api.noAudio"), "bad_response");
return { url: URL.createObjectURL(blob), type: blob.type };
}
/* ------------------------------------------------------------------ */
/* Make my homework */
/* ------------------------------------------------------------------ */
export type HomeworkInput = {
mode: HomeworkMode;
images?: string[];
text?: string;
/** Mode "check": the student's own attempt at the assignment. */
solution?: string;
/** Mode "check": photos of the attempt (handwritten work, scanned pages). */
solutionImages?: string[];
language?: string;
};
export type HomeworkProgress = {
stage: "vision" | "generation" | "done";
message: string;
};
export type StreamedHomework =
| { mode: "make"; title: string; language?: string; answer: string; sourceText?: string }
| { mode: "guide"; title: string; language?: string; steps: HomeworkStep[]; sourceText?: string }
| {
mode: "check";
title: string;
language?: string;
verdict: string;
correct: string[];
wrong: Array<{ issue: string; why: string; fix: string }>;
corrected: string;
sourceText?: string;
};
type HomeworkStreamEvent =
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
| { type: "homework"; homework: StreamedHomework; meta: Record<string, unknown> }
| { type: "error"; error: string; kind?: string }
| { type: "ping" };
/**
* Calls POST /api/homework and consumes the NDJSON progress stream. Same shape
* as generateDeck: stage events first, then one final payload.
*/
export async function generateHomework(
input: HomeworkInput,
options: { onProgress?: (progress: HomeworkProgress) => void; signal?: AbortSignal } = {},
): Promise<{ homework: StreamedHomework; meta: Record<string, unknown> }> {
const generation = providersStore.getSnapshot().generation;
// Same guards as generateDeck: no infinite loading screen on a hung model.
const controller = new AbortController();
const forwarded = options.signal;
let headersTimedOut = false;
if (forwarded) {
if (forwarded.aborted) controller.abort();
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
}
const headersTimer = setTimeout(() => {
headersTimedOut = true;
controller.abort();
}, RESPONSE_GUARD_MS);
let response: Response;
try {
response = await fetch("/api/homework", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(
generation ? { ...input, textModel: generation.textModel, visionModel: generation.visionModel } : input,
),
signal: controller.signal,
});
} catch {
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
throw new ApiError(t("api.unreachable"), "network");
} finally {
clearTimeout(headersTimer);
}
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
let homework: StreamedHomework | null = null;
let meta: Record<string, unknown> = {};
let streamError: ApiError | null = null;
await consumeNdjson(response, (raw) => {
const event = raw as HomeworkStreamEvent;
if (event.type === "stage") {
options.onProgress?.({ stage: event.stage, message: event.message });
} else if (event.type === "homework") {
homework = event.homework;
meta = event.meta ?? {};
options.onProgress?.({ stage: "done", message: t("api.almost") });
} else if (event.type === "error") {
streamError = new ApiError(event.error, event.kind);
}
}, { stallMs: STREAM_GUARD_MS });
if (streamError) throw streamError;
if (!homework) {
throw new ApiError(t("api.noHomework"), "bad_response");
}
const result = homework as StreamedHomework;
if (result.mode === "make") {
if (!result.answer?.trim()) throw new ApiError(t("api.noHomework"), "bad_response");
} else if (result.mode === "check") {
if (!result.corrected?.trim() || !result.verdict?.trim()) {
throw new ApiError(t("api.noHomework"), "bad_response");
}
} else {
const steps = result.steps?.filter((step) => step && typeof step.title === "string");
if (!steps || steps.length < 2) throw new ApiError(t("api.noHomework"), "bad_response");
// Checkpoints must look like real activities; drop anything malformed
// rather than crash the guide.
const cleaned = steps.map((step) => ({
...step,
checkpoint:
step.checkpoint && typeof step.checkpoint.type === "string" && typeof step.checkpoint.id === "string"
? step.checkpoint
: undefined,
}));
return { homework: { ...result, steps: cleaned }, meta };
}
return { homework: result, meta };
}
/** Follow-up on a finished result: shorter, more detail, or explain again. */
export async function refineHomework(input: {
instruction: "shorter" | "more" | "explain";
text: string;
context?: string;
signal?: AbortSignal;
}): Promise<string> {
const generation = providersStore.getSnapshot().generation;
let response: Response;
try {
response = await fetch("/api/homework/refine", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(
generation
? { instruction: input.instruction, text: input.text, context: input.context, model: generation.textModel }
: { instruction: input.instruction, text: input.text, context: input.context },
),
signal: input.signal,
});
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
throw new ApiError(t("api.cancelled"), "aborted");
}
throw new ApiError(t("api.unreachable"), "network");
}
const body = (await response.json().catch(() => null)) as { text?: string; error?: string; kind?: string } | null;
if (!response.ok) {
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
if (!body?.text?.trim()) throw new ApiError(t("api.noHomework"), "bad_response");
return body.text.trim();
}
export async function gradeAnswer(input: {
question: string;
referenceAnswer: string;
keyPoints: string[];
userAnswer: string;
language?: string;
signal?: AbortSignal;
}): Promise<{ correct: boolean; score: number; feedback: string; missing: string[] }> {
const generation = providersStore.getSnapshot().generation;
let response: Response;
try {
response = await fetch("/api/grade", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(generation ? { ...input, model: generation.textModel } : input),
signal: input.signal,
});
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
throw new ApiError(t("api.gradingCancelled"), "aborted");
}
throw new ApiError(t("api.unreachableShort"), "network");
}
const body = (await response.json().catch(() => null)) as
| { correct: boolean; score: number; feedback: string; missing?: string[]; error?: string; kind?: string }
| null;
if (!response.ok || !body || body.error) {
throw new ApiError(body?.error ?? t("api.gradingFailed", { status: response.status }), body?.kind);
}
return {
correct: Boolean(body.correct),
score: Math.max(0, Math.min(100, Math.round(body.score))),
feedback: body.feedback,
missing: body.missing ?? [],
};
}
export type HealthModel = {
configured: string;
resolved: string;
available: boolean | null;
note?: string;
};
export type HealthChatModel = {
name: string;
badge: "text" | "vision" | "both";
available: boolean | null;
};
export type Health = {
ok: boolean;
reachable: boolean;
mode: "local-daemon" | "direct-cloud";
authMode: "user-key" | "server" | "local";
needsKey: boolean;
keyAccepted: boolean | null;
baseUrl: string;
hasApiKey: boolean;
vision: HealthModel;
text: HealthModel;
/** Chat model allowlist; optional so older servers/mocks stay valid. */
chatModels?: HealthChatModel[];
latencyMs: number | null;
modelCount: number | null;
error: string | null;
kind?: string;
checkedAt: string;
};
export async function fetchHealth(): Promise<Health> {
let response: Response;
try {
response = await fetch("/api/health", { cache: "no-store", headers: { ...authHeaders(), ...localeHeader() } });
} catch {
throw new ApiError(t("api.unreachableShort"), "network");
}
const body = (await response.json().catch(() => null)) as (Partial<Health> & { error?: string; kind?: string }) | null;
if (!response.ok) {
throw new ApiError(
body?.error ?? t("api.healthFailed", { status: response.status }),
body?.kind ?? "http",
);
}
// Guard against anything that isn't a health payload (e.g. a proxy error
// page) so consumers can rely on the shape.
if (!body || typeof body !== "object" || !body.vision || !body.text) {
throw new ApiError(t("api.healthUnexpected"), "bad_response");
}
return body as Health;
}
export type KeyCheck =
| { result: "valid" }
| { result: "invalid"; message: string }
| { result: "offline"; message: string };
/**
* Checks a *draft* key before it is saved anywhere, by asking our own
* `/api/health` route (which performs a real one-token chat probe against
* Ollama — model listings alone prove nothing). The key travels in the
* request header for this single call and is never stored server-side.
*/
export async function verifyOllamaKey(apiKey: string): Promise<KeyCheck> {
let response: Response;
try {
response = await fetch("/api/health", {
cache: "no-store",
headers: { [API_KEY_HEADER]: apiKey, ...localeHeader() },
});
} catch {
return {
result: "offline",
message: t("api.unreachable"),
};
}
const body = (await response.json().catch(() => null)) as Health | null;
if (!response.ok || !body) {
return {
result: "offline",
message: t("api.serverUnexpected"),
};
}
if (body.keyAccepted === false) {
return {
result: "invalid",
message:
body.error ?? t("api.keyRejected"),
};
}
if (body.keyAccepted === true) return { result: "valid" };
if (!body.reachable) {
return {
result: "offline",
message:
body.error ?? t("api.ollamaUnreachable"),
};
}
// Reachable, but the probe was inconclusive (usually a model-availability
// matter rather than a bad key). Let the user in — Settings shows details.
return { result: "valid" };
}