Podcast mode: notes read aloud with a lyric-style word highlight

The third segment of the note-powered wizards (Study deck / Homework /
Podcast) takes the same intake - photos, documents, pasted text, the share
target - and turns it into a spoken episode.

- prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown,
  no visual references), chaptered into 3-8 sections around a length target.
- /api/podcast mirrors /api/generate's plumbing (rate limit, model gate,
  provider keys, heartbeats, the 2x deadline, the optional vision pass).
- storage: frezmo.podcasts.v1, zod-validated, included in the backup and the
  encrypted sync payload; a new library tab lists episodes with listen /
  rename / delete-with-undo, and the wizard switch deep-links in.
- player: the transcript is word-wrapped spans and the word under the voice
  rides a lyric-style highlight that keeps itself centred. Two clocks: exact
  onboundary events where the browser reports them, a per-word length timer
  where it does not (iOS Safari); the first boundary retires the timer.
  Pause = cancel + re-speak from the word, which makes resume, chapter jumps
  and tap-any-word-to-seek the same operation; the position saves on pause,
  page-hide and unmount, so reopening resumes. Speed rebuilds the engine and
  keeps playing; the Aero soundtrack pauses itself when the voice starts; a
  browser with no voice gets an honest note and a readable transcript.
- the script sits on a glass pane (plain paragraphs over Aero's grass failed
  contrast; the scan now covers the player too).

tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and
drives the highlight by hand: 26 checks. All suites green, contrast clean.
Version 0.7.3, SW v95.
This commit is contained in:
Martin Dvořák 2026-10-16 11:42:15 +00:00
commit d95c5c7151
30 changed files with 2575 additions and 26 deletions

View file

@ -37,7 +37,13 @@ const RESPONSE_GUARD_MS = 45_000;
* routes heartbeat every 15s, so 90s of silence really is silence. */
const STREAM_GUARD_MS = 90_000;
type StreamEvent =
type StreamEvent = {
type: "podcast";
podcast: StreamedPodcast;
meta?: Record<string, unknown>;
} | StreamEventBase;
type StreamEventBase =
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
| { type: "deck"; deck: StreamedDeck; meta: Record<string, unknown> }
| { type: "error"; error: string; kind?: string }
@ -200,6 +206,91 @@ export async function generateDeck(
return { deck: { ...(deck as StreamedDeck), activities: valid }, meta };
}
/* ------------------------------------------------------------------ */
/* Podcast mode */
/* ------------------------------------------------------------------ */
export type PodcastInput = {
images?: string[];
text?: string;
/** Target episode length in minutes. */
minutes: number;
focus?: string;
language?: string;
};
export type StreamedPodcast = {
title: string;
language?: string;
summary?: string;
sourceText: string;
sections: { heading: string; text: string }[];
};
/** Notes → a spoken episode. Same stream shape as the deck generator. */
export async function generatePodcast(
input: PodcastInput,
options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
): Promise<{ podcast: StreamedPodcast; meta: Record<string, unknown> }> {
const generation = providersStore.getSnapshot().generation;
const controller = new AbortController();
const forwarded = options.signal;
let headersTimedOut = false;
if (forwarded) {
if (forwarded.aborted) controller.abort();
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
}
const headersTimer = setTimeout(() => {
headersTimedOut = true;
controller.abort();
}, RESPONSE_GUARD_MS);
let response: Response;
try {
response = await fetch("/api/podcast", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(
generation ? { ...input, textModel: generation.textModel, visionModel: generation.visionModel } : input,
),
signal: controller.signal,
});
} catch {
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
throw new ApiError(t("api.unreachable"), "network");
} finally {
clearTimeout(headersTimer);
}
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
let podcast: StreamedPodcast | null = null;
let streamError: ApiError | null = null;
await consumeNdjson(response, (raw) => {
const event = raw as StreamEvent;
if (event.type === "stage") {
options.onProgress?.({ stage: event.stage, message: event.message });
} else if (event.type === "podcast") {
podcast = event.podcast as StreamedPodcast;
options.onProgress?.({ stage: "done", message: t("api.almost") });
} else if (event.type === "error") {
streamError = new ApiError(event.error, event.kind);
}
}, { stallMs: STREAM_GUARD_MS });
if (streamError) throw streamError;
const ready = podcast as StreamedPodcast | null;
if (!ready || !Array.isArray(ready.sections) || ready.sections.length === 0) {
throw new ApiError(t("api.noPodcast"), "bad_response");
}
return { podcast: ready, meta: {} };
}
/* ------------------------------------------------------------------ */
/* Make my homework */
/* ------------------------------------------------------------------ */