Podcast mode: notes read aloud with a lyric-style word highlight
The third segment of the note-powered wizards (Study deck / Homework / Podcast) takes the same intake - photos, documents, pasted text, the share target - and turns it into a spoken episode. - prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown, no visual references), chaptered into 3-8 sections around a length target. - /api/podcast mirrors /api/generate's plumbing (rate limit, model gate, provider keys, heartbeats, the 2x deadline, the optional vision pass). - storage: frezmo.podcasts.v1, zod-validated, included in the backup and the encrypted sync payload; a new library tab lists episodes with listen / rename / delete-with-undo, and the wizard switch deep-links in. - player: the transcript is word-wrapped spans and the word under the voice rides a lyric-style highlight that keeps itself centred. Two clocks: exact onboundary events where the browser reports them, a per-word length timer where it does not (iOS Safari); the first boundary retires the timer. Pause = cancel + re-speak from the word, which makes resume, chapter jumps and tap-any-word-to-seek the same operation; the position saves on pause, page-hide and unmount, so reopening resumes. Speed rebuilds the engine and keeps playing; the Aero soundtrack pauses itself when the voice starts; a browser with no voice gets an honest note and a readable transcript. - the script sits on a glass pane (plain paragraphs over Aero's grass failed contrast; the scan now covers the player too). tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and drives the highlight by hand: 26 checks. All suites green, contrast clean. Version 0.7.3, SW v95.
This commit is contained in:
parent
84a94581dc
commit
d95c5c7151
30 changed files with 2575 additions and 26 deletions
|
|
@ -37,7 +37,13 @@ const RESPONSE_GUARD_MS = 45_000;
|
|||
* routes heartbeat every 15s, so 90s of silence really is silence. */
|
||||
const STREAM_GUARD_MS = 90_000;
|
||||
|
||||
type StreamEvent =
|
||||
type StreamEvent = {
|
||||
type: "podcast";
|
||||
podcast: StreamedPodcast;
|
||||
meta?: Record<string, unknown>;
|
||||
} | StreamEventBase;
|
||||
|
||||
type StreamEventBase =
|
||||
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
|
||||
| { type: "deck"; deck: StreamedDeck; meta: Record<string, unknown> }
|
||||
| { type: "error"; error: string; kind?: string }
|
||||
|
|
@ -200,6 +206,91 @@ export async function generateDeck(
|
|||
return { deck: { ...(deck as StreamedDeck), activities: valid }, meta };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Podcast mode */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export type PodcastInput = {
|
||||
images?: string[];
|
||||
text?: string;
|
||||
/** Target episode length in minutes. */
|
||||
minutes: number;
|
||||
focus?: string;
|
||||
language?: string;
|
||||
};
|
||||
|
||||
export type StreamedPodcast = {
|
||||
title: string;
|
||||
language?: string;
|
||||
summary?: string;
|
||||
sourceText: string;
|
||||
sections: { heading: string; text: string }[];
|
||||
};
|
||||
|
||||
/** Notes → a spoken episode. Same stream shape as the deck generator. */
|
||||
export async function generatePodcast(
|
||||
input: PodcastInput,
|
||||
options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
|
||||
): Promise<{ podcast: StreamedPodcast; meta: Record<string, unknown> }> {
|
||||
const generation = providersStore.getSnapshot().generation;
|
||||
const controller = new AbortController();
|
||||
const forwarded = options.signal;
|
||||
let headersTimedOut = false;
|
||||
if (forwarded) {
|
||||
if (forwarded.aborted) controller.abort();
|
||||
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
|
||||
}
|
||||
const headersTimer = setTimeout(() => {
|
||||
headersTimedOut = true;
|
||||
controller.abort();
|
||||
}, RESPONSE_GUARD_MS);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch("/api/podcast", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
|
||||
body: JSON.stringify(
|
||||
generation ? { ...input, textModel: generation.textModel, visionModel: generation.visionModel } : input,
|
||||
),
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch {
|
||||
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
|
||||
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
|
||||
throw new ApiError(t("api.unreachable"), "network");
|
||||
} finally {
|
||||
clearTimeout(headersTimer);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
|
||||
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
|
||||
}
|
||||
|
||||
let podcast: StreamedPodcast | null = null;
|
||||
let streamError: ApiError | null = null;
|
||||
|
||||
await consumeNdjson(response, (raw) => {
|
||||
const event = raw as StreamEvent;
|
||||
if (event.type === "stage") {
|
||||
options.onProgress?.({ stage: event.stage, message: event.message });
|
||||
} else if (event.type === "podcast") {
|
||||
podcast = event.podcast as StreamedPodcast;
|
||||
options.onProgress?.({ stage: "done", message: t("api.almost") });
|
||||
} else if (event.type === "error") {
|
||||
streamError = new ApiError(event.error, event.kind);
|
||||
}
|
||||
}, { stallMs: STREAM_GUARD_MS });
|
||||
|
||||
if (streamError) throw streamError;
|
||||
const ready = podcast as StreamedPodcast | null;
|
||||
if (!ready || !Array.isArray(ready.sections) || ready.sections.length === 0) {
|
||||
throw new ApiError(t("api.noPodcast"), "bad_response");
|
||||
}
|
||||
|
||||
return { podcast: ready, meta: {} };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Make my homework */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue