Podcast mode: notes read aloud with a lyric-style word highlight

The third segment of the note-powered wizards (Study deck / Homework /
Podcast) takes the same intake - photos, documents, pasted text, the share
target - and turns it into a spoken episode.

- prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown,
  no visual references), chaptered into 3-8 sections around a length target.
- /api/podcast mirrors /api/generate's plumbing (rate limit, model gate,
  provider keys, heartbeats, the 2x deadline, the optional vision pass).
- storage: frezmo.podcasts.v1, zod-validated, included in the backup and the
  encrypted sync payload; a new library tab lists episodes with listen /
  rename / delete-with-undo, and the wizard switch deep-links in.
- player: the transcript is word-wrapped spans and the word under the voice
  rides a lyric-style highlight that keeps itself centred. Two clocks: exact
  onboundary events where the browser reports them, a per-word length timer
  where it does not (iOS Safari); the first boundary retires the timer.
  Pause = cancel + re-speak from the word, which makes resume, chapter jumps
  and tap-any-word-to-seek the same operation; the position saves on pause,
  page-hide and unmount, so reopening resumes. Speed rebuilds the engine and
  keeps playing; the Aero soundtrack pauses itself when the voice starts; a
  browser with no voice gets an honest note and a readable transcript.
- the script sits on a glass pane (plain paragraphs over Aero's grass failed
  contrast; the scan now covers the player too).

tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and
drives the highlight by hand: 26 checks. All suites green, contrast clean.
Version 0.7.3, SW v95.
This commit is contained in:
Martin Dvořák 2026-10-16 11:42:15 +00:00
commit d95c5c7151
30 changed files with 2575 additions and 26 deletions

View file

@ -20,6 +20,8 @@ import {
HOMEWORK_CHECK_SYSTEM_PROMPT,
HOMEWORK_GUIDE_SYSTEM_PROMPT,
HOMEWORK_MAKE_SYSTEM_PROMPT,
PODCAST_SYSTEM_PROMPT,
buildPodcastUserPrompt,
REFINE_SYSTEM_PROMPT,
VISION_SYSTEM_PROMPT,
VISION_USER_INSTRUCTION,
@ -27,6 +29,7 @@ import {
import {
extractedNotesSchema,
generatedActivitySchema,
generatedPodcastSchema,
gradeResultSchema,
homeworkCheckSchema,
homeworkMakeSchema,
@ -38,6 +41,7 @@ import {
type HomeworkCheck,
type HomeworkGuide,
type HomeworkMake,
type GeneratedPodcast,
} from "./schemas";
import { parseJsonLoose } from "./utils";
import type { OllamaAuth } from "./config";
@ -401,6 +405,58 @@ export async function generateActivities(input: {
return { deck: value, requested };
}
/* ------------------------------------------------------------------ */
/* Podcast */
/* ------------------------------------------------------------------ */
const PODCAST_SHAPE_HINT = `{"title":"Nothing but the sky","language":"English","summary":"One sentence.","sections":[{"heading":"Why blue","text":"Sixty to a hundred and forty words of narration."}]}`;
export async function generatePodcast(input: {
model: string;
auth: OllamaAuth;
provider?: ProviderTarget;
notes: string;
minutes: number;
language?: string;
focus?: string;
signal?: AbortSignal;
}): Promise<{ podcast: GeneratedPodcast }> {
const minutes = Math.min(Math.max(input.minutes, 1), 15);
const { value } = await runJsonStep<GeneratedPodcast>({
model: input.model,
auth: input.auth,
provider: input.provider,
system: PODCAST_SYSTEM_PROMPT,
user: buildPodcastUserPrompt({
notes: input.notes,
minutes,
language: input.language,
focus: input.focus,
}),
/* One usable section beats a retry loop: the repair pass fills gaps. */
validate: (parsed) => {
const result = generatedPodcastSchema.safeParse(parsed);
if (!result.success) return { ok: false as const, error: summariseZodError(result.error) };
const sections = result.data.sections
.map((section) => ({
heading: section.heading.trim(),
text: section.text.trim().replace(/\s+/g, " "),
}))
.filter((section) => section.text.length > 0);
if (sections.length === 0) {
return { ok: false as const, error: "No sections with spoken text were found." };
}
return { ok: true as const, value: { ...result.data, sections } };
},
shapeHint: PODCAST_SHAPE_HINT,
label: "Writing the script",
signal: input.signal,
temperature: 0.75,
});
return { podcast: value };
}
/* ------------------------------------------------------------------ */
/* Grading */
/* ------------------------------------------------------------------ */

View file

@ -15,9 +15,10 @@ import { applyTheme, type Theme } from "./theme";
import { chatHistoryStore, ensureChatsHydrated, replaceChats } from "./chat-store";
import { replaceFolders, folderStore } from "./folders";
import { homeworkStore, replaceHomework } from "./homework";
import { podcastStore, replacePodcasts } from "./podcasts";
import { ensureLearningHydrated, learningStore, replaceLearning } from "./history";
import { setLocale, t } from "./i18n";
import { folderSchema, deckSchema, homeworkRecordSchema, answerRecordSchema, srsRecordSchema, chatConversationSchema } from "./schemas";
import { folderSchema, deckSchema, homeworkRecordSchema, podcastSchema, answerRecordSchema, srsRecordSchema, chatConversationSchema } from "./schemas";
import { saveSettings, settingsStore } from "./settings";
import { deckStore, replaceDecks } from "./storage";
@ -34,6 +35,8 @@ export const backupDataSchema = z.object({
decks: z.array(deckSchema).max(2000),
folders: z.array(folderSchema).max(500),
homework: z.array(homeworkRecordSchema).max(2000),
/** Podcast episodes. Optional for old backups. */
podcasts: z.array(podcastSchema).max(2000).optional(),
/** Answer log and SRS schedule (IndexedDB). Optional for old backups. */
history: z.array(answerRecordSchema).max(20_000).optional(),
srs: z.array(srsRecordSchema).max(50_000).optional(),
@ -67,6 +70,7 @@ export async function buildBackup(): Promise<BackupFile> {
decks: deckStore.getSnapshot(),
folders: folderStore.getSnapshot(),
homework: homeworkStore.getSnapshot(),
podcasts: podcastStore.getSnapshot(),
history: learningStore.getAnswersSnapshot(),
srs: learningStore.getSrsSnapshot(),
chats: chatHistoryStore.getSnapshot(),
@ -143,6 +147,7 @@ export type ImportPreview = {
decks: ImportCounts;
folders: ImportCounts;
homework: ImportCounts;
podcasts: ImportCounts;
chats: ImportCounts;
};
@ -151,6 +156,7 @@ export function previewImport(backup: BackupFile): ImportPreview {
decks: countMerge(deckStore.getSnapshot(), backup.data.decks),
folders: countMerge(folderStore.getSnapshot(), backup.data.folders),
homework: countMerge(homeworkStore.getSnapshot(), backup.data.homework),
podcasts: countMerge(podcastStore.getSnapshot(), backup.data.podcasts ?? []),
chats: countMerge(chatHistoryStore.getSnapshot(), backup.data.chats ?? []),
};
}
@ -167,12 +173,15 @@ export async function applyImport(
mode === "replace" ? data.folders : mergeById(folderStore.getSnapshot(), data.folders);
const homework =
mode === "replace" ? data.homework : mergeById(homeworkStore.getSnapshot(), data.homework);
const podcasts =
mode === "replace" ? data.podcasts ?? [] : mergeById(podcastStore.getSnapshot(), data.podcasts ?? []);
// Folders first so imported decks can point at them. Importing is a real
// local change, so it also marks sync dirty (via the stores' write()).
replaceFolders(folders);
replaceDecks(decks);
replaceHomework(homework);
replacePodcasts(podcasts);
// Answer log + schedule: merge by id (answers) and by key (schedules), or
// replace wholesale when mode === "replace".

View file

@ -37,7 +37,13 @@ const RESPONSE_GUARD_MS = 45_000;
* routes heartbeat every 15s, so 90s of silence really is silence. */
const STREAM_GUARD_MS = 90_000;
type StreamEvent =
type StreamEvent = {
type: "podcast";
podcast: StreamedPodcast;
meta?: Record<string, unknown>;
} | StreamEventBase;
type StreamEventBase =
| { type: "stage"; stage: "vision" | "generation" | "done"; message: string }
| { type: "deck"; deck: StreamedDeck; meta: Record<string, unknown> }
| { type: "error"; error: string; kind?: string }
@ -200,6 +206,91 @@ export async function generateDeck(
return { deck: { ...(deck as StreamedDeck), activities: valid }, meta };
}
/* ------------------------------------------------------------------ */
/* Podcast mode */
/* ------------------------------------------------------------------ */
export type PodcastInput = {
images?: string[];
text?: string;
/** Target episode length in minutes. */
minutes: number;
focus?: string;
language?: string;
};
export type StreamedPodcast = {
title: string;
language?: string;
summary?: string;
sourceText: string;
sections: { heading: string; text: string }[];
};
/** Notes → a spoken episode. Same stream shape as the deck generator. */
export async function generatePodcast(
input: PodcastInput,
options: { onProgress?: (progress: GenerateProgress) => void; signal?: AbortSignal } = {},
): Promise<{ podcast: StreamedPodcast; meta: Record<string, unknown> }> {
const generation = providersStore.getSnapshot().generation;
const controller = new AbortController();
const forwarded = options.signal;
let headersTimedOut = false;
if (forwarded) {
if (forwarded.aborted) controller.abort();
else forwarded.addEventListener("abort", () => controller.abort(), { once: true });
}
const headersTimer = setTimeout(() => {
headersTimedOut = true;
controller.abort();
}, RESPONSE_GUARD_MS);
let response: Response;
try {
response = await fetch("/api/podcast", {
method: "POST",
headers: { "Content-Type": "application/json", ...authHeaders(), ...providerHeaders(generation?.provider), ...localeHeader() },
body: JSON.stringify(
generation ? { ...input, textModel: generation.textModel, visionModel: generation.visionModel } : input,
),
signal: controller.signal,
});
} catch {
if (forwarded?.aborted) throw new ApiError(t("api.cancelled"), "aborted");
if (headersTimedOut) throw new ApiError(t("api.stalled"), "timeout");
throw new ApiError(t("api.unreachable"), "network");
} finally {
clearTimeout(headersTimer);
}
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
let podcast: StreamedPodcast | null = null;
let streamError: ApiError | null = null;
await consumeNdjson(response, (raw) => {
const event = raw as StreamEvent;
if (event.type === "stage") {
options.onProgress?.({ stage: event.stage, message: event.message });
} else if (event.type === "podcast") {
podcast = event.podcast as StreamedPodcast;
options.onProgress?.({ stage: "done", message: t("api.almost") });
} else if (event.type === "error") {
streamError = new ApiError(event.error, event.kind);
}
}, { stallMs: STREAM_GUARD_MS });
if (streamError) throw streamError;
const ready = podcast as StreamedPodcast | null;
if (!ready || !Array.isArray(ready.sections) || ready.sections.length === 0) {
throw new ApiError(t("api.noPodcast"), "bad_response");
}
return { podcast: ready, meta: {} };
}
/* ------------------------------------------------------------------ */
/* Make my homework */
/* ------------------------------------------------------------------ */

View file

@ -162,7 +162,7 @@ const en: Record<string, string> = {
"settings.exportAll": "Export all data",
"settings.exportNote": "Decks, homework, progress and settings — never your API key.",
"settings.importBackup": "Import backup",
"settings.importPreview": "This backup has {decks} decks, {homework} homework results, {folders} folders and {chats} chats.",
"settings.importPreview": "This backup has {decks} decks, {homework} homework results, {podcasts} podcasts, {folders} folders and {chats} chats.",
"settings.importMergeNote": "Merge keeps what's here and adds the backup; matching IDs are replaced.",
"settings.importReplaceNote": "Replace wipes this browser's data and uses the backup instead.",
"settings.importMerge": "Merge",
@ -274,7 +274,61 @@ const en: Record<string, string> = {
"homework.nav": "New homework",
"wizard.mode.deck": "Study deck",
"wizard.mode.homework": "Make my homework",
"wizard.mode.aria": "Study deck or homework",
"wizard.mode.podcast": "Podcast",
"wizard.mode.aria": "Study deck, homework or podcast",
/* Podcast mode (notes → a spoken episode with a highlighted transcript) */
"podcast.title": "New podcast",
"podcast.subtitle": "Freezmo reads your notes back to you as a short episode — and highlights every word as it is spoken, like lyrics.",
"podcast.addNotes": "Add your notes",
"podcast.addPhotos": "Photos",
"podcast.addDocs": "Documents",
"podcast.addFirst": "Add notes first",
"podcast.generate": "Create the episode",
"podcast.lengthAria": "Episode length",
"podcast.lengthHelp": "A rough target — the script stays as long as the notes need.",
"podcast.focusLabel": "What should it focus on? (optional)",
"podcast.minutes.one": "{n} minute",
"podcast.minutes.few": "{n} minutes",
"podcast.minutes.many": "{n} minutes",
"podcast.sectionCount.one": "{n} section",
"podcast.sectionCount.few": "{n} sections",
"podcast.sectionCount.many": "{n} sections",
"podcast.wordCount.one": "{n} word",
"podcast.wordCount.few": "{n} words",
"podcast.wordCount.many": "{n} words",
"podcast.sectionOf": "Part {n} of {total}",
"podcast.play": "Play the episode",
"podcast.pause": "Pause the episode",
"podcast.restart": "Start from the beginning",
"podcast.speedAria": "Playback speed",
"podcast.tapToJump": "Tap any word to jump there.",
"podcast.noVoice": "This browser has no speech voice installed, so the transcript cannot be read aloud here. You can still read along.",
"podcast.loadingEpisode": "Loading the episode",
"podcast.notFound": "Episode not found",
"podcast.notFoundBody": "It was probably deleted, or it lives in another browser's storage.",
"podcast.loading.read1": "Reading your notes…",
"podcast.loading.read2": "Making sense of the pages…",
"podcast.loading.write1": "Writing the script…",
"podcast.loading.write2": "Finding the through-line…",
"podcast.loading.write3": "Polishing the spoken bits…",
"podcast.tip1": "The transcript highlights every word as it is read — tap any word to jump there.",
"podcast.tip2": "A four-minute walk is a great length for one subject.",
/* Library: the podcasts tab */
"tabPodcasts": "Podcasts",
"podcast.library.empty": "No episodes yet",
"podcast.library.emptyBody": "Turn a page of notes into a spoken episode and it will wait here.",
"podcast.library.new": "New podcast",
"podcast.card.sections": "{n} parts",
"podcast.card.minutes": "~{n} min",
"podcast.card.actions": "Episode actions",
"podcast.card.open": "Listen",
"podcast.card.rename": "Rename",
"podcast.card.delete": "Delete episode",
"podcast.card.deleted": "Episode deleted",
"podcast.card.renamed": "Episode renamed",
"podcast.regenerate": "Write it again",
"homeworkHeroTitle": "Make my homework",
"homework.subtitle": "Paste it, photograph it or add a file — then pick how I should help.",
"homework.phone.assignment": "The assignment",
@ -1175,7 +1229,7 @@ const cs: Record<string, string> = {
"settings.exportAll": "Exportovat všechna data",
"settings.exportNote": "Sady, úkoly, pokrok a nastavení — nikdy tvůj API klíč.",
"settings.importBackup": "Importovat zálohu",
"settings.importPreview": "Záloha obsahuje {decks} sad, {homework} výsledků úkolů, {folders} složek a {chats} chatů.",
"settings.importPreview": "Záloha obsahuje {decks} sad, {homework} výsledků úkolů, {podcasts} podcastů, {folders} složek a {chats} chatů.",
"settings.importMergeNote": "Sloučení ponechá stávající data a přidá zálohu; stejná ID se nahradí.",
"settings.importReplaceNote": "Nahrazení smaže data v tomto prohlížeči a použije zálohu.",
"settings.importMerge": "Sloučit",
@ -1287,7 +1341,61 @@ const cs: Record<string, string> = {
"homework.nav": "Nový úkol",
"wizard.mode.deck": "Studijní sada",
"wizard.mode.homework": "Udělej mi úkol",
"wizard.mode.aria": "Studijní sada nebo úkol",
"wizard.mode.podcast": "Podcast",
"wizard.mode.aria": "Studijní sada, úkol nebo podcast",
/* Podcast mode (notes → a spoken episode with a highlighted transcript) */
"podcast.title": "Nový podcast",
"podcast.subtitle": "Freezmo ti tvoje poznámky přečte jako krátkou epizodu — a každé slovo při čtení zvýrazní, jako texty písní.",
"podcast.addNotes": "Přidej poznámky",
"podcast.addPhotos": "Fotky",
"podcast.addDocs": "Dokumenty",
"podcast.addFirst": "Nejdřív přidej poznámky",
"podcast.generate": "Vytvořit epizodu",
"podcast.lengthAria": "Délka epizody",
"podcast.lengthHelp": "Jen orientační cíl — scénář zůstane tak dlouhý, jak poznámky potřebují.",
"podcast.focusLabel": "Na co se má zaměřit? (nepovinné)",
"podcast.minutes.one": "{n} minuta",
"podcast.minutes.few": "{n} minuty",
"podcast.minutes.many": "{n} minut",
"podcast.sectionCount.one": "{n} část",
"podcast.sectionCount.few": "{n} části",
"podcast.sectionCount.many": "{n} částí",
"podcast.wordCount.one": "{n} slovo",
"podcast.wordCount.few": "{n} slova",
"podcast.wordCount.many": "{n} slov",
"podcast.sectionOf": "Část {n} z {total}",
"podcast.play": "Přehrát epizodu",
"podcast.pause": "Pozastavit epizodu",
"podcast.restart": "Začít od začátku",
"podcast.speedAria": "Rychlost přehrávání",
"podcast.tapToJump": "Klikni na kterékoli slovo a přeskočíš tam.",
"podcast.noVoice": "Tento prohlížeč nemá nainstalovaný hlas, takže se přepis nedá přečíst nahlas. Číst ho ale můžeš dál.",
"podcast.loadingEpisode": "Načítání epizody",
"podcast.notFound": "Epizoda nenalezena",
"podcast.notFoundBody": "Nejspíš byla smazaná, nebo je uložená v jiném prohlížeči.",
"podcast.loading.read1": "Čtu tvoje poznámky…",
"podcast.loading.read2": "Dávám stránky dohromady…",
"podcast.loading.write1": "Píšu scénář…",
"podcast.loading.write2": "Hledám hlavní linii…",
"podcast.loading.write3": "Ladím mluvené pasáže…",
"podcast.tip1": "Přepis zvýrazňuje každé čtené slovo — kliknutím na slovo se tam přesuneš.",
"podcast.tip2": "Čtyřminutová procházka je pro jedno téma tak akorát.",
/* Library: the podcasts tab */
"tabPodcasts": "Podcasty",
"podcast.library.empty": "Zatím žádné epizody",
"podcast.library.emptyBody": "Proměň stránku poznámek v mluvenou epizodu a počká tady na tebe.",
"podcast.library.new": "Nový podcast",
"podcast.card.sections": "{n} částí",
"podcast.card.minutes": "~{n} min",
"podcast.card.actions": "Akce epizody",
"podcast.card.open": "Poslouchat",
"podcast.card.rename": "Přejmenovat",
"podcast.card.delete": "Smazat epizodu",
"podcast.card.deleted": "Epizoda smazána",
"podcast.card.renamed": "Epizoda přejmenována",
"podcast.regenerate": "Napsat znovu",
"homeworkHeroTitle": "Udělej mi úkol",
"homework.subtitle": "Vlož ho, vyfoť nebo přidej soubor — a vyber, jak ti mám pomoct.",
"homework.phone.assignment": "Zadání",

247
lib/podcast-speech.ts Normal file
View file

@ -0,0 +1,247 @@
"use client";
/**
* The podcast player's voice: one section at a time, with the spoken word
* reported back so the transcript can highlight it (the Spotify-lyrics bit).
*
* Two clocks drive the highlight. Where the browser fires `onboundary`
* (desktop Chrome and friends) that is exact — the character index maps onto
* the word list. Where it does not (iOS Safari, several Android engines) a
* timer estimates each word from its length and the playback rate, so the
* highlight still moves at roughly speaking speed. The first boundary seen
* retires the timer for the rest of the session.
*
* Pause/seek are implemented by cancelling and re-speaking from a word: the
* platform `pause()` is unreliable across browsers, and re-speaking also makes
* "jump to a chapter" and "resume where you left off" the same operation.
*/
import { pickVoice, speechAvailable } from "./speech";
import type { PodcastSection } from "./schemas";
export type PodcastVoice = { section: number; word: number };
export type PodcastSpeechEvents = {
sections: PodcastSection[];
locale: string;
rate: number;
/** Global word position changed (word = index inside its section). */
onWord: (position: PodcastVoice) => void;
/** Playback actually began (after the engine accepted it). */
onPlay?: () => void;
onPause?: () => void;
onEnd: () => void;
onError?: (message: string) => void;
};
export type Token = {
section: number;
/** Index of the word inside its section. */
index: number;
/** Index across the whole episode (the transcript's data-word). */
global: number;
start: number;
end: number;
text: string;
};
/** Split into words, keeping each one's character span inside its section. */
export function tokenizeSections(sections: PodcastSection[]): Token[] {
const tokens: Token[] = [];
sections.forEach((section, sectionIndex) => {
const text = section.text;
const pattern = /\S+/g;
let index = 0;
let match: RegExpExecArray | null;
while ((match = pattern.exec(text)) !== null) {
tokens.push({
section: sectionIndex,
index,
global: tokens.length,
start: match.index,
end: match.index + match[0].length,
text: match[0],
});
index += 1;
}
});
return tokens;
}
/** Estimated spoken time of one word at rate 1 (ms). */
function wordDuration(text: string): number {
return 95 + 42 * Math.max(1, text.replace(/[^\p{L}\p{N}]/gu, "").length);
}
export type PodcastSpeech = {
play: (from?: PodcastVoice) => void;
pause: () => void;
stop: () => void;
playing: () => boolean;
position: () => PodcastVoice | null;
words: Token[];
};
export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech {
const tokens = tokenizeSections(events.sections);
let playing = false;
let position: PodcastVoice | null = null;
let boundariesWork: boolean | null = null;
let timer: number | null = null;
/** Cleared by cancel(): stops stale timers/events from a previous utterance. */
let generation = 0;
const clearTimer = () => {
if (timer !== null) {
window.clearTimeout(timer);
timer = null;
}
};
const finishPass = () => {
playing = false;
events.onPause?.();
};
const wordsIn = (section: number) => tokens.filter((token) => token.section === section);
/** Speak the section from `word` on. Empty word list → the next section. */
const speakFrom = (target: PodcastVoice) => {
if (!speechAvailable()) {
events.onError?.("unsupported");
return;
}
const section = events.sections[target.section];
if (!section) {
position = null;
finishPass();
events.onEnd();
return;
}
const sectionTokens = wordsIn(target.section);
if (sectionTokens.length === 0) {
speakFrom({ section: target.section + 1, word: 0 });
return;
}
const word = Math.min(Math.max(0, target.word), sectionTokens.length - 1);
position = { section: target.section, word };
const from = sectionTokens[word].start;
const text = section.text.slice(from);
const current = ++generation;
try {
window.speechSynthesis.cancel();
const utterance = new SpeechSynthesisUtterance(text);
utterance.lang = events.locale;
utterance.rate = events.rate;
const voice = pickVoice(events.locale);
if (voice) utterance.voice = voice;
const offsetFor = (charIndex: number) => {
/* First token whose span ends after the reported character. */
let found = word;
for (let i = word; i < sectionTokens.length; i += 1) {
const token = sectionTokens[i];
if (token.end > from + charIndex) {
found = i;
break;
}
found = i;
}
return found;
};
utterance.onboundary = (event) => {
if (generation !== current) return;
if (event.name && event.name !== "word") return;
boundariesWork = true;
clearTimer();
const index = offsetFor(event.charIndex);
position = { section: target.section, word: index };
events.onWord(position);
};
utterance.onstart = () => {
if (generation !== current) return;
playing = true;
events.onPlay?.();
events.onWord(position ?? { section: target.section, word });
/* The fallback clock: only while this browser has never reported a
* boundary. Each tick schedules the next word by its length. */
if (boundariesWork !== true) {
let index = word;
const tick = () => {
if (generation !== current) return;
index += 1;
if (index >= sectionTokens.length) {
/* Let onend take over; a long word list can outlast the audio
* by a hair, and stopping the tick avoids double-advancing. */
timer = null;
return;
}
position = { section: target.section, word: index };
events.onWord(position);
timer = window.setTimeout(tick, wordDuration(sectionTokens[index].text) / events.rate);
};
timer = window.setTimeout(tick, wordDuration(sectionTokens[word].text) / events.rate);
}
};
utterance.onend = () => {
if (generation !== current) return;
clearTimer();
/* An utterance that ended without a single boundary means the browser
does not report them: the fallback clock was the right call. */
if (boundariesWork === null) boundariesWork = false;
if (target.section + 1 >= events.sections.length) {
position = null;
finishPass();
events.onEnd();
return;
}
speakFrom({ section: target.section + 1, word: 0 });
};
utterance.onerror = (event) => {
if (generation !== current) return;
clearTimer();
playing = false;
/* "interrupted"/"canceled" are our own stops, not failures. */
const reason = (event as SpeechSynthesisErrorEvent).error;
if (reason !== "interrupted" && reason !== "canceled") {
events.onError?.(String(reason ?? "error"));
events.onPause?.();
}
};
window.speechSynthesis.speak(utterance);
} catch {
playing = false;
events.onError?.("error");
}
};
return {
play(from) {
speakFrom(from ?? position ?? { section: 0, word: 0 });
},
pause() {
generation += 1;
clearTimer();
try {
window.speechSynthesis.cancel();
} catch {
/* already gone */
}
playing = false;
events.onPause?.();
},
stop() {
this.pause();
position = null;
},
playing: () => playing,
position: () => position,
words: tokens,
};
}

172
lib/podcasts.ts Normal file
View file

@ -0,0 +1,172 @@
"use client";
/**
* Podcast episodes backed by localStorage — the same tiny observable store as
* the homework history. An episode is the script the voice reads, the notes it
* came from (so it can be regenerated) and the word the listener got to.
*
* Photos and documents were reduced to text on the client before this point;
* originals are never stored.
*/
import { podcastSchema, type Podcast } from "./schemas";
import { markSyncDirty } from "./sync-state";
const STORAGE_KEY = "freezmo.podcasts.v1";
const EMPTY: Podcast[] = [];
/** Enough to regenerate from, while staying well under the storage quota. */
export const MAX_STORED_SOURCE = 24_000;
type Listener = () => void;
let episodes: Podcast[] | null = null;
let cached: Podcast[] = EMPTY;
const listeners = new Set<Listener>();
function read(): Podcast[] {
if (typeof window === "undefined") return EMPTY;
try {
const raw = window.localStorage.getItem(STORAGE_KEY);
if (!raw) return EMPTY;
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) return EMPTY;
const valid: Podcast[] = [];
for (const entry of parsed) {
const result = podcastSchema.safeParse(entry);
if (result.success) valid.push(result.data);
else console.warn("[freezmo] skipping invalid podcast", result.error.issues);
}
return valid;
} catch (error) {
console.warn("[freezmo] could not read the podcasts", error);
return EMPTY;
}
}
function ensureLoaded(): void {
if (episodes !== null) return;
episodes = read();
cached = episodes;
}
function write(next: Podcast[]): void {
episodes = next;
cached = next;
try {
window.localStorage.setItem(STORAGE_KEY, JSON.stringify(next));
} catch (error) {
console.warn("[freezmo] could not save the podcasts (storage full?)", error);
}
/* An episode change with sync on pushes itself (see lib/sync-auto). */
if (markSyncDirty()) void import("./sync-auto").then((module) => module.scheduleAutoSync());
listeners.forEach((listener) => listener());
}
/** Used by backup import and sync: wholesale replacement of the list. */
export function replacePodcasts(next: Podcast[]): void {
const valid: Podcast[] = [];
for (const entry of next) {
const result = podcastSchema.safeParse(entry);
if (result.success) valid.push(entry);
else console.warn("[freezmo] skipping invalid podcast in import", result.error.issues);
}
write(valid);
}
function onStorage(event: StorageEvent): void {
if (event.key === STORAGE_KEY) {
episodes = null;
ensureLoaded();
listeners.forEach((listener) => listener());
}
}
export const podcastStore = {
subscribe(listener: Listener): () => void {
ensureLoaded();
if (listeners.size === 0) window.addEventListener("storage", onStorage);
listeners.add(listener);
return () => {
listeners.delete(listener);
if (listeners.size === 0) window.removeEventListener("storage", onStorage);
};
},
getSnapshot(): Podcast[] {
ensureLoaded();
return cached;
},
getServerSnapshot(): Podcast[] {
return EMPTY;
},
};
/** Newest first. */
export function sortPodcasts(list: Podcast[]): Podcast[] {
return [...list].sort((a, b) => b.updatedAt - a.updatedAt);
}
export function getPodcast(id: string): Podcast | undefined {
return podcastStore.getSnapshot().find((episode) => episode.id === id);
}
export function upsertPodcast(episode: Podcast): void {
ensureLoaded();
const current = podcastStore.getSnapshot();
const index = current.findIndex((entry) => entry.id === episode.id);
const next = index === -1 ? [episode, ...current] : current.map((entry) => (entry.id === episode.id ? episode : entry));
write(next);
}
export function renamePodcast(id: string, title: string): void {
ensureLoaded();
write(
podcastStore.getSnapshot().map((entry) =>
entry.id === id ? { ...entry, title, updatedAt: Date.now() } : entry,
),
);
}
export function deletePodcast(id: string): void {
ensureLoaded();
write(podcastStore.getSnapshot().filter((entry) => entry.id !== id));
}
/**
* Where the listener got to. Written on pause/leave rather than per word, so
* a long episode does not thrash localStorage.
*/
export function savePodcastProgress(id: string, progress: { section: number; word: number }): void {
ensureLoaded();
const current = podcastStore.getSnapshot();
const episode = current.find((entry) => entry.id === id);
if (!episode) return;
const at = Date.now();
const previous = episode.progress;
/* First words of a section are the same place a retry would start: skip the
write when nothing meaningfully moved. */
if (previous && previous.section === progress.section && Math.abs(previous.word - progress.word) < 2) {
return;
}
const next = current.map((entry) =>
entry.id === id ? { ...entry, progress: { ...progress, at } } : entry,
);
/* updatedAt deliberately untouched: resuming must not reorder the library. */
episodes = next;
cached = next;
try {
window.localStorage.setItem(STORAGE_KEY, JSON.stringify(next));
} catch {
/* storage full — progress is a nicety, not worth a warning */
}
}
/** Rough spoken length of the whole script, in minutes (150 words a minute). */
export function podcastMinutes(episode: Podcast): number {
const words = episode.sections.reduce((total, section) => total + section.text.split(/\s+/).filter(Boolean).length, 0);
return Math.max(1, Math.round(words / 150));
}
/** The episode's script as one string (used by tests and fallbacks). */
export function podcastScript(episode: Podcast): string {
return episode.sections.map((section) => section.text).join("\n\n");
}

View file

@ -171,6 +171,69 @@ export function buildGenerationUserPrompt({
return lines.filter(Boolean).join("\n");
}
/* ------------------------------------------------------------------ */
/* Podcast mode */
/* ------------------------------------------------------------------ */
/**
* The episode is heard, not read: no markdown, no lists, no headings inside
* the prose. Sections are chapters (shown in the player); the text of each one
* is what the voice reads, so it has to survive being spoken aloud.
*/
export const PODCAST_SYSTEM_PROMPT = `You are Freezmo's podcast writer. You turn a student's own study notes into a short spoken episode — one friendly, curious host talking the listener through the material, the way a good explainer podcast does.
${LANGUAGE_RULES}
Hard rules:
- Use ONLY information contained in the provided notes. Never invent facts, definitions, numbers, dates or formulas. If the notes are thin, the episode is shorter — never padded with outside knowledge.
- Write for the ear. Short sentences. Natural spoken rhythm. Contractions are fine.
- Plain prose only: no markdown, no bullet points, no headings inside a section's text, no stage directions, no speaker labels, no sound effects, no "in this episode" boilerplate beyond a short spoken greeting if it fits naturally.
- No visual references ("as you can see", "the table above"), no page numbers, no "the notes" — talk to the listener about the subject itself.
- Explain, don't just list: connect ideas, give the why, use one small analogy when the notes support it, and call back to earlier parts of the episode where it helps.
- Do not read the notes back verbatim; rewrite them into spoken language while keeping every fact and number exactly as written.
- Spell out anything that is painful to hear as symbols (say "for example", not "e.g."); keep formulas and units spoken naturally.
- Cover the whole notes, in a sensible teaching order.
Required JSON shape:
{
"title": "short episode title, max 8 words, in the notes' language",
"language": "English name of the language you wrote in",
"summary": "one spoken-style sentence about the episode",
"sections": [
{ "heading": "2-4 word chapter label", "text": "60-140 words of narration" }
]
}`;
export function buildPodcastUserPrompt({
notes,
minutes,
language,
focus,
}: {
notes: string;
/** Target episode length in minutes (words are estimated at ~150/min). */
minutes: number;
language?: string;
focus?: string;
}): string {
const words = Math.round(minutes * 150);
const sections = Math.max(3, Math.min(8, Math.round(minutes)));
const lines = [
`Write a ${minutes}-minute podcast episode (about ${words} words, ${sections} sections) from the notes below.`,
language
? `LANGUAGE: write everything in ${language}. Do not translate into English.`
: "LANGUAGE: detect the language of the notes below and write everything in that same language.",
focus ? `The listener especially wants the episode to cover: ${focus}` : "",
"",
"=== STUDENT NOTES (the only allowed source of facts) ===",
notes.slice(0, 48_000),
"=== END OF NOTES ===",
"",
"Respond with the JSON object only.",
];
return lines.filter(Boolean).join("\n");
}
/* ------------------------------------------------------------------ */
/* Short-answer grading */
/* ------------------------------------------------------------------ */

View file

@ -261,6 +261,60 @@ export type HomeworkMakeRecord = Extract<HomeworkRecord, { mode: "make" }>;
export type HomeworkGuideRecord = Extract<HomeworkRecord, { mode: "guide" }>;
export type HomeworkCheckRecord = Extract<HomeworkRecord, { mode: "check" }>;
/* ------------------------------------------------------------------ */
/* Podcasts (notes turned into a spoken episode) */
/* ------------------------------------------------------------------ */
export type PodcastSection = { heading: string; text: string };
/** As produced by the model (no id yet). */
export const generatedPodcastSchema = z.object({
title: z.string().min(1).max(200),
language: z.string().max(60).optional(),
summary: z.string().max(400).optional(),
sections: z
.array(
z.object({
heading: z.string().min(1).max(120),
text: z.string().min(1).max(4000),
}),
)
.min(1)
.max(12),
});
export type GeneratedPodcast = z.infer<typeof generatedPodcastSchema>;
/** Stored episode: the script plus where it came from and where the listener
* got to (so reopening it resumes, Spotify-style). */
export type Podcast = {
id: string;
title: string;
summary?: string;
language?: string;
sourceText: string;
createdAt: number;
updatedAt: number;
sections: PodcastSection[];
progress?: { section: number; word: number; at: number };
};
export const podcastSchema: z.ZodType<Podcast> = z.object({
id: z.string().min(1).max(80),
title: z.string().min(1).max(200),
summary: z.string().max(400).optional(),
language: z.string().max(60).optional(),
sourceText: z.string().max(40_000),
createdAt: z.number(),
updatedAt: z.number(),
sections: z
.array(z.object({ heading: z.string().min(1).max(120), text: z.string().min(1).max(4000) }))
.min(1)
.max(12),
progress: z
.object({ section: z.number().int().min(0), word: z.number().int().min(0), at: z.number() })
.optional(),
});
/* ------------------------------------------------------------------ */
/* Client-side deck types (activity + id/xp, plus storage metadata) */
/* ------------------------------------------------------------------ */

View file

@ -76,8 +76,9 @@ export function speechLocale(language: string | undefined, fallback: string): st
return fallback === "cs" ? "cs-CZ" : "en-US";
}
/** The installed voice closest to `locale` (exact match, then language). */
function pickVoice(locale: string): SpeechSynthesisVoice | null {
/** The installed voice closest to `locale` (exact match, then language).
* Shared with the podcast player (lib/podcast-speech). */
export function pickVoice(locale: string): SpeechSynthesisVoice | null {
try {
const voices = window.speechSynthesis.getVoices();
if (voices.length === 0) return null;

View file

@ -17,6 +17,7 @@ import { chatHistoryStore, ensureChatsHydrated, replaceChats } from "./chat-stor
import { ApiError } from "./client-api";
import { replaceFolders } from "./folders";
import { replaceHomework } from "./homework";
import { podcastStore, replacePodcasts } from "./podcasts";
import { ensureLearningHydrated, learningStore, replaceLearning } from "./history";
import { localeHeader, t } from "./i18n";
import { authHeaders } from "./settings";
@ -173,6 +174,7 @@ async function buildPayload(): Promise<SyncPayload> {
decks: deckStore.getSnapshot(),
folders: folderStore.getSnapshot(),
homework: homeworkStore.getSnapshot(),
podcasts: podcastStore.getSnapshot(),
history: learningStore.getAnswersSnapshot(),
srs: learningStore.getSrsSnapshot(),
chats: chatHistoryStore.getSnapshot(),
@ -188,6 +190,7 @@ async function applyRemote(key: CryptoKey, remote: { updatedAt: number; envelope
replaceFolders(parsed.data.folders);
replaceDecks(parsed.data.decks);
replaceHomework(parsed.data.homework);
replacePodcasts(parsed.data.podcasts ?? []);
replaceChats(parsed.data.chats ?? []);
});
await replaceLearning(parsed.data.history ?? [], parsed.data.srs ?? []);