Podcast voices: natural provider TTS, and system-voice hygiene

Two reports, one root: the platform's speech engine.

"Loud cracking at the end of every sentence" is the browser/OS TTS popping
between its internal sentence chunks (an idle cancel makes it worse). Hygiene
for that path: never cancel when nothing is pending, normalise the spoken text
(control characters, ellipses, dashes), and a 140ms beat between sections so
speech never runs into the engine's tail.

The real fix is a second voice source that does not use the platform engine:
- lib/providers carries TTS profiles (OpenAI gpt-4o-mini-tts, Groq
  playai-tts, any OpenAI-compatible /audio/speech behind the custom provider),
  male voices first.
- /api/tts proxies them with the same guards and the key still in headers.
- the picker gains System / Natural, with the provider's voice list.
- lib/podcast-audio plays one clip per section through a hidden <audio>
  element, measures the clip's real duration and gives each word a slice of it
  proportional to its spoken length - the highlight follows the audio exactly,
  and pause/seek/chapters become real audio operations. Clips cache per
  session, the next section generates while the current plays, and a voice
  change disposes the element and its blobs. Speed applies live (setRate)
  without dropping the cache.

podcast-test stubs /api/tts with a real silent WAV and pins the path: the
voice list, the request body, exactly one playing element, and the highlight
riding currentTime. 43 checks. tests/lib.mjs launches Chromium with the
autoplay policy relaxed (a real device has the listener's gesture).
Version 0.7.6, SW v98.
This commit is contained in:
Martin Dvořák 2026-10-16 16:34:46 +00:00
commit 8e1cd0855d
13 changed files with 997 additions and 63 deletions

View file

@ -291,6 +291,43 @@ export async function generatePodcast(
return { podcast: ready, meta: {} };
}
/* ------------------------------------------------------------------ */
/* Natural podcast voices (provider text-to-speech) */
/* ------------------------------------------------------------------ */
/**
* One section of the script, spoken by the picked provider. The key rides in
* the headers exactly like every other model call; the audio comes back as a
* blob URL the player hands to an <audio> element.
*/
export async function synthesizeSpeech(
input: { text: string; voice?: string; model?: string; providerId: string },
options: { signal?: AbortSignal } = {},
): Promise<{ url: string; type: string }> {
const response = await fetch("/api/tts", {
method: "POST",
headers: {
"Content-Type": "application/json",
...authHeaders(),
...providerHeaders(input.providerId),
...localeHeader(),
},
body: JSON.stringify({ text: input.text, voice: input.voice, model: input.model }),
signal: options.signal,
}).catch(() => {
throw new ApiError(t("api.unreachable"), "network");
});
if (!response.ok) {
const body = (await response.json().catch(() => null)) as { error?: string; kind?: string } | null;
throw new ApiError(body?.error ?? t("api.httpError", { status: response.status }), body?.kind);
}
const blob = await response.blob();
if (blob.size === 0) throw new ApiError(t("api.noAudio"), "bad_response");
return { url: URL.createObjectURL(blob), type: blob.type };
}
/* ------------------------------------------------------------------ */
/* Make my homework */
/* ------------------------------------------------------------------ */

View file

@ -307,6 +307,14 @@ const en: Record<string, string> = {
"podcast.voiceLocal": "installed",
"podcast.voiceNote": "Voices come from your browser and system — pick the one that sounds best to you.",
"podcast.voiceIos": "On iPhone, Safari offers web pages a single voice per language — the system's preferred one. For more (and better) voices: Settings → Accessibility → Spoken Content → Voices → your language, download a voice in Enhanced or Premium quality and make it the preferred one. Freezmo then reads with it.",
"podcast.voiceSystem": "System voices",
"podcast.voiceNatural": "Natural voice",
"podcast.voiceMale": "male",
"podcast.voiceFemale": "female",
"podcast.voiceNaturalNote": "Generated by your own provider key — the words are timed to the audio, so the highlight still follows the voice.",
"podcast.voiceNeedsKey": "Add an OpenAI or Groq key in Settings → Providers to use natural voices.",
"podcast.voiceNoSystem": "This device reports no system voices. The natural voice works anywhere — or add a voice in your system settings.",
"podcast.voiceGenerating": "Generating the voice…",
"podcast.voiceRobotic": "Only the basic system voices are installed. For a natural host voice, add one in your system settings (Windows: Speech; macOS: Spoken Content; Chrome ships the Google voices).",
"podcast.tapToJump": "Tap any word to jump there.",
"podcast.noVoice": "This browser has no speech voice installed, so the transcript cannot be read aloud here. You can still read along.",
@ -808,6 +816,8 @@ const en: Record<string, string> = {
"api.streamLost": "The connection dropped mid-run. Check your connection and try again.",
"api.almost": "Almost there…",
"api.noDeck": "No deck came back from the server. Check your Ollama settings and try again.",
"api.noPodcast": "No script came back from the server. Check the model setup and try again.",
"api.noAudio": "The voice came back empty. Try another voice or provider.",
"api.incomplete": "The server finished without sending a deck.",
"api.noActivities": "The generated deck had no usable activities. Try again.",
"api.gradingCancelled": "Grading was cancelled.",
@ -822,6 +832,10 @@ const en: Record<string, string> = {
"server.bodyNotJson": "Request body was not valid JSON.",
"server.invalidRequest": "Invalid request: {detail}",
"server.providerModelNeeded": "Pick the generation model in Settings → Providers.",
"server.ttsNeedsProvider": "Natural voices come from a provider — add a provider key in Settings → Providers first.",
"server.ttsUnsupported": "{provider} does not offer text-to-speech. OpenAI, Groq or a custom OpenAI-compatible server will.",
"server.ttsFailed": "{provider} could not generate the voice for this part.",
"server.ttsTimeout": "{provider} took too long to generate the voice.",
"server.needContent": "Add at least one photo of your notes or paste some text to build a deck.",
"server.needAssignment":
"Add the assignment first — paste its text, photograph it, or add a document.",
@ -1380,6 +1394,14 @@ const cs: Record<string, string> = {
"podcast.voiceLocal": "nainstalovaný",
"podcast.voiceNote": "Hlasy dodává prohlížeč a systém — vyber si ten, co zní nejlíp.",
"podcast.voiceIos": "Na iPhonu nabízí Safari webovým stránkám jen jeden hlas na jazyk — ten preferovaný v systému. Pro další (a lepší) hlasy: Nastavení → Zpřístupnění → Mluvený obsah → Hlasy → tvůj jazyk, stáhni si hlas ve verzi Vylepšený nebo Prémiový a nastav ho jako preferovaný. Freezmo pak čte s ním.",
"podcast.voiceSystem": "Systémové hlasy",
"podcast.voiceNatural": "Přirozený hlas",
"podcast.voiceMale": "mužský",
"podcast.voiceFemale": "ženský",
"podcast.voiceNaturalNote": "Generuje ho klíč tvého providera — slova jsou časovaná podle audia, takže zvýraznění drží s hlasem.",
"podcast.voiceNeedsKey": "Pro přirozené hlasy si v Nastavení → Provideři přidej klíč OpenAI nebo Groq.",
"podcast.voiceNoSystem": "Tohle zařízení nehlásí žádné systémové hlasy. Přirozený hlas funguje všude — nebo si hlas přidej v nastavení systému.",
"podcast.voiceGenerating": "Generuji hlas…",
"podcast.voiceRobotic": "Jsou nainstalované jen základní systémové hlasy. Pro přirozený hlas moderátora si nějaký přidej v nastavení systému (Windows: Řeč; macOS: Mluvený obsah; Chrome má hlasy Google).",
"podcast.tapToJump": "Klikni na kterékoli slovo a přeskočíš tam.",
"podcast.noVoice": "Tento prohlížeč nemá nainstalovaný hlas, takže se přepis nedá přečíst nahlas. Číst ho ale můžeš dál.",
@ -1881,6 +1903,8 @@ const cs: Record<string, string> = {
"api.streamLost": "Připojení se během generování přerušilo. Zkontrolujte připojení a zkuste to znovu.",
"api.almost": "Už to bude…",
"api.noDeck": "Ze serveru nepřišla žádná sada. Zkontroluj nastavení Ollamy a zkus to znovu.",
"api.noPodcast": "Ze serveru nepřišel žádný scénář. Zkontroluj nastavení modelu a zkus to znovu.",
"api.noAudio": "Hlas přišel prázdný. Zkus jiný hlas nebo providera.",
"api.incomplete": "Server skončil, aniž by poslal sadu.",
"api.noActivities": "Vygenerovaná sada neměla žádné použitelné aktivity. Zkus to znovu.",
"api.gradingCancelled": "Známkování bylo zrušeno.",
@ -1895,6 +1919,10 @@ const cs: Record<string, string> = {
"server.bodyNotJson": "Tělo požadavku nebylo platné JSON.",
"server.invalidRequest": "Neplatný požadavek: {detail}",
"server.providerModelNeeded": "Vyberte model pro generování v Nastavení → Poskytovatelé.",
"server.ttsNeedsProvider": "Přirozené hlasy dodává provider — nejdřív si v Nastavení → Provideři přidej klíč.",
"server.ttsUnsupported": "{provider} text-to-speech neumí. Umí ho OpenAI, Groq nebo vlastní OpenAI-compatible server.",
"server.ttsFailed": "{provider} nedokázal pro tuhle část vygenerovat hlas.",
"server.ttsTimeout": "{provider} generoval hlas příliš dlouho.",
"server.needContent": "Přidej aspoň jednu fotku poznámek nebo vlož text, ze kterého sada vznikne.",
"server.needAssignment":
"Nejdřív přidej zadání — vlož jeho text, vyfoť ho nebo přidej dokument.",

283
lib/podcast-audio.ts Normal file
View file

@ -0,0 +1,283 @@
"use client";
/**
* The provider-voice playback engine: one section at a time is turned into
* audio by the browser's provider key (/api/tts) and played through an
* <audio> element.
*
* Word highlighting has no boundary events here, but it does not need to
* guess: the clip's real duration is known, so each word gets a slice of it
* proportional to its spoken length. That keeps the highlight honest from the
* first word to the last, and it makes pause/seek real audio operations —
* `currentTime` is the clock.
*
* Sections are cached per session (and the next one is fetched while the
* current plays), so listening to an episode twice costs one generation per
* section, not one per replay.
*/
import { synthesizeSpeech } from "./client-api";
import { tokenizeSections, type PodcastVoice } from "./podcast-speech";
import type { PodcastSection } from "./schemas";
export type PodcastAudioEvents = {
sections: PodcastSection[];
providerId: string;
voice: string;
model?: string;
/** Initial playback speed (0.5–2); `setRate` changes it while playing. */
rate?: number;
onWord: (position: PodcastVoice) => void;
onPlay?: () => void;
onPause?: () => void;
onEnd: () => void;
onError: (message: string) => void;
/** True while a section's voice is being generated. */
onBusy?: (busy: boolean) => void;
};
type Clip = { url: string; duration: number; type: string };
export type PodcastAudio = {
play: (from?: PodcastVoice) => void;
pause: () => void;
stop: () => void;
playing: () => boolean;
position: () => PodcastVoice | null;
/** Warm the cache for a section without playing it. */
prefetch: (index: number) => void;
/** Change the playback speed without losing the generated clips. */
setRate: (rate: number) => void;
/** Drop the element and the cached clips (the player unmounted). */
dispose: () => void;
};
export function createPodcastAudio(events: PodcastAudioEvents): PodcastAudio {
const tokens = tokenizeSections(events.sections);
const clips = new Map<number, Clip>();
const loading = new Map<number, Promise<Clip>>();
const audio = new Audio();
audio.preload = "auto";
/* In the document (hidden): some engines only start audio for attached
elements, and it keeps the element inspectable for debugging. */
audio.setAttribute("data-podcast-audio", "1");
audio.style.display = "none";
if (typeof document !== "undefined") document.body.appendChild(audio);
let playing = false;
let position: PodcastVoice | null = null;
let generation = 0;
let frame = 0;
let currentSection = 0;
let rate = Math.min(2, Math.max(0.5, events.rate ?? 1));
audio.playbackRate = rate;
const wordsOf = (section: number) => tokens.filter((token) => token.section === section);
/** Each word's start time inside the clip, by its share of the text. */
const startsFor = (section: number, duration: number): number[] => {
const words = wordsOf(section);
const weights = words.map((word) => Math.max(1, word.text.replace(/[^\p{L}\p{N}]/gu, "").length));
const total = weights.reduce((sum, weight) => sum + weight, 0) || 1;
const starts: number[] = [];
let cursor = 0;
for (const weight of weights) {
starts.push((cursor / total) * duration);
cursor += weight;
}
return starts;
};
const load = (index: number): Promise<Clip> => {
const cached = clips.get(index);
if (cached) return Promise.resolve(cached);
const pending = loading.get(index);
if (pending) return pending;
const section = events.sections[index];
if (!section) return Promise.reject(new Error("no-section"));
events.onBusy?.(true);
const promise = synthesizeSpeech(
{
text: `${section.heading ? `${section.heading}. ` : ""}${section.text}`,
voice: events.voice,
model: events.model,
providerId: events.providerId,
},
)
.then(async ({ url, type }) => {
const duration = await measure(url);
const clip = { url, duration, type };
clips.set(index, clip);
loading.delete(index);
events.onBusy?.(false);
return clip;
})
.catch((error: unknown) => {
loading.delete(index);
events.onBusy?.(false);
throw error;
});
loading.set(index, promise);
return promise;
};
/** The clip's real length, read before playing so the clock can be built. */
const measure = (url: string): Promise<number> =>
new Promise((resolve) => {
const probe = new Audio();
probe.preload = "metadata";
const done = () => {
const duration = Number.isFinite(probe.duration) && probe.duration > 0 ? probe.duration : 0;
resolve(duration);
};
probe.onloadedmetadata = done;
probe.onerror = () => resolve(0);
probe.src = url;
/* Some engines never fire the event for blob URLs without a nudge. */
window.setTimeout(done, 2500);
});
const stopClock = () => {
if (frame) cancelAnimationFrame(frame);
frame = 0;
};
const startClock = () => {
const mine = generation;
const section = currentSection;
const clip = clips.get(section);
if (!clip) return;
const starts = startsFor(section, clip.duration);
const tick = () => {
if (generation !== mine) return;
const at = audio.currentTime;
/* Last word whose start is behind the cursor. */
let index = 0;
for (let i = 0; i < starts.length; i += 1) {
if (starts[i] <= at) index = i;
else break;
}
const next = { section, word: index };
if (!position || position.section !== section || position.word !== index) {
position = next;
events.onWord(next);
}
if (clip.duration > 0 && at >= clip.duration - 0.05) {
playFrom({ section: section + 1, word: 0 });
return;
}
frame = requestAnimationFrame(tick);
};
frame = requestAnimationFrame(tick);
};
const playFrom = (target: PodcastVoice) => {
const section = events.sections[target.section];
if (!section) {
playing = false;
position = null;
events.onPause?.();
events.onEnd();
return;
}
const mine = ++generation;
stopClock();
playing = false;
void load(target.section)
.then((clip) => {
if (generation !== mine) return;
const words = wordsOf(target.section);
const starts = startsFor(target.section, clip.duration);
const word = Math.min(Math.max(0, target.word), Math.max(0, words.length - 1));
currentSection = target.section;
position = { section: target.section, word };
events.onWord(position);
if (audio.src !== clip.url) audio.src = clip.url;
audio.playbackRate = rate;
try {
audio.currentTime = starts[word] ?? 0;
} catch {
/* not seekable yet: start from the top */
}
audio
.play()
.then(() => {
if (generation !== mine) return;
playing = true;
events.onPlay?.();
startClock();
/* The next part is generated while this one plays. */
if (events.sections[target.section + 1]) void load(target.section + 1).catch(() => {});
})
.catch(() => {
if (generation !== mine) return;
playing = false;
events.onError?.("blocked");
});
})
.catch((error: unknown) => {
if (generation !== mine) return;
const message = error instanceof Error ? error.message : "error";
events.onError?.(message);
});
};
audio.onended = () => {
if (currentSection + 1 >= events.sections.length) {
playing = false;
position = null;
events.onPause?.();
events.onEnd();
return;
}
playFrom({ section: currentSection + 1, word: 0 });
};
audio.onerror = () => {
playing = false;
events.onError?.("error");
};
return {
play(from) {
playFrom(from ?? position ?? { section: 0, word: 0 });
},
pause() {
generation += 1;
stopClock();
try {
audio.pause();
} catch {
/* already gone */
}
playing = false;
events.onPause?.();
},
stop() {
this.pause();
position = null;
},
playing: () => playing,
position: () => position,
prefetch(index) {
if (events.sections[index]) void load(index).catch(() => {});
},
setRate(next) {
rate = Math.min(2, Math.max(0.5, next));
audio.playbackRate = rate;
},
dispose() {
generation += 1;
stopClock();
try {
audio.pause();
audio.removeAttribute("src");
audio.remove();
} catch {
/* already gone */
}
for (const clip of clips.values()) URL.revokeObjectURL(clip.url);
clips.clear();
},
};
}

77
lib/podcast-natural.ts Normal file
View file

@ -0,0 +1,77 @@
"use client";
/**
* The provider-voice preference: which TTS provider and voice the listener
* picked for the podcast player ("system" keeps the platform's own voices).
* One blob in localStorage, read through a tiny accessor.
*/
import { PROVIDERS, TTS_PROFILES, ttsProfile, type ProviderInfo } from "./providers";
const STORAGE_KEY = "freezmo.podcast-natural.v1";
export type NaturalPref = {
source: "system" | "natural";
provider: string;
voice: string | null;
model: string | null;
};
const FALLBACK: NaturalPref = { source: "system", provider: "", voice: null, model: null };
export function readNaturalPref(): NaturalPref {
if (typeof window === "undefined") return FALLBACK;
try {
const raw = window.localStorage.getItem(STORAGE_KEY);
if (!raw) return FALLBACK;
const parsed = JSON.parse(raw) as Partial<NaturalPref> | null;
if (!parsed || typeof parsed !== "object") return FALLBACK;
return {
source: parsed.source === "natural" ? "natural" : "system",
provider: typeof parsed.provider === "string" ? parsed.provider : "",
voice: typeof parsed.voice === "string" ? parsed.voice : null,
model: typeof parsed.model === "string" ? parsed.model : null,
};
} catch {
return FALLBACK;
}
}
export function writeNaturalPref(next: Partial<NaturalPref>): NaturalPref {
const merged = { ...readNaturalPref(), ...next };
try {
window.localStorage.setItem(STORAGE_KEY, JSON.stringify(merged));
} catch {
/* private mode */
}
return merged;
}
/** Providers with a TTS profile that the listener actually has a key for. */
export function ttsProvidersWithKeys(keys: Record<string, string | undefined>): string[] {
return Object.keys(TTS_PROFILES).filter((id) => Boolean(keys[id]));
}
/** The provider to use when the listener has not chosen one. */
export function defaultTtsProvider(keys: Record<string, string | undefined>, generation?: string): string {
const available = ttsProvidersWithKeys(keys);
if (available.length === 0) return "";
if (generation && available.includes(generation)) return generation;
/* Prefer OpenAI when both are configured: its voices are the better ones. */
if (available.includes("openai")) return "openai";
return available[0];
}
export function voiceForProvider(providerId: string, wanted: string | null): string {
const profile = ttsProfile(providerId);
if (!profile) return "";
if (wanted && profile.voices.some((voice) => voice.id === wanted)) return wanted;
return profile.voices.find((voice) => voice.male)?.id ?? profile.voices[0]?.id ?? "";
}
/** Provider labels for the picker (id → label). */
export function ttsProviderLabel(id: string): string {
return PROVIDERS.find((provider: ProviderInfo) => provider.id === id)?.label ?? id;
}
export { TTS_PROFILES };

View file

@ -21,6 +21,16 @@ import type { PodcastSection } from "./schemas";
export type PodcastVoice = { section: number; word: number };
/** Tidy text for the speech engine (see the note where it is used). */
export function normaliseSpoken(raw: string): string {
return raw
.replace(/[\u0000-\u001f\u007f-\u009f\u200b-\u200f\ufeff]/g, " ")
.replace(/…/g, ", ")
.replace(/\s*—\s*/g, " — ")
.replace(/\s+/g, " ")
.trim();
}
export type PodcastSpeechEvents = {
sections: PodcastSection[];
locale: string;
@ -129,10 +139,16 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
position = { section: target.section, word };
const from = sectionTokens[word].start;
const text = section.text.slice(from);
/* Spoken text hygiene: engines click on control characters, stray
double spaces and unicode ellipses, and some pause oddly around them. */
const text = normaliseSpoken(section.text.slice(from));
const current = ++generation;
try {
window.speechSynthesis.cancel();
/* Only cancel when something is actually pending: an idle cancel makes
several engines pop. */
if (window.speechSynthesis.speaking || window.speechSynthesis.pending) {
window.speechSynthesis.cancel();
}
const utterance = new SpeechSynthesisUtterance(text);
utterance.lang = events.locale;
utterance.rate = events.rate;
@ -200,7 +216,12 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
events.onEnd();
return;
}
speakFrom({ section: target.section + 1, word: 0 });
/* A short beat between parts: speaking into the engine's tail is
what makes the boundary pop on several platforms. */
window.setTimeout(() => {
if (generation !== current) return;
speakFrom({ section: target.section + 1, word: 0 });
}, 140);
};
utterance.onerror = (event) => {

View file

@ -87,6 +87,64 @@ export const PROVIDERS: ProviderInfo[] = [
},
];
/**
* Text-to-speech for the podcast player. Only OpenAI-compatible
* `/audio/speech` endpoints are wired (OpenAI, Groq and any local server the
* custom provider points at); the rest simply do not offer TTS. Voice ids come
* from each provider's catalogue — the male ones lead because a podcast host
* was the ask.
*/
export type TtsVoice = { id: string; label: string; male?: boolean };
export type TtsProfile = {
/** Default model for /audio/speech. */
model: string;
voices: TtsVoice[];
/** Response format to request (Groq speaks WAV, everything else MP3). */
format: "mp3" | "wav";
};
export const TTS_PROFILES: Record<string, TtsProfile> = {
openai: {
model: "gpt-4o-mini-tts",
format: "mp3",
voices: [
{ id: "onyx", label: "Onyx", male: true },
{ id: "echo", label: "Echo", male: true },
{ id: "ash", label: "Ash", male: true },
{ id: "fable", label: "Fable" },
{ id: "nova", label: "Nova" },
{ id: "coral", label: "Coral" },
{ id: "shimmer", label: "Shimmer" },
],
},
groq: {
model: "playai-tts",
format: "wav",
voices: [
{ id: "Fritz-PlayAI", label: "Fritz", male: true },
{ id: "Thunder-PlayAI", label: "Thunder", male: true },
{ id: "Mason-PlayAI", label: "Mason", male: true },
{ id: "Celeste-PlayAI", label: "Celeste" },
{ id: "Cheyenne-PlayAI", label: "Cheyenne" },
],
},
custom: {
model: "tts-1",
format: "mp3",
voices: [
{ id: "onyx", label: "Onyx", male: true },
{ id: "echo", label: "Echo", male: true },
{ id: "alloy", label: "Alloy" },
{ id: "nova", label: "Nova" },
],
},
};
export function ttsProfile(providerId: string): TtsProfile | null {
return TTS_PROFILES[providerId] ?? null;
}
export function providerInfo(id: string): ProviderInfo | undefined {
return PROVIDERS.find((provider) => provider.id === id);
}