Podcast voices: natural provider TTS, and system-voice hygiene
Two reports, one root: the platform's speech engine. "Loud cracking at the end of every sentence" is the browser/OS TTS popping between its internal sentence chunks (an idle cancel makes it worse). Hygiene for that path: never cancel when nothing is pending, normalise the spoken text (control characters, ellipses, dashes), and a 140ms beat between sections so speech never runs into the engine's tail. The real fix is a second voice source that does not use the platform engine: - lib/providers carries TTS profiles (OpenAI gpt-4o-mini-tts, Groq playai-tts, any OpenAI-compatible /audio/speech behind the custom provider), male voices first. - /api/tts proxies them with the same guards and the key still in headers. - the picker gains System / Natural, with the provider's voice list. - lib/podcast-audio plays one clip per section through a hidden <audio> element, measures the clip's real duration and gives each word a slice of it proportional to its spoken length - the highlight follows the audio exactly, and pause/seek/chapters become real audio operations. Clips cache per session, the next section generates while the current plays, and a voice change disposes the element and its blobs. Speed applies live (setRate) without dropping the cache. podcast-test stubs /api/tts with a real silent WAV and pins the path: the voice list, the request body, exactly one playing element, and the highlight riding currentTime. 43 checks. tests/lib.mjs launches Chromium with the autoplay policy relaxed (a real device has the listener's gesture). Version 0.7.6, SW v98.
This commit is contained in:
parent
9882d775e7
commit
8e1cd0855d
13 changed files with 997 additions and 63 deletions
|
|
@ -4,6 +4,16 @@ import Link from "next/link";
|
|||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { useI18n } from "@/lib/i18n";
|
||||
import { createPodcastAudio, type PodcastAudio } from "@/lib/podcast-audio";
|
||||
import {
|
||||
defaultTtsProvider,
|
||||
readNaturalPref,
|
||||
ttsProviderLabel,
|
||||
voiceForProvider,
|
||||
writeNaturalPref,
|
||||
} from "@/lib/podcast-natural";
|
||||
import { TTS_PROFILES } from "@/lib/providers";
|
||||
import { useProviders } from "@/lib/providers-client";
|
||||
import {
|
||||
createPodcastSpeech,
|
||||
tokenizeSections,
|
||||
|
|
@ -54,6 +64,79 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
const [position, setPosition] = useState<PodcastVoice | null>(null);
|
||||
const [unsupported, setUnsupported] = useState(false);
|
||||
const [voiceOpen, setVoiceOpen] = useState(false);
|
||||
/** "system" = the platform's voices; "natural" = the provider's TTS. */
|
||||
const [voiceSource, setVoiceSource] = useState<"system" | "natural">(() => readNaturalPref().source);
|
||||
const [naturalProvider, setNaturalProvider] = useState<string>(() => readNaturalPref().provider);
|
||||
const [naturalVoiceId, setNaturalVoiceId] = useState<string | null>(() => readNaturalPref().voice);
|
||||
const [naturalError, setNaturalError] = useState<string | null>(null);
|
||||
const [naturalBusy, setNaturalBusy] = useState(false);
|
||||
const providers = useProviders();
|
||||
|
||||
/* Either engine drives the same onWord/onEnd events; the transport below
|
||||
routes to whichever one the listeners picked. */
|
||||
const natural = useMemo<PodcastAudio | null>(() => {
|
||||
if (voiceSource !== "natural" || !naturalProvider) return null;
|
||||
return createPodcastAudio({
|
||||
sections: episode.sections,
|
||||
providerId: naturalProvider,
|
||||
voice: voiceForProvider(naturalProvider, naturalVoiceId),
|
||||
model: readNaturalPref().model ?? undefined,
|
||||
rate,
|
||||
onWord: (next) => {
|
||||
latest.current = next;
|
||||
setPosition(next);
|
||||
},
|
||||
onPlay: () => {
|
||||
setPlaying(true);
|
||||
setNaturalError(null);
|
||||
window.dispatchEvent(new CustomEvent("freezmo:foreground-audio", { detail: "podcast" }));
|
||||
},
|
||||
onPause: () => setPlaying(false),
|
||||
onEnd: () => {
|
||||
setPlaying(false);
|
||||
setPosition(null);
|
||||
latest.current = null;
|
||||
savePodcastProgress(episode.id, { section: 0, word: 0 });
|
||||
},
|
||||
onError: (message) => {
|
||||
setPlaying(false);
|
||||
setNaturalError(message);
|
||||
},
|
||||
onBusy: setNaturalBusy,
|
||||
});
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps -- speed is applied live (setRate below), so it must not rebuild the engine
|
||||
}, [voiceSource, naturalProvider, naturalVoiceId, episode.id, episode.sections]);
|
||||
|
||||
/* Speed changes apply to the element, not to the engine's cache. */
|
||||
useEffect(() => {
|
||||
natural?.setRate(rate);
|
||||
}, [natural, rate]);
|
||||
|
||||
/* The engine owns an <audio> element: release it when the engine is
|
||||
replaced (a different voice) or the player closes. */
|
||||
useEffect(() => {
|
||||
if (!natural) return;
|
||||
return () => {
|
||||
try {
|
||||
natural.dispose();
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
};
|
||||
}, [natural]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => naturalRef.current?.dispose();
|
||||
}, []);
|
||||
|
||||
/* The default pick the first time someone opens the natural panel. */
|
||||
useEffect(() => {
|
||||
if (naturalProvider) return;
|
||||
const keys = providers.keys;
|
||||
const fallback = defaultTtsProvider(keys, providers.generation?.provider);
|
||||
if (fallback) setNaturalProvider(fallback);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps -- once per mount
|
||||
}, [providers.keys]);
|
||||
const transcript = useRef<HTMLDivElement>(null);
|
||||
const activeWord = useRef<HTMLElement | null>(null);
|
||||
/** Line-following state: the last followed line and a manual-scroll pause. */
|
||||
|
|
@ -100,6 +183,10 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
}
|
||||
return bestVoice(voiceLocale, voiceList);
|
||||
}, [voiceURI, availableVoices, voiceLocale, voiceList]);
|
||||
const ttsProviderIds = useMemo(
|
||||
() => Object.keys(TTS_PROFILES).filter((id) => Boolean(providers.keys[id])),
|
||||
[providers.keys],
|
||||
);
|
||||
const roboticOnly = useMemo(
|
||||
() => onlyRoboticVoices(voiceLocale, voiceList),
|
||||
[voiceLocale, voiceList],
|
||||
|
|
@ -172,10 +259,14 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
/* The engine instance is rebuilt on a speed change; keep a ref so the
|
||||
unmount cleanup always stops whichever one is current. */
|
||||
const speechRef = useRef(speech);
|
||||
const naturalRef = useRef<PodcastAudio | null>(natural);
|
||||
const playingRef = useRef(false);
|
||||
useEffect(() => {
|
||||
speechRef.current = speech;
|
||||
}, [speech]);
|
||||
useEffect(() => {
|
||||
naturalRef.current = natural;
|
||||
}, [natural]);
|
||||
useEffect(() => {
|
||||
playingRef.current = playing;
|
||||
}, [playing]);
|
||||
|
|
@ -289,27 +380,33 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
);
|
||||
}, [position, tokens]);
|
||||
|
||||
const naturalActive = voiceSource === "natural" && Boolean(natural);
|
||||
/** The engines share their shape; the transport routes by source. */
|
||||
const canPlay = naturalActive ? true : supported;
|
||||
|
||||
const harness = () => (naturalActive ? natural : speech);
|
||||
|
||||
const toggle = () => {
|
||||
if (!speech) return;
|
||||
if (!canPlay) return;
|
||||
if (playing) {
|
||||
speech.pause();
|
||||
harness()?.pause();
|
||||
persist();
|
||||
return;
|
||||
}
|
||||
/* Other readers (a card's read-aloud) yield to the episode. */
|
||||
stopSpeaking();
|
||||
speech.play(position ?? { section: 0, word: 0 });
|
||||
harness()?.play(position ?? { section: 0, word: 0 });
|
||||
};
|
||||
|
||||
const seek = (next: PodcastVoice) => {
|
||||
if (!speech) return;
|
||||
if (!canPlay) return;
|
||||
latest.current = next;
|
||||
setPosition(next);
|
||||
/* A seek is an instruction to show that place: follow it at once. */
|
||||
lastLineTop.current = null;
|
||||
followPausedUntil.current = 0;
|
||||
stopSpeaking();
|
||||
speech.play(next);
|
||||
harness()?.play(next);
|
||||
};
|
||||
|
||||
const jump = (section: number) => seek({ section, word: 0 });
|
||||
|
|
@ -375,7 +472,7 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
{sections[currentSection]?.heading ?? ""}
|
||||
</p>
|
||||
</div>
|
||||
{availableVoices.length > 0 ? (
|
||||
{availableVoices.length > 0 || ttsProviderIds.length > 0 ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setVoiceOpen((open) => !open)}
|
||||
|
|
@ -398,6 +495,14 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
) : null}
|
||||
</button>
|
||||
) : null}
|
||||
{naturalBusy ? (
|
||||
<span
|
||||
role="status"
|
||||
className="shrink-0 rounded-full bg-[var(--surface-sunken)] px-2.5 py-2 text-[11px] font-semibold text-[var(--text-muted)]"
|
||||
>
|
||||
{t("podcast.voiceGenerating")}
|
||||
</span>
|
||||
) : null}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
|
|
@ -426,7 +531,7 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
className="no-scrollbar min-h-0 flex-1 overflow-y-auto overscroll-contain px-4 pb-40 pt-24"
|
||||
>
|
||||
<div className="mx-auto w-full max-w-3xl space-y-6">
|
||||
{!supported || unsupported ? (
|
||||
{(!supported || unsupported) && !naturalActive ? (
|
||||
<p className="rounded-2xl border border-amber-300/50 bg-amber-50/70 px-3.5 py-3 text-[12.5px] leading-relaxed text-amber-800 dark:border-amber-400/25 dark:bg-amber-400/10 dark:text-amber-200">
|
||||
{t("podcast.noVoice")}
|
||||
{isAppleMobile() ? ` ${t("podcast.voiceIos")}` : ""}
|
||||
|
|
@ -497,62 +602,147 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
{t("common.close")}
|
||||
</button>
|
||||
</div>
|
||||
{voiceURI ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setVoiceURI(null);
|
||||
window.localStorage.removeItem(
|
||||
"freezmo.podcast-voice.v1",
|
||||
);
|
||||
}}
|
||||
className="nav-row touch-target w-full text-left text-[13px]"
|
||||
>
|
||||
{t("podcast.voiceAuto")}
|
||||
</button>
|
||||
) : null}
|
||||
{availableVoices.map((entry) => {
|
||||
const active = entry.voiceURI === voice?.voiceURI;
|
||||
return (
|
||||
{/* Source: the platform's voices, or the provider's TTS. */}
|
||||
<div className="grid grid-cols-2 gap-1.5 px-1 pb-1">
|
||||
{(["system", "natural"] as const).map((source) => (
|
||||
<button
|
||||
key={entry.voiceURI}
|
||||
key={source}
|
||||
type="button"
|
||||
aria-pressed={active}
|
||||
aria-pressed={voiceSource === source}
|
||||
onClick={() => {
|
||||
setVoiceURI(entry.voiceURI);
|
||||
setStoredVoice(voiceLocale, entry.voiceURI);
|
||||
setVoiceOpen(false);
|
||||
setVoiceSource(source);
|
||||
writeNaturalPref({ source });
|
||||
setNaturalError(null);
|
||||
/* Swapping engines mid-sentence: stop the other one. */
|
||||
if (playing) {
|
||||
harness()?.pause();
|
||||
setPlaying(false);
|
||||
}
|
||||
}}
|
||||
className={cn(
|
||||
"nav-row touch-target flex w-full items-center gap-2 text-left text-[13px]",
|
||||
active &&
|
||||
"border-[var(--nav-active-border)] bg-[var(--nav-active-bg)]",
|
||||
"touch-target rounded-xl border px-3 py-2 text-[12.5px] font-semibold transition",
|
||||
voiceSource === source
|
||||
? "border-[var(--nav-active-border)] bg-[var(--nav-active-bg)] text-[var(--text)]"
|
||||
: "border-[var(--hairline)] bg-[var(--surface-sunken)] text-[var(--text-muted)] hover:bg-[var(--hover)]",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block truncate font-medium text-[var(--text)]">
|
||||
{entry.name}
|
||||
</span>
|
||||
<span className="block text-[11.5px] text-[var(--text-subtle)]">
|
||||
{entry.lang}
|
||||
{entry.localService
|
||||
? ` · ${t("podcast.voiceLocal")}`
|
||||
: ""}
|
||||
</span>
|
||||
</span>
|
||||
{active ? (
|
||||
<CheckCircleIcon className="size-4 shrink-0 text-accent-600 dark:text-accent-200" />
|
||||
) : null}
|
||||
{source === "system" ? t("podcast.voiceSystem") : t("podcast.voiceNatural")}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
<p className="px-1 pt-1 text-[11.5px] leading-relaxed text-[var(--text-subtle)]">
|
||||
{iosLimited
|
||||
? t("podcast.voiceIos")
|
||||
: roboticOnly
|
||||
? t("podcast.voiceRobotic")
|
||||
: t("podcast.voiceNote")}
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{voiceSource === "system" ? (
|
||||
<>
|
||||
{availableVoices.length === 0 ? (
|
||||
<p className="px-1 pt-1 text-[11.5px] leading-relaxed text-[var(--text-subtle)]">
|
||||
{t("podcast.voiceNoSystem")}
|
||||
</p>
|
||||
) : null}
|
||||
{voiceURI ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setVoiceURI(null);
|
||||
window.localStorage.removeItem("freezmo.podcast-voice.v1");
|
||||
}}
|
||||
className="nav-row touch-target w-full text-left text-[13px]"
|
||||
>
|
||||
{t("podcast.voiceAuto")}
|
||||
</button>
|
||||
) : null}
|
||||
{availableVoices.map((entry) => {
|
||||
const active = entry.voiceURI === voice?.voiceURI;
|
||||
return (
|
||||
<button
|
||||
key={entry.voiceURI}
|
||||
type="button"
|
||||
aria-pressed={active}
|
||||
onClick={() => {
|
||||
setVoiceURI(entry.voiceURI);
|
||||
setStoredVoice(voiceLocale, entry.voiceURI);
|
||||
setVoiceOpen(false);
|
||||
}}
|
||||
className={cn(
|
||||
"nav-row touch-target flex w-full items-center gap-2 text-left text-[13px]",
|
||||
active && "border-[var(--nav-active-border)] bg-[var(--nav-active-bg)]",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block truncate font-medium text-[var(--text)]">{entry.name}</span>
|
||||
<span className="block text-[11.5px] text-[var(--text-subtle)]">
|
||||
{entry.lang}
|
||||
{entry.localService ? ` · ${t("podcast.voiceLocal")}` : ""}
|
||||
</span>
|
||||
</span>
|
||||
{active ? <CheckCircleIcon className="size-4 shrink-0 text-accent-600 dark:text-accent-200" /> : null}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
<p className="px-1 pt-1 text-[11.5px] leading-relaxed text-[var(--text-subtle)]">
|
||||
{iosLimited ? t("podcast.voiceIos") : roboticOnly ? t("podcast.voiceRobotic") : t("podcast.voiceNote")}
|
||||
</p>
|
||||
</>
|
||||
) : ttsProviderIds.length === 0 ? (
|
||||
<p className="px-1 pt-1 text-[11.5px] leading-relaxed text-[var(--text-subtle)]">
|
||||
{t("podcast.voiceNeedsKey")}
|
||||
</p>
|
||||
) : (
|
||||
<>
|
||||
{ttsProviderIds.length > 1 ? (
|
||||
<div className="flex flex-wrap gap-1.5 px-1 pb-1 pt-1">
|
||||
{ttsProviderIds.map((id) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
aria-pressed={naturalProvider === id}
|
||||
onClick={() => {
|
||||
setNaturalProvider(id);
|
||||
setNaturalVoiceId(null);
|
||||
writeNaturalPref({ provider: id, voice: null });
|
||||
}}
|
||||
className={cn(
|
||||
"rounded-full px-2.5 py-1.5 text-[11.5px] font-semibold transition",
|
||||
naturalProvider === id
|
||||
? "bg-[var(--nav-active-bg)] text-[var(--text)] ring-1 ring-[var(--nav-active-border)]"
|
||||
: "bg-[var(--surface-sunken)] text-[var(--text-muted)] hover:bg-[var(--hover)]",
|
||||
)}
|
||||
>
|
||||
{ttsProviderLabel(id)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
{(TTS_PROFILES[naturalProvider]?.voices ?? []).map((entry) => {
|
||||
const active = entry.id === voiceForProvider(naturalProvider, naturalVoiceId);
|
||||
return (
|
||||
<button
|
||||
key={entry.id}
|
||||
type="button"
|
||||
aria-pressed={active}
|
||||
onClick={() => {
|
||||
setNaturalVoiceId(entry.id);
|
||||
writeNaturalPref({ voice: entry.id });
|
||||
}}
|
||||
className={cn(
|
||||
"nav-row touch-target flex w-full items-center gap-2 text-left text-[13px]",
|
||||
active && "border-[var(--nav-active-border)] bg-[var(--nav-active-bg)]",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block truncate font-medium text-[var(--text)]">{entry.label}</span>
|
||||
<span className="block text-[11.5px] text-[var(--text-subtle)]">
|
||||
{entry.male ? t("podcast.voiceMale") : t("podcast.voiceFemale")}
|
||||
</span>
|
||||
</span>
|
||||
{active ? <CheckCircleIcon className="size-4 shrink-0 text-accent-600 dark:text-accent-200" /> : null}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
<p className="px-1 pt-1 text-[11.5px] leading-relaxed text-[var(--text-subtle)]">
|
||||
{naturalError ?? t("podcast.voiceNaturalNote")}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
|
@ -573,7 +763,7 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
disabled={!supported}
|
||||
disabled={!canPlay}
|
||||
aria-label={playing ? t("podcast.pause") : t("podcast.play")}
|
||||
title={playing ? t("podcast.pause") : t("podcast.play")}
|
||||
className="grid size-12 shrink-0 place-items-center rounded-full grad-ice text-[#06111f] transition disabled:opacity-45"
|
||||
|
|
@ -605,7 +795,7 @@ export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
|||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
speech?.stop();
|
||||
harness()?.stop();
|
||||
setPosition({ section: 0, word: 0 });
|
||||
latest.current = { section: 0, word: 0 };
|
||||
savePodcastProgress(episode.id, { section: 0, word: 0 });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue