Podcast mode: notes read aloud with a lyric-style word highlight
The third segment of the note-powered wizards (Study deck / Homework / Podcast) takes the same intake - photos, documents, pasted text, the share target - and turns it into a spoken episode. - prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown, no visual references), chaptered into 3-8 sections around a length target. - /api/podcast mirrors /api/generate's plumbing (rate limit, model gate, provider keys, heartbeats, the 2x deadline, the optional vision pass). - storage: frezmo.podcasts.v1, zod-validated, included in the backup and the encrypted sync payload; a new library tab lists episodes with listen / rename / delete-with-undo, and the wizard switch deep-links in. - player: the transcript is word-wrapped spans and the word under the voice rides a lyric-style highlight that keeps itself centred. Two clocks: exact onboundary events where the browser reports them, a per-word length timer where it does not (iOS Safari); the first boundary retires the timer. Pause = cancel + re-speak from the word, which makes resume, chapter jumps and tap-any-word-to-seek the same operation; the position saves on pause, page-hide and unmount, so reopening resumes. Speed rebuilds the engine and keeps playing; the Aero soundtrack pauses itself when the voice starts; a browser with no voice gets an honest note and a readable transcript. - the script sits on a glass pane (plain paragraphs over Aero's grass failed contrast; the scan now covers the player too). tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and drives the highlight by hand: 26 checks. All suites green, contrast clean. Version 0.7.3, SW v95.
This commit is contained in:
parent
84a94581dc
commit
d95c5c7151
30 changed files with 2575 additions and 26 deletions
376
components/podcast/PodcastPlayer.tsx
Normal file
376
components/podcast/PodcastPlayer.tsx
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { useI18n } from "@/lib/i18n";
|
||||
import { createPodcastSpeech, tokenizeSections, type PodcastVoice } from "@/lib/podcast-speech";
|
||||
import { savePodcastProgress } from "@/lib/podcasts";
|
||||
import { speechAvailable, speechLocale, stopSpeaking } from "@/lib/speech";
|
||||
import type { Podcast } from "@/lib/schemas";
|
||||
import { useAppReducedMotion as useReducedMotion } from "@/lib/motion-pref";
|
||||
import { ChevronLeftIcon, ClockIcon, PauseIcon, PlayIcon, RefreshIcon } from "@/components/icons";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
/** Playback speeds, cycled by one button. */
|
||||
const RATES = [1, 1.15, 1.3, 0.9] as const;
|
||||
|
||||
/**
|
||||
* Reads an episode aloud and highlights the word being spoken — the transcript
|
||||
* is the player, Spotify-lyrics style: past words stay dark, upcoming words
|
||||
* are dimmed, the word under the voice rides a rounded highlight that keeps
|
||||
* itself centred as the episode plays.
|
||||
*
|
||||
* The heavy lifting (boundary events, the timer fallback, seeking by
|
||||
* re-speaking) lives in lib/podcast-speech; this component owns the layout,
|
||||
* the controls and the saved position.
|
||||
*/
|
||||
export function PodcastPlayer({ episode }: { episode: Podcast }) {
|
||||
const { t, plural, locale } = useI18n();
|
||||
const reduceMotion = useReducedMotion();
|
||||
const [rate, setRate] = useState<number>(1);
|
||||
const [playing, setPlaying] = useState(false);
|
||||
const [position, setPosition] = useState<PodcastVoice | null>(null);
|
||||
const [unsupported, setUnsupported] = useState(false);
|
||||
const transcript = useRef<HTMLDivElement>(null);
|
||||
const activeWord = useRef<HTMLElement | null>(null);
|
||||
/** Latest position for the unmount/pause save, without re-creating callbacks. */
|
||||
const latest = useRef<PodcastVoice | null>(episode.progress ?? null);
|
||||
|
||||
const tokens = useMemo(() => tokenizeSections(episode.sections), [episode.sections]);
|
||||
const supported = useMemo(() => speechAvailable(), []);
|
||||
|
||||
const speech = useMemo(() => {
|
||||
if (!supported) return null;
|
||||
return createPodcastSpeech({
|
||||
sections: episode.sections,
|
||||
locale: speechLocale(episode.language, locale),
|
||||
rate,
|
||||
onWord: (next) => {
|
||||
latest.current = next;
|
||||
setPosition(next);
|
||||
},
|
||||
onPlay: () => {
|
||||
setPlaying(true);
|
||||
/* The Aero soundtrack steps aside for the voice (see AeroMusic). */
|
||||
window.dispatchEvent(new CustomEvent("freezmo:foreground-audio", { detail: "podcast" }));
|
||||
},
|
||||
onPause: () => setPlaying(false),
|
||||
onEnd: () => {
|
||||
setPlaying(false);
|
||||
setPosition(null);
|
||||
latest.current = null;
|
||||
savePodcastProgress(episode.id, { section: 0, word: 0 });
|
||||
},
|
||||
onError: (message) => {
|
||||
if (message === "unsupported") setUnsupported(true);
|
||||
setPlaying(false);
|
||||
},
|
||||
});
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps -- rebuilt only when the episode, the speed or the locale changes
|
||||
}, [episode.id, episode.sections, rate, supported, locale]);
|
||||
|
||||
/* Resume where the listener left off (a reopened episode starts there). */
|
||||
useEffect(() => {
|
||||
if (episode.progress) {
|
||||
latest.current = { section: episode.progress.section, word: episode.progress.word };
|
||||
setPosition({ section: episode.progress.section, word: episode.progress.word });
|
||||
}
|
||||
}, [episode.id, episode.progress]);
|
||||
|
||||
const persist = useCallback(() => {
|
||||
if (latest.current) savePodcastProgress(episode.id, latest.current);
|
||||
}, [episode.id]);
|
||||
|
||||
/* The engine instance is rebuilt on a speed change; keep a ref so the
|
||||
unmount cleanup always stops whichever one is current. */
|
||||
const speechRef = useRef(speech);
|
||||
const playingRef = useRef(false);
|
||||
useEffect(() => {
|
||||
speechRef.current = speech;
|
||||
}, [speech]);
|
||||
useEffect(() => {
|
||||
playingRef.current = playing;
|
||||
}, [playing]);
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
persist();
|
||||
speechRef.current?.stop();
|
||||
};
|
||||
}, [persist]);
|
||||
|
||||
/* Closing the tab, switching apps or hard-navigating away: the unmount
|
||||
cleanup never runs there, so the position also rides the page lifecycle. */
|
||||
useEffect(() => {
|
||||
const save = () => persist();
|
||||
const onHide = () => {
|
||||
if (document.visibilityState === "hidden") save();
|
||||
};
|
||||
window.addEventListener("pagehide", save);
|
||||
document.addEventListener("visibilitychange", onHide);
|
||||
return () => {
|
||||
window.removeEventListener("pagehide", save);
|
||||
document.removeEventListener("visibilitychange", onHide);
|
||||
};
|
||||
}, [persist]);
|
||||
|
||||
/* A rebuilt engine (speed change) resumes the same word when it was
|
||||
playing — the listener should not notice the swap. */
|
||||
const firstEngine = useRef(true);
|
||||
useEffect(() => {
|
||||
if (firstEngine.current) {
|
||||
firstEngine.current = false;
|
||||
return;
|
||||
}
|
||||
if (!playingRef.current || !speech) return;
|
||||
speech.play(latest.current ?? { section: 0, word: 0 });
|
||||
}, [speech]);
|
||||
|
||||
/* The highlight: one class on the active word (imperative, so a few hundred
|
||||
spans do not re-render per word), then keep it in the comfortable band. */
|
||||
useEffect(() => {
|
||||
const container = transcript.current;
|
||||
if (!container) return;
|
||||
activeWord.current?.removeAttribute("data-word-active");
|
||||
activeWord.current?.classList.remove("podcast-word-active");
|
||||
if (!position) {
|
||||
activeWord.current = null;
|
||||
return;
|
||||
}
|
||||
const index = tokens.findIndex((token) => token.section === position.section && token.index === position.word);
|
||||
if (index === -1) return;
|
||||
const element = container.querySelector<HTMLElement>(`[data-word="${index}"]`);
|
||||
if (!element) return;
|
||||
element.setAttribute("data-word-active", "1");
|
||||
element.classList.add("podcast-word-active");
|
||||
activeWord.current = element;
|
||||
|
||||
const box = element.getBoundingClientRect();
|
||||
const view = container.getBoundingClientRect();
|
||||
const top = box.top - view.top;
|
||||
const comfortable = view.height * 0.38;
|
||||
if (top < view.height * 0.18 || top > view.height * 0.62) {
|
||||
container.scrollTo({
|
||||
top: container.scrollTop + top - comfortable,
|
||||
behavior: reduceMotion ? "auto" : "smooth",
|
||||
});
|
||||
}
|
||||
}, [position, tokens, reduceMotion]);
|
||||
|
||||
const sections = episode.sections;
|
||||
const words = tokens.length;
|
||||
const done = useMemo(() => {
|
||||
if (!position) return 0;
|
||||
return tokens.findIndex((token) => token.section === position.section && token.index === position.word);
|
||||
}, [position, tokens]);
|
||||
|
||||
const toggle = () => {
|
||||
if (!speech) return;
|
||||
if (playing) {
|
||||
speech.pause();
|
||||
persist();
|
||||
return;
|
||||
}
|
||||
/* Other readers (a card's read-aloud) yield to the episode. */
|
||||
stopSpeaking();
|
||||
speech.play(position ?? { section: 0, word: 0 });
|
||||
};
|
||||
|
||||
const jump = (section: number) => {
|
||||
if (!speech) return;
|
||||
const next = { section, word: 0 };
|
||||
latest.current = next;
|
||||
setPosition(next);
|
||||
stopSpeaking();
|
||||
speech.play(next);
|
||||
};
|
||||
|
||||
const jumpWord = (index: number) => {
|
||||
if (!speech) return;
|
||||
const token = tokens[index];
|
||||
if (!token) return;
|
||||
const next = { section: token.section, word: token.index };
|
||||
latest.current = next;
|
||||
setPosition(next);
|
||||
stopSpeaking();
|
||||
speech.play(next);
|
||||
};
|
||||
|
||||
/* Word spans, built once per section: the highlight is a class flip. */
|
||||
const rendered = useMemo(() => {
|
||||
return sections.map((section, sectionIndex) => {
|
||||
const sectionTokens = tokens.filter((token) => token.section === sectionIndex);
|
||||
const nodes: React.ReactNode[] = [];
|
||||
let cursor = 0;
|
||||
sectionTokens.forEach((token) => {
|
||||
if (token.start > cursor) nodes.push(section.text.slice(cursor, token.start));
|
||||
const globalIndex = token.global;
|
||||
nodes.push(
|
||||
<span key={`${token.start}`} data-word={globalIndex} className="podcast-word">
|
||||
{section.text.slice(token.start, token.end)}
|
||||
</span>,
|
||||
);
|
||||
cursor = token.end;
|
||||
});
|
||||
if (cursor < section.text.length) nodes.push(section.text.slice(cursor));
|
||||
return { section, sectionIndex, nodes };
|
||||
});
|
||||
}, [sections, tokens]);
|
||||
|
||||
const currentSection = position?.section ?? 0;
|
||||
|
||||
return (
|
||||
<div className="frost-bg fixed inset-0 z-[45] flex flex-col fade-in-mobile">
|
||||
<header className="pt-safe absolute inset-x-0 top-0 z-30 px-3">
|
||||
<div className="study-chrome mx-auto flex max-w-3xl items-center gap-2.5 rounded-full px-2.5 py-2">
|
||||
<Link
|
||||
href="/?tab=podcasts"
|
||||
aria-label={t("feed.back")}
|
||||
title={t("feed.back")}
|
||||
className="grid size-10 shrink-0 place-items-center rounded-full bg-[var(--surface-sunken)] text-[var(--text)] transition hover:bg-[var(--hover)]"
|
||||
>
|
||||
<ChevronLeftIcon className="size-4" />
|
||||
</Link>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="truncate text-[13px] font-semibold leading-tight text-[var(--text)]">{episode.title}</p>
|
||||
<p className="truncate text-[11.5px] text-[var(--text-muted)]">
|
||||
{t("podcast.sectionOf", { n: Math.min(currentSection + 1, sections.length), total: sections.length })}
|
||||
{" · "}
|
||||
{sections[currentSection]?.heading ?? ""}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
const index = RATES.indexOf(rate as (typeof RATES)[number]);
|
||||
setRate(RATES[(index + 1) % RATES.length]);
|
||||
}}
|
||||
aria-label={t("podcast.speedAria")}
|
||||
title={t("podcast.speedAria")}
|
||||
className="shrink-0 rounded-full bg-[var(--surface-sunken)] px-3 py-2 text-[12px] font-extrabold tabular-nums text-[var(--text)] transition hover:bg-[var(--hover)]"
|
||||
>
|
||||
{rate}×
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div
|
||||
ref={transcript}
|
||||
data-testid="podcast-transcript"
|
||||
onClick={(event) => {
|
||||
/* Word taps seek: the transcript is the scrubber, like lyrics. */
|
||||
const target = event.target as HTMLElement;
|
||||
if (!target.classList.contains("podcast-word")) return;
|
||||
const index = Number(target.dataset.word);
|
||||
if (Number.isInteger(index)) jumpWord(index);
|
||||
}}
|
||||
className="no-scrollbar min-h-0 flex-1 overflow-y-auto overscroll-contain px-4 pb-40 pt-24"
|
||||
>
|
||||
<div className="mx-auto w-full max-w-3xl space-y-6">
|
||||
{!supported || unsupported ? (
|
||||
<p className="rounded-2xl border border-amber-300/50 bg-amber-50/70 px-3.5 py-3 text-[12.5px] leading-relaxed text-amber-800 dark:border-amber-400/25 dark:bg-amber-400/10 dark:text-amber-200">
|
||||
{t("podcast.noVoice")}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<div className="card space-y-2 p-5">
|
||||
<h1 className="text-[22px] font-extrabold leading-tight tracking-[-0.01em] text-[var(--text)]">
|
||||
{episode.title}
|
||||
</h1>
|
||||
{episode.summary ? (
|
||||
<p className="text-[13.5px] leading-relaxed text-[var(--text-muted)]">{episode.summary}</p>
|
||||
) : null}
|
||||
<p className="flex items-center gap-1.5 text-[12px] font-medium text-[var(--text-subtle)]">
|
||||
<ClockIcon className="size-3.5" />
|
||||
{plural(episode.sections.length, "podcast.sectionCount")} · {plural(words, "podcast.wordCount")}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* The script reads on a glass pane: in Aero the wallpaper behind
|
||||
plain paragraphs is bright grass and sun (the contrast scan caught
|
||||
it), and the same treatment keeps every theme consistent. */}
|
||||
<div className="card space-y-6 p-5" data-testid="podcast-script">
|
||||
{rendered.map(({ section, sectionIndex, nodes }) => (
|
||||
<section key={sectionIndex} className="space-y-2">
|
||||
<h2 className="flex items-center gap-2 text-[12px] font-bold uppercase tracking-wide text-[var(--text-subtle)]">
|
||||
<span className="grid size-6 place-items-center rounded-full bg-[var(--surface-sunken)] text-[11px] tabular-nums">
|
||||
{sectionIndex + 1}
|
||||
</span>
|
||||
{section.heading}
|
||||
</h2>
|
||||
<p
|
||||
className={cn(
|
||||
"text-[17px] leading-[1.85] text-[var(--text-muted)] transition-colors sm:text-[18px]",
|
||||
sectionIndex === currentSection && "text-[var(--text)]",
|
||||
)}
|
||||
>
|
||||
{nodes}
|
||||
</p>
|
||||
</section>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<p className="pb-6 text-center text-[11.5px] text-[var(--text-subtle)]">{t("podcast.tapToJump")}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Controls: chapter list on the left, transport in the middle. */}
|
||||
<div className="pointer-events-none absolute inset-x-0 bottom-0 z-30 pb-safe">
|
||||
<div className="pointer-events-auto mx-auto max-w-3xl px-3 pb-3">
|
||||
<div className="study-chrome rounded-[1.4rem] px-3 py-2.5">
|
||||
<div className="mb-2 h-1 overflow-hidden rounded-full bg-[var(--track)]" aria-hidden>
|
||||
<div
|
||||
className="h-full rounded-full grad-ice transition-[width] duration-200"
|
||||
style={{ width: words ? `${Math.round((done / words) * 100)}%` : "0%" }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
disabled={!supported}
|
||||
aria-label={playing ? t("podcast.pause") : t("podcast.play")}
|
||||
title={playing ? t("podcast.pause") : t("podcast.play")}
|
||||
className="grid size-12 shrink-0 place-items-center rounded-full grad-ice text-[#06111f] transition disabled:opacity-45"
|
||||
>
|
||||
{playing ? <PauseIcon className="size-5" /> : <PlayIcon className="size-5" />}
|
||||
</button>
|
||||
<div className="no-scrollbar flex min-w-0 flex-1 items-center gap-1.5 overflow-x-auto">
|
||||
{sections.map((section, index) => (
|
||||
<button
|
||||
key={index}
|
||||
type="button"
|
||||
onClick={() => jump(index)}
|
||||
aria-current={index === currentSection ? "true" : undefined}
|
||||
className={cn(
|
||||
"shrink-0 rounded-full px-3 py-2 text-[12px] font-semibold transition",
|
||||
index === currentSection
|
||||
? "bg-[var(--nav-active-bg)] text-[var(--text)] ring-1 ring-[var(--nav-active-border)]"
|
||||
: "bg-[var(--surface-sunken)] text-[var(--text-muted)] hover:bg-[var(--hover)]",
|
||||
)}
|
||||
>
|
||||
{section.heading}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
speech?.stop();
|
||||
setPosition({ section: 0, word: 0 });
|
||||
latest.current = { section: 0, word: 0 };
|
||||
savePodcastProgress(episode.id, { section: 0, word: 0 });
|
||||
}}
|
||||
aria-label={t("podcast.restart")}
|
||||
title={t("podcast.restart")}
|
||||
className="grid size-10 shrink-0 place-items-center rounded-full bg-[var(--surface-sunken)] text-[var(--text-muted)] transition hover:bg-[var(--hover)] hover:text-[var(--text)]"
|
||||
>
|
||||
<RefreshIcon className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue