diff --git a/README.md b/README.md index a9a8b61..60e8a3f 100644 --- a/README.md +++ b/README.md @@ -1492,6 +1492,32 @@ Things I chose while building, with the reasoning: into view as you type. No strings, so no i18n; the additive change sits on top of the merged scene-heat rework unchanged. +103. **Podcast mode: your notes, read aloud, highlighted word by word.** The third segment of + the note-powered wizards (Study deck ⇄ Homework ⇄ Podcast) takes the same intake — photos, + documents, pasted text, the share target — and turns it into a spoken episode instead of + activities: `PODCAST_SYSTEM_PROMPT` asks for plain prose written for the ear (no markdown, no + lists, no "as you can see"), chaptered into 3-8 sections of 60-140 words around a + length target, and the new `/api/podcast` route mirrors `/api/generate`'s plumbing exactly + (rate limit, model gate, provider keys, heartbeats, the 2x deadline, the optional vision pass). + Episodes are stored like homework results (`freezmo.podcasts.v1`, zod-validated, riding the + backup and the encrypted sync payload) and listed in a new library tab with listen / rename / + delete-with-undo. + The player is the Spotify-lyrics part: the transcript is word-wrapped spans, and the word + under the voice rides a highlight that keeps itself centred as the episode plays. Two clocks + drive it — `onboundary` where the browser reports it (the character index maps onto the word + list) and a per-word length timer where it does not (iOS Safari, several Android engines); + the first boundary seen retires the timer for the session. Pause is a cancel plus a + re-speak-from-word, which makes "resume", "jump to a chapter" and "tap any word to seek" the + same operation, and the position is saved on pause, page-hide and unmount, so reopening an + episode resumes where you stopped. A speed button rebuilds the engine and keeps playing, the + Aero soundtrack pauses itself when the voice starts (`freezmo:foreground-audio`), and a + browser with no installed voice gets an honest note with the transcript still readable. The + script sits on one glass pane — in Aero, plain paragraphs over grass and sun failed + contrast, and the scan now covers this screen too. `tests/podcast-test.mjs` (17th suite) + stubs both the model and the speech engine: it drives a boundary event, checks the + pause/resume word, the chapter jump, the speed change, the stored position and the plugin's + 400/stream guards (26 checks). + 102. **An API key is not a login.** "When trying to use a different provider on the web app it glitches out completely" — a photo of the first-run gate with iOS's saved-password sheet laid over it ("password for proxmox…"). Two causes, both fixed. (1) The key fields were diff --git a/app/api/podcast/route.ts b/app/api/podcast/route.ts new file mode 100644 index 0000000..6d6d071 --- /dev/null +++ b/app/api/podcast/route.ts @@ -0,0 +1,234 @@ +import { z } from "zod"; + +import { extractNotes, generatePodcast, notesToText, type PipelineStage } from "@/lib/ai"; +import { getServerConfig, resolveModelName } from "@/lib/config"; +import { INERT_OLLAMA_AUTH, resolveModelTarget } from "@/lib/providers-server"; +import { friendlyErrorMessage, OllamaError } from "@/lib/ollama"; +import { rejectCrossOrigin, rejectOversizedBody, requestLocale } from "@/lib/http-guard"; +import { clientKey, rateLimit, rateLimitResponse } from "@/lib/rate-limit"; +import { acquireModelGate } from "@/lib/model-guard"; +import { translate } from "@/lib/i18n-dict"; +import { dataUrlSchema } from "@/lib/schemas"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; +export const maxDuration = 300; + +/** ~8 MB per image after client-side resizing; up to 16 images (photos + PDF pages). */ +const MAX_IMAGE_CHARS = 8_000_000; + +const requestSchema = z.object({ + textModel: z.string().min(1).max(200).optional(), + visionModel: z.string().min(1).max(200).optional(), + images: z.array(dataUrlSchema(MAX_IMAGE_CHARS)).max(16).optional(), + text: z.string().max(120_000).optional(), + /** Target episode length in minutes. */ + minutes: z.number().int().min(1).max(15).optional(), + focus: z.string().max(300).optional(), + /** Language the episode must be written in (usually detected by the vision step). */ + language: z.string().max(60).optional(), +}); + +type StreamEvent = + | { type: "stage"; stage: PipelineStage; message: string } + | { type: "podcast"; podcast: unknown; meta: Record } + | { type: "error"; error: string; kind?: string } + | { type: "ping" }; + +function encoder() { + const enc = new TextEncoder(); + return (event: StreamEvent) => enc.encode(`${JSON.stringify(event)}\n`); +} + +/** Generous but finite: ~16 compressed images plus JSON overhead. */ +const MAX_BODY_BYTES = 33554432; + +/** + * Notes → a spoken episode. Same plumbing as /api/generate (rate limit, model + * gate, provider keys, heartbeats, one hard deadline), one model call for the + * script after the optional vision pass. + */ +export async function POST(request: Request) { + const locale = requestLocale(request); + const crossOrigin = rejectCrossOrigin(request); + if (crossOrigin) return crossOrigin; + const oversized = rejectOversizedBody(request, MAX_BODY_BYTES); + if (oversized) return oversized; + + const config = getServerConfig(); + + const limit = rateLimit(`podcast:${clientKey(request)}`, config.rateLimits.generate, config.rateLimits.windowMs); + if (!limit.allowed) return rateLimitResponse(limit, locale); + + const resolved = resolveModelTarget(config, request, locale); + if ("error" in resolved) { + return Response.json({ error: resolved.error.message, kind: resolved.error.kind }, { status: resolved.error.status }); + } + const target = resolved.target; + const provider = target.kind === "provider" ? target.target : undefined; + const baseAuth = target.kind === "provider" ? INERT_OLLAMA_AUTH : target.auth; + + let payload: unknown; + try { + payload = await request.json(); + } catch { + return Response.json({ error: translate(locale, "server.bodyNotJson") }, { status: 400 }); + } + + const parsed = requestSchema.safeParse(payload); + if (!parsed.success) { + return Response.json( + { + error: translate(locale, "server.invalidRequest", { + detail: parsed.error.issues + .slice(0, 3) + .map((issue) => `${issue.path.join(".") || "body"} ${issue.message}`) + .join("; "), + }), + }, + { status: 400 }, + ); + } + + const { images, text, minutes, focus, language } = parsed.data; + const imageList = (images ?? []).map(stripDataUrlPrefix).filter(Boolean); + + if (imageList.length === 0 && !text?.trim()) { + return Response.json({ error: translate(locale, "server.needContent") }, { status: 400 }); + } + + const providerTextModel = provider ? parsed.data.textModel ?? parsed.data.visionModel : undefined; + const providerVisionModel = provider ? parsed.data.visionModel ?? parsed.data.textModel : undefined; + if (provider && !providerTextModel) { + return Response.json( + { error: translate(locale, "server.providerModelNeeded"), kind: "bad_provider" }, + { status: 400 }, + ); + } + const textModel = provider ? providerTextModel! : config.textModel; + const visionModel = provider ? providerVisionModel ?? providerTextModel! : config.visionModel; + + const gate = acquireModelGate(request, config, locale); + if ("response" in gate) return gate.response; + + const encode = encoder(); + const abort = new AbortController(); + /* One vision pass plus one script: the same 2x deadline as the deck flow. */ + const globalTimer = setTimeout(() => abort.abort(), config.timeoutMs * 2); + const onRequestAbort = () => abort.abort(); + if (request.signal.aborted) abort.abort(); + else request.signal.addEventListener("abort", onRequestAbort, { once: true }); + let heartbeat: ReturnType | null = null; + + const stream = new ReadableStream({ + async start(controller) { + const send = (event: StreamEvent) => { + try { + controller.enqueue(encode(event)); + } catch { + /* client went away */ + } + }; + + heartbeat = setInterval(() => send({ type: "ping" }), 15_000); + try { + let sourceText: string; + let detectedLanguage: string | undefined = language; + let title: string | undefined; + let summary: string | undefined; + let repaired = false; + + if (imageList.length > 0) { + send({ type: "stage", stage: "vision", message: "Reading your notes…" }); + const extraction = await extractNotes({ + model: visionModel, + repairModel: textModel, + auth: baseAuth, + provider, + images: imageList, + text, + signal: abort.signal, + onProgress: (message) => send({ type: "stage", stage: "vision", message }), + }); + sourceText = notesToText(extraction.notes); + title = extraction.notes.title; + summary = extraction.notes.summary; + detectedLanguage = extraction.notes.language || language; + repaired = repaired || extraction.repaired || extraction.languageRepaired; + send({ type: "stage", stage: "generation", message: "Writing the script…" }); + } else { + sourceText = text!.trim(); + send({ type: "stage", stage: "generation", message: "Writing the script…" }); + } + + const generated = await generatePodcast({ + model: textModel, + auth: baseAuth, + provider, + notes: sourceText, + minutes: minutes ?? 4, + language: detectedLanguage, + focus, + signal: abort.signal, + }); + + send({ + type: "podcast", + podcast: { + title: generated.podcast.title || title || "Your podcast", + language: generated.podcast.language || detectedLanguage, + summary: generated.podcast.summary ?? summary, + sourceText, + sections: generated.podcast.sections, + }, + meta: { + repaired, + language: generated.podcast.language || detectedLanguage || null, + visionModel: imageList.length ? (provider ? visionModel : resolveModelName(config.visionModel, baseAuth.directCloud)) : null, + textModel: provider ? textModel : resolveModelName(config.textModel, baseAuth.directCloud), + }, + }); + } catch (error) { + const message = friendlyErrorMessage(error, baseAuth.baseUrl || provider?.baseUrl || "", locale); + // Never log the request body: it may contain the user's notes. + console.error("[api/podcast]", error instanceof OllamaError ? `${error.kind}: ${error.message}` : error); + send({ + type: "error", + error: message, + kind: error instanceof OllamaError ? error.kind : "unknown", + }); + } finally { + if (heartbeat) clearInterval(heartbeat); + clearTimeout(globalTimer); + request.signal.removeEventListener("abort", onRequestAbort); + gate.gate.release(); + try { + controller.close(); + } catch { + /* already closed */ + } + } + }, + cancel() { + if (heartbeat) clearInterval(heartbeat); + clearTimeout(globalTimer); + request.signal.removeEventListener("abort", onRequestAbort); + gate.gate.release(); + abort.abort(); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "application/x-ndjson; charset=utf-8", + "Cache-Control": "no-store, no-transform", + Connection: "keep-alive", + }, + }); +} + +function stripDataUrlPrefix(value: string): string { + const commaIndex = value.indexOf(","); + if (value.startsWith("data:") && commaIndex !== -1) return value.slice(commaIndex + 1); + return value; +} diff --git a/app/globals.css b/app/globals.css index 8f8d188..d8c4088 100644 --- a/app/globals.css +++ b/app/globals.css @@ -1306,6 +1306,23 @@ body:has(.scene-pause) .ice-scene :is(.ice-orb, .ice-bubble) { .chip { min-height: 44px; } } +/* Podcast transcript: the word under the voice rides a highlight, the way + lyrics do in Spotify. Words are plain spans; the player flips one class. */ +.podcast-word { + border-radius: 0.4rem; + transition: background-color 140ms ease, color 140ms ease; + cursor: pointer; +} +.podcast-word-active, +.podcast-word:hover { + background: var(--nav-active-bg); + color: var(--text); + box-shadow: 0 0 0 2px var(--nav-active-bg); +} +.podcast-word-active { + font-weight: 600; +} + /* The AI note under a surface that shows model output -------------------- */ .ai-disclaimer { font-size: 12px; diff --git a/app/page.tsx b/app/page.tsx index 1b3eeaa..3086924 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -19,7 +19,9 @@ export default async function HomePage({
- +
diff --git a/app/podcast/[id]/page.tsx b/app/podcast/[id]/page.tsx new file mode 100644 index 0000000..47cdad3 --- /dev/null +++ b/app/podcast/[id]/page.tsx @@ -0,0 +1,72 @@ +"use client"; + +import Link from "next/link"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect, useState } from "react"; + +import { ApiKeyGate } from "@/components/ApiKeyGate"; +import { PodcastPlayer } from "@/components/podcast/PodcastPlayer"; +import { Button } from "@/components/ui"; +import { useI18n } from "@/lib/i18n"; +import { getPodcast } from "@/lib/podcasts"; +import type { Podcast } from "@/lib/schemas"; + +export default function PodcastEpisodePage() { + const params = useParams<{ id: string }>(); + const router = useRouter(); + const id = typeof params.id === "string" ? params.id : ""; + const { t } = useI18n(); + const [episode, setEpisode] = useState(null); + const [loaded, setLoaded] = useState(false); + + useEffect(() => { + if (!id) { + setLoaded(true); + return; + } + setEpisode(getPodcast(id) ?? null); + setLoaded(true); + }, [id]); + + if (!loaded) { + return ( +
+
+
+
+
+
+
+ ); + } + + if (!episode) { + return ( +
+
+

+ 🎙️ +

+

{t("podcast.notFound")}

+

{t("podcast.notFoundBody")}

+
+ + {t("review.done")} + + +
+
+
+ ); + } + + return ( + +
+ +
+ + ); +} diff --git a/app/podcast/page.tsx b/app/podcast/page.tsx new file mode 100644 index 0000000..91c3b69 --- /dev/null +++ b/app/podcast/page.tsx @@ -0,0 +1,34 @@ +import { ApiKeyGate } from "@/components/ApiKeyGate"; +import { AppHeader } from "@/components/AppHeader"; +import { RouteTransition } from "@/components/RouteTransition"; +import { Sidebar, SidebarOffset } from "@/components/Sidebar"; +import { PodcastWizard } from "@/components/podcast/PodcastWizard"; +import { WizardModeSwitch } from "@/components/WizardModeSwitch"; + +export const metadata = { + title: "New podcast", +}; + +/** + * Podcast mode's intake: the same notes as a deck, but the output is a spoken + * episode with a synced transcript (see components/podcast). + */ +export default function PodcastPage() { + return ( + +
+ + +
+ +
+ + + + +
+
+
+ + ); +} diff --git a/components/AeroMusic.tsx b/components/AeroMusic.tsx index 1d55826..09549a2 100644 --- a/components/AeroMusic.tsx +++ b/components/AeroMusic.tsx @@ -187,6 +187,17 @@ export function AeroMusic() { }; }, [show, autoStart, playing, startFromGesture]); + /* Something with a voice started (the podcast player): the soundtrack + steps aside rather than talking over it. */ + useEffect(() => { + const onForeground = () => { + widget.current?.pause(); + setAutoStart(false); + }; + window.addEventListener("freezmo:foreground-audio", onForeground); + return () => window.removeEventListener("freezmo:foreground-audio", onForeground); + }, []); + if (!show) return null; const title = track?.title ?? FALLBACK_TITLE; diff --git a/components/DataSection.tsx b/components/DataSection.tsx index c32b94e..9301dfc 100644 --- a/components/DataSection.tsx +++ b/components/DataSection.tsx @@ -95,6 +95,7 @@ export function DataSection() { homework: pending.preview.homework.added + pending.preview.homework.replaced, folders: pending.preview.folders.added + pending.preview.folders.replaced, chats: pending.preview.chats.added + pending.preview.chats.replaced, + podcasts: pending.preview.podcasts.added + pending.preview.podcasts.replaced, })}

{t("settings.importMergeNote")}

diff --git a/components/LibraryScreen.tsx b/components/LibraryScreen.tsx index 5dae38e..0234cc5 100644 --- a/components/LibraryScreen.tsx +++ b/components/LibraryScreen.tsx @@ -8,6 +8,7 @@ import { DeckCard } from "@/components/DeckCard"; import { useAppReducedMotion as useReducedMotion } from "@/lib/motion-pref"; import { FolderBar } from "@/components/FolderBar"; import { HomeworkCard } from "@/components/HomeworkCard"; +import { PodcastList } from "@/components/podcast/PodcastList"; import { ChatIcon, EllipsisIcon, FolderIcon, UploadIcon } from "@/components/icons"; import { LinkLoadingScreen } from "@/components/LinkLoadingScreen"; import { SegmentedPill } from "@/components/SegmentedPill"; @@ -33,7 +34,7 @@ import { cn } from "@/lib/utils"; * saved homework (solutions and step-by-step guides). The tab bar picks one, * so the two never mix in a single list. */ -export function LibraryScreen({ initialTab = "decks" }: { initialTab?: "decks" | "homework" }) { +export function LibraryScreen({ initialTab = "decks" }: { initialTab?: "decks" | "homework" | "podcasts" }) { const reduceMotion = useReducedMotion(); const tab = initialTab; const decks = useSyncExternalStore(deckStore.subscribe, deckStore.getSnapshot, deckStore.getServerSnapshot); @@ -269,6 +270,14 @@ export function LibraryScreen({ initialTab = "decks" }: { initialTab?: "decks" | {isPhone && tab === "homework" ? : null} {t("tabHomework")} + + {isPhone && tab === "podcasts" ? : null} + {t("tabPodcasts")} + ) : null} + {tab === "podcasts" ? : null} + diff --git a/components/WizardModeSwitch.tsx b/components/WizardModeSwitch.tsx index b9362ce..6f79c1e 100644 --- a/components/WizardModeSwitch.tsx +++ b/components/WizardModeSwitch.tsx @@ -9,12 +9,13 @@ import { useWizardBusy } from "@/lib/wizard-busy"; import { cn } from "@/lib/utils"; /** - * In-page switch between the two wizards (New deck ⇄ Make my homework). - * Phones reach both through this control and the centre tab; desktop keeps - * the rail rows as well. Hidden while either wizard is generating: leaving a - * loading run through it would abandon the request mid-flight. + * In-page switch between the three note-powered wizards (New deck ⇄ Make my + * homework ⇄ New podcast). Phones reach the first two through this control and + * the centre tab; desktop keeps the rail rows as well. Hidden while a wizard + * is generating: leaving a loading run through it would abandon the request + * mid-flight. */ -export function WizardModeSwitch({ mode }: { mode: "deck" | "homework" }) { +export function WizardModeSwitch({ mode }: { mode: "deck" | "homework" | "podcast" }) { const { t } = useI18n(); const isPhone = useIsPhone(); const busy = useWizardBusy(); @@ -39,6 +40,14 @@ export function WizardModeSwitch({ mode }: { mode: "deck" | "homework" }) { {isPhone && mode === "homework" ? : null} {t("wizard.mode.homework")} + + {isPhone && mode === "podcast" ? : null} + {t("wizard.mode.podcast")} + ); } diff --git a/components/podcast/PodcastList.tsx b/components/podcast/PodcastList.tsx new file mode 100644 index 0000000..d474e95 --- /dev/null +++ b/components/podcast/PodcastList.tsx @@ -0,0 +1,179 @@ +"use client"; + +import Link from "next/link"; +import { AnimatePresence, motion } from "framer-motion"; +import { useMemo, useState, useSyncExternalStore } from "react"; + +import { useI18n } from "@/lib/i18n"; +import { useAppReducedMotion as useReducedMotion } from "@/lib/motion-pref"; +import { deletePodcast, podcastMinutes, podcastStore, renamePodcast, sortPodcasts, upsertPodcast } from "@/lib/podcasts"; +import { showToast } from "@/lib/toast"; +import { EmptyState } from "@/components/ui"; +import { ClockIcon, EllipsisIcon, MusicIcon, PlayIcon } from "@/components/icons"; +import { cn } from "@/lib/utils"; + +/** + * The library's podcasts tab: short episode cards with listen / rename / + * delete, the same interaction language as the homework cards. + */ +export function PodcastList() { + const { t, plural } = useI18n(); + const reduceMotion = useReducedMotion(); + const episodes = useSyncExternalStore( + podcastStore.subscribe, + podcastStore.getSnapshot, + podcastStore.getServerSnapshot, + ); + const sorted = useMemo(() => sortPodcasts(episodes), [episodes]); + const [menuFor, setMenuFor] = useState(null); + const [renaming, setRenaming] = useState(null); + const [draft, setDraft] = useState(""); + + if (sorted.length === 0) { + return ( + + {t("podcast.library.new")} + + } + > + {t("podcast.library.emptyBody")} + + ); + } + + return ( +
    + + {sorted.map((episode, index) => { + const minutes = podcastMinutes(episode); + const progress = episode.progress; + return ( + +
    + + + +
    + {renaming === episode.id ? ( + setDraft(event.target.value)} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") { + const title = draft.trim(); + if (title) { + renamePodcast(episode.id, title); + showToast({ message: t("podcast.card.renamed") }); + } + setRenaming(null); + } + if (event.key === "Escape") setRenaming(null); + }} + maxLength={200} + aria-label={t("podcast.card.rename")} + className="w-full rounded-lg border border-[var(--hairline)] bg-[var(--surface-strong)] px-2.5 py-1.5 text-[15px] font-semibold text-[var(--text)] outline-none focus:border-accent-400" + /> + ) : ( +

    + {episode.title} +

    + )} +

    + + + {t("podcast.card.minutes", { n: minutes })} + + · + {t("podcast.card.sections", { n: episode.sections.length })} + {progress ? ( + <> + · + {t("podcast.sectionOf", { n: progress.section + 1, total: episode.sections.length })} + + ) : null} +

    +
    + +
    + + {episode.summary ? ( +

    {episode.summary}

    + ) : null} + + {menuFor === episode.id ? ( +
    + + +
    + ) : null} + +
    + + + {t("podcast.card.open")} + + + {plural(episode.sections.length, "podcast.sectionCount")} + +
    +     + ); + })} +     + + {/* Tapping anywhere else closes an open menu. */} + {menuFor ? ( +
  • setMenuFor(null)} /> + ) : null} +
+ ); +} diff --git a/components/podcast/PodcastPlayer.tsx b/components/podcast/PodcastPlayer.tsx new file mode 100644 index 0000000..17841ef --- /dev/null +++ b/components/podcast/PodcastPlayer.tsx @@ -0,0 +1,376 @@ +"use client"; + +import Link from "next/link"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import { useI18n } from "@/lib/i18n"; +import { createPodcastSpeech, tokenizeSections, type PodcastVoice } from "@/lib/podcast-speech"; +import { savePodcastProgress } from "@/lib/podcasts"; +import { speechAvailable, speechLocale, stopSpeaking } from "@/lib/speech"; +import type { Podcast } from "@/lib/schemas"; +import { useAppReducedMotion as useReducedMotion } from "@/lib/motion-pref"; +import { ChevronLeftIcon, ClockIcon, PauseIcon, PlayIcon, RefreshIcon } from "@/components/icons"; +import { cn } from "@/lib/utils"; + +/** Playback speeds, cycled by one button. */ +const RATES = [1, 1.15, 1.3, 0.9] as const; + +/** + * Reads an episode aloud and highlights the word being spoken — the transcript + * is the player, Spotify-lyrics style: past words stay dark, upcoming words + * are dimmed, the word under the voice rides a rounded highlight that keeps + * itself centred as the episode plays. + * + * The heavy lifting (boundary events, the timer fallback, seeking by + * re-speaking) lives in lib/podcast-speech; this component owns the layout, + * the controls and the saved position. + */ +export function PodcastPlayer({ episode }: { episode: Podcast }) { + const { t, plural, locale } = useI18n(); + const reduceMotion = useReducedMotion(); + const [rate, setRate] = useState(1); + const [playing, setPlaying] = useState(false); + const [position, setPosition] = useState(null); + const [unsupported, setUnsupported] = useState(false); + const transcript = useRef(null); + const activeWord = useRef(null); + /** Latest position for the unmount/pause save, without re-creating callbacks. */ + const latest = useRef(episode.progress ?? null); + + const tokens = useMemo(() => tokenizeSections(episode.sections), [episode.sections]); + const supported = useMemo(() => speechAvailable(), []); + + const speech = useMemo(() => { + if (!supported) return null; + return createPodcastSpeech({ + sections: episode.sections, + locale: speechLocale(episode.language, locale), + rate, + onWord: (next) => { + latest.current = next; + setPosition(next); + }, + onPlay: () => { + setPlaying(true); + /* The Aero soundtrack steps aside for the voice (see AeroMusic). */ + window.dispatchEvent(new CustomEvent("freezmo:foreground-audio", { detail: "podcast" })); + }, + onPause: () => setPlaying(false), + onEnd: () => { + setPlaying(false); + setPosition(null); + latest.current = null; + savePodcastProgress(episode.id, { section: 0, word: 0 }); + }, + onError: (message) => { + if (message === "unsupported") setUnsupported(true); + setPlaying(false); + }, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps -- rebuilt only when the episode, the speed or the locale changes + }, [episode.id, episode.sections, rate, supported, locale]); + + /* Resume where the listener left off (a reopened episode starts there). */ + useEffect(() => { + if (episode.progress) { + latest.current = { section: episode.progress.section, word: episode.progress.word }; + setPosition({ section: episode.progress.section, word: episode.progress.word }); + } + }, [episode.id, episode.progress]); + + const persist = useCallback(() => { + if (latest.current) savePodcastProgress(episode.id, latest.current); + }, [episode.id]); + + /* The engine instance is rebuilt on a speed change; keep a ref so the + unmount cleanup always stops whichever one is current. */ + const speechRef = useRef(speech); + const playingRef = useRef(false); + useEffect(() => { + speechRef.current = speech; + }, [speech]); + useEffect(() => { + playingRef.current = playing; + }, [playing]); + useEffect(() => { + return () => { + persist(); + speechRef.current?.stop(); + }; + }, [persist]); + + /* Closing the tab, switching apps or hard-navigating away: the unmount + cleanup never runs there, so the position also rides the page lifecycle. */ + useEffect(() => { + const save = () => persist(); + const onHide = () => { + if (document.visibilityState === "hidden") save(); + }; + window.addEventListener("pagehide", save); + document.addEventListener("visibilitychange", onHide); + return () => { + window.removeEventListener("pagehide", save); + document.removeEventListener("visibilitychange", onHide); + }; + }, [persist]); + + /* A rebuilt engine (speed change) resumes the same word when it was + playing — the listener should not notice the swap. */ + const firstEngine = useRef(true); + useEffect(() => { + if (firstEngine.current) { + firstEngine.current = false; + return; + } + if (!playingRef.current || !speech) return; + speech.play(latest.current ?? { section: 0, word: 0 }); + }, [speech]); + + /* The highlight: one class on the active word (imperative, so a few hundred + spans do not re-render per word), then keep it in the comfortable band. */ + useEffect(() => { + const container = transcript.current; + if (!container) return; + activeWord.current?.removeAttribute("data-word-active"); + activeWord.current?.classList.remove("podcast-word-active"); + if (!position) { + activeWord.current = null; + return; + } + const index = tokens.findIndex((token) => token.section === position.section && token.index === position.word); + if (index === -1) return; + const element = container.querySelector(`[data-word="${index}"]`); + if (!element) return; + element.setAttribute("data-word-active", "1"); + element.classList.add("podcast-word-active"); + activeWord.current = element; + + const box = element.getBoundingClientRect(); + const view = container.getBoundingClientRect(); + const top = box.top - view.top; + const comfortable = view.height * 0.38; + if (top < view.height * 0.18 || top > view.height * 0.62) { + container.scrollTo({ + top: container.scrollTop + top - comfortable, + behavior: reduceMotion ? "auto" : "smooth", + }); + } + }, [position, tokens, reduceMotion]); + + const sections = episode.sections; + const words = tokens.length; + const done = useMemo(() => { + if (!position) return 0; + return tokens.findIndex((token) => token.section === position.section && token.index === position.word); + }, [position, tokens]); + + const toggle = () => { + if (!speech) return; + if (playing) { + speech.pause(); + persist(); + return; + } + /* Other readers (a card's read-aloud) yield to the episode. */ + stopSpeaking(); + speech.play(position ?? { section: 0, word: 0 }); + }; + + const jump = (section: number) => { + if (!speech) return; + const next = { section, word: 0 }; + latest.current = next; + setPosition(next); + stopSpeaking(); + speech.play(next); + }; + + const jumpWord = (index: number) => { + if (!speech) return; + const token = tokens[index]; + if (!token) return; + const next = { section: token.section, word: token.index }; + latest.current = next; + setPosition(next); + stopSpeaking(); + speech.play(next); + }; + + /* Word spans, built once per section: the highlight is a class flip. */ + const rendered = useMemo(() => { + return sections.map((section, sectionIndex) => { + const sectionTokens = tokens.filter((token) => token.section === sectionIndex); + const nodes: React.ReactNode[] = []; + let cursor = 0; + sectionTokens.forEach((token) => { + if (token.start > cursor) nodes.push(section.text.slice(cursor, token.start)); + const globalIndex = token.global; + nodes.push( + + {section.text.slice(token.start, token.end)} + , + ); + cursor = token.end; + }); + if (cursor < section.text.length) nodes.push(section.text.slice(cursor)); + return { section, sectionIndex, nodes }; + }); + }, [sections, tokens]); + + const currentSection = position?.section ?? 0; + + return ( +
+
+
+ + + +
+

{episode.title}

+

+ {t("podcast.sectionOf", { n: Math.min(currentSection + 1, sections.length), total: sections.length })} + {" · "} + {sections[currentSection]?.heading ?? ""} +

+
+ +
+
+ +
{ + /* Word taps seek: the transcript is the scrubber, like lyrics. */ + const target = event.target as HTMLElement; + if (!target.classList.contains("podcast-word")) return; + const index = Number(target.dataset.word); + if (Number.isInteger(index)) jumpWord(index); + }} + className="no-scrollbar min-h-0 flex-1 overflow-y-auto overscroll-contain px-4 pb-40 pt-24" + > +
+ {!supported || unsupported ? ( +

+ {t("podcast.noVoice")} +

+ ) : null} + +
+

+ {episode.title} +

+ {episode.summary ? ( +

{episode.summary}

+ ) : null} +

+ + {plural(episode.sections.length, "podcast.sectionCount")} · {plural(words, "podcast.wordCount")} +

+
+ + {/* The script reads on a glass pane: in Aero the wallpaper behind + plain paragraphs is bright grass and sun (the contrast scan caught + it), and the same treatment keeps every theme consistent. */} +
+ {rendered.map(({ section, sectionIndex, nodes }) => ( +
+

+ + {sectionIndex + 1} + + {section.heading} +

+

+ {nodes} +

+
+ ))} +
+ +

{t("podcast.tapToJump")}

+
+
+ + {/* Controls: chapter list on the left, transport in the middle. */} +
+
+
+
+
+
+
+ +
+ {sections.map((section, index) => ( + + ))} +
+ +
+
+
+
+
+ ); +} diff --git a/components/podcast/PodcastWizard.tsx b/components/podcast/PodcastWizard.tsx new file mode 100644 index 0000000..4d6604a --- /dev/null +++ b/components/podcast/PodcastWizard.tsx @@ -0,0 +1,505 @@ +"use client"; + +import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import { AiDisclaimer } from "@/components/AiDisclaimer"; +import { ApiError, generatePodcast, type GenerateProgress } from "@/lib/client-api"; +import { + DOCUMENT_ACCEPT_ATTR, + KIND_LABELS, + describeDocumentError, + documentKindOf, + extractDocument, + isDocumentFile, + type DocumentKind, + type ExtractedDocument, +} from "@/lib/documents"; +import { + ACCEPTED_TYPES, + MAX_IMAGES, + describeFileError, + imageFilesFromDataTransfer, + isImageFile, + prepareImage, + type PreparedImage, +} from "@/lib/image"; +import { useI18n } from "@/lib/i18n"; +import { ModelErrorActions } from "@/components/ModelErrorActions"; +import { takePendingShare } from "@/lib/share-pending"; +import { upsertPodcast } from "@/lib/podcasts"; +import { useIsPhone } from "@/lib/use-phone"; +import { useOnline } from "@/lib/use-online"; +import { uid } from "@/lib/utils"; +import { wizardBusy } from "@/lib/wizard-busy"; +import { CameraIcon, ClockIcon, MusicIcon, PlusIcon, SparklesIcon, XIcon } from "@/components/icons"; +import { cn } from "@/lib/utils"; +import { Button } from "@/components/ui"; +import { LoadingState, type LoadingStage } from "@/components/LoadingState"; +import { WizardActionBar } from "@/components/WizardActionBar"; + +type Phase = "idle" | "loading" | "error"; + +type DocItem = { + id: string; + name: string; + kind: DocumentKind; + status: "extracting" | "ready" | "error"; + progress?: string; + extracted?: ExtractedDocument; + error?: string; +}; + +const MAX_DOCUMENTS = 6; +const MAX_IMAGES_TO_SEND = 12; +const MAX_TEXT_TO_SEND = 100_000; + +/** Episode lengths, in minutes; the prompt turns these into a word target. */ +const LENGTH_OPTIONS = [2, 4, 7] as const; + +const PODCAST_STAGES: LoadingStage[] = [ + { id: "vision", labelKey: "podcast.loading.read1", messageKeys: ["podcast.loading.read1", "podcast.loading.read2"] }, + { id: "generation", labelKey: "podcast.loading.write1", messageKeys: ["podcast.loading.write1", "podcast.loading.write2", "podcast.loading.write3"] }, +]; + +const LOADING_TIPS = ["podcast.tip1", "podcast.tip2"]; + +/** + * Podcast mode: the same notes intake as the deck wizard, but the output is a + * spoken episode. One screen — the choices are only "how long" and an optional + * topic hint — so it does not need the deck wizard's three steps. + */ +export function PodcastWizard() { + const router = useRouter(); + const { t, plural } = useI18n(); + const isPhone = useIsPhone(); + const online = useOnline(); + + const [images, setImages] = useState([]); + const [documents, setDocuments] = useState([]); + const [text, setText] = useState(""); + const [focus, setFocus] = useState(""); + const [minutes, setMinutes] = useState(4); + const [phase, setPhase] = useState("idle"); + const [error, setError] = useState<{ message: string; kind?: string } | null>(null); + const [fileError, setFileError] = useState(null); + const [dragging, setDragging] = useState(false); + const [progress, setProgress] = useState({ stage: "generation", message: "" }); + + const fileInput = useRef(null); + const docInput = useRef(null); + const cameraInput = useRef(null); + const abortRef = useRef(null); + + const documentText = useMemo( + () => + documents + .filter((entry) => entry.status === "ready" && entry.extracted) + .map((entry) => entry.extracted?.text ?? "") + .filter(Boolean) + .join("\n\n"), + [documents], + ); + + const imagesToSend = useMemo( + () => + [ + ...images.map((image) => image.dataUrl), + ...documents.flatMap((entry) => (entry.extracted?.images ?? []).map((image) => image.dataUrl)), + ].slice(0, MAX_IMAGES_TO_SEND), + [images, documents], + ); + + const mergedText = [text.trim(), documentText.trim()].filter(Boolean).join("\n\n").slice(0, MAX_TEXT_TO_SEND); + const canGenerate = online && phase !== "loading" && (imagesToSend.length > 0 || mergedText.length > 0); + const extracting = documents.some((entry) => entry.status === "extracting"); + + useEffect(() => { + wizardBusy.set(phase === "loading"); + return () => wizardBusy.set(false); + }, [phase]); + + const addImages = useCallback( + async (files: File[]) => { + setFileError(null); + const room = MAX_IMAGES - images.length; + if (room <= 0) { + setFileError(t("upload.tooManyPhotos", { n: MAX_IMAGES })); + return; + } + const accepted = files.slice(0, room); + for (const file of accepted) { + const failure = describeFileError(file); + if (failure) { + setFileError(failure); + continue; + } + try { + const prepared = await prepareImage(file); + setImages((current) => (current.length >= MAX_IMAGES ? current : [...current, prepared])); + } catch (error) { + setFileError(error instanceof Error ? error.message : t("upload.couldNotReadFile")); + } + } + if (files.length > room) setFileError(t("upload.tooManyPhotos", { n: MAX_IMAGES })); + }, + [images.length, t], + ); + + const addDocuments = useCallback( + async (files: File[]) => { + setFileError(null); + for (const file of files) { + const kind = documentKindOf(file); + if (!kind) { + setFileError(describeDocumentError(file)); + continue; + } + if (documents.length >= MAX_DOCUMENTS) { + setFileError(t("upload.tooManyDocs", { n: MAX_DOCUMENTS })); + break; + } + const id = uid("doc"); + setDocuments((current) => [ + ...current, + { id, name: file.name, kind, status: "extracting", progress: t("upload.readingDocs") }, + ]); + try { + const extracted = await extractDocument(file, (update) => + setDocuments((current) => + current.map((entry) => (entry.id === id ? { ...entry, progress: update.message } : entry)), + ), + ); + setDocuments((current) => + current.map((entry) => (entry.id === id ? { ...entry, status: "ready", extracted } : entry)), + ); + } catch (error) { + setDocuments((current) => + current.map((entry) => + entry.id === id + ? { + ...entry, + status: "error", + error: error instanceof Error ? error.message : t("upload.couldNotReadDoc"), + } + : entry, + ), + ); + } + } + }, + [documents.length, t], + ); + + /* A share into the app (web share target) lands here as pending notes. */ + useEffect(() => { + const pending = takePendingShare(); + if (!pending) return; + if (pending.text) setText((current) => (current ? `${current}\n\n${pending.text}` : pending.text)); + if (pending.files.length > 0) void addImages(pending.files.filter(isImageFile)); + }, [addImages]); + + const generate = async () => { + if (!canGenerate) return; + setError(null); + setPhase("loading"); + setProgress({ + stage: imagesToSend.length ? "vision" : "generation", + message: imagesToSend.length ? t("podcast.loading.read1") : t("podcast.loading.write1"), + }); + + const controller = new AbortController(); + abortRef.current = controller; + + try { + const { podcast } = await generatePodcast( + { + images: imagesToSend.length ? imagesToSend : undefined, + text: mergedText || undefined, + minutes, + focus: focus.trim() || undefined, + }, + { onProgress: setProgress, signal: controller.signal }, + ); + + const now = Date.now(); + const stored = { + ...podcast, + id: uid("pod"), + createdAt: now, + updatedAt: now, + }; + upsertPodcast(stored); + router.push(`/podcast/${stored.id}`); + } catch (generateError) { + setError({ + message: generateError instanceof ApiError ? generateError.message : t("upload.genericError"), + kind: generateError instanceof ApiError ? generateError.kind : undefined, + }); + setPhase("error"); + } finally { + abortRef.current = null; + } + }; + + if (phase === "loading") { + return ( + { + abortRef.current?.abort(); + setPhase("idle"); + }} + /> + ); + } + + const notesReady = imagesToSend.length > 0 || mergedText.length > 0; + + const dropzone = ( + + ); + + return ( +
+
+

+ + {t("podcast.title")} +

+

{t("podcast.subtitle")}

+
+ +
+ {dropzone} +
+ + + {isPhone ? ( + + ) : null} +
+ + {images.length > 0 || documents.length > 0 ? ( +
    + {images.map((image) => ( +
  • + {/* eslint-disable-next-line @next/next/no-img-element */} + + +
    • + ))} + {documents.map((entry) => ( +
  • + + {KIND_LABELS[entry.kind]} + + + {entry.name} + + + {entry.status === "extracting" ? entry.progress ?? t("upload.readingDocs") : null} + {entry.status === "error" ? entry.error : null} + + +
    • + ))} +
+ ) : null} + + {fileError ? ( +

+ {fileError} +

+ ) : null} +
+ +
+
+

+ {t("podcast.lengthAria")} +

+
+ {LENGTH_OPTIONS.map((option) => ( + + ))} +
+

{t("podcast.lengthHelp")}

+
+ +
+ + setFocus(event.target.value)} + placeholder={t("upload.topicPlaceholder")} + inputMode="text" + enterKeyHint="done" + autoComplete="off" + autoCapitalize="sentences" + spellCheck={false} + className="w-full rounded-2xl border border-[var(--hairline)] bg-[var(--surface-strong)] px-4 py-3 text-sm text-[var(--text)] outline-none transition placeholder:text-[var(--text-subtle)] focus:border-accent-400" + /> +
+ +
+ +