Podcast mode: notes read aloud with a lyric-style word highlight
The third segment of the note-powered wizards (Study deck / Homework / Podcast) takes the same intake - photos, documents, pasted text, the share target - and turns it into a spoken episode. - prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown, no visual references), chaptered into 3-8 sections around a length target. - /api/podcast mirrors /api/generate's plumbing (rate limit, model gate, provider keys, heartbeats, the 2x deadline, the optional vision pass). - storage: frezmo.podcasts.v1, zod-validated, included in the backup and the encrypted sync payload; a new library tab lists episodes with listen / rename / delete-with-undo, and the wizard switch deep-links in. - player: the transcript is word-wrapped spans and the word under the voice rides a lyric-style highlight that keeps itself centred. Two clocks: exact onboundary events where the browser reports them, a per-word length timer where it does not (iOS Safari); the first boundary retires the timer. Pause = cancel + re-speak from the word, which makes resume, chapter jumps and tap-any-word-to-seek the same operation; the position saves on pause, page-hide and unmount, so reopening resumes. Speed rebuilds the engine and keeps playing; the Aero soundtrack pauses itself when the voice starts; a browser with no voice gets an honest note and a readable transcript. - the script sits on a glass pane (plain paragraphs over Aero's grass failed contrast; the scan now covers the player too). tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and drives the highlight by hand: 26 checks. All suites green, contrast clean. Version 0.7.3, SW v95.
This commit is contained in:
parent
84a94581dc
commit
d95c5c7151
30 changed files with 2575 additions and 26 deletions
505
components/podcast/PodcastWizard.tsx
Normal file
505
components/podcast/PodcastWizard.tsx
Normal file
|
|
@ -0,0 +1,505 @@
|
|||
"use client";
|
||||
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { AiDisclaimer } from "@/components/AiDisclaimer";
|
||||
import { ApiError, generatePodcast, type GenerateProgress } from "@/lib/client-api";
|
||||
import {
|
||||
DOCUMENT_ACCEPT_ATTR,
|
||||
KIND_LABELS,
|
||||
describeDocumentError,
|
||||
documentKindOf,
|
||||
extractDocument,
|
||||
isDocumentFile,
|
||||
type DocumentKind,
|
||||
type ExtractedDocument,
|
||||
} from "@/lib/documents";
|
||||
import {
|
||||
ACCEPTED_TYPES,
|
||||
MAX_IMAGES,
|
||||
describeFileError,
|
||||
imageFilesFromDataTransfer,
|
||||
isImageFile,
|
||||
prepareImage,
|
||||
type PreparedImage,
|
||||
} from "@/lib/image";
|
||||
import { useI18n } from "@/lib/i18n";
|
||||
import { ModelErrorActions } from "@/components/ModelErrorActions";
|
||||
import { takePendingShare } from "@/lib/share-pending";
|
||||
import { upsertPodcast } from "@/lib/podcasts";
|
||||
import { useIsPhone } from "@/lib/use-phone";
|
||||
import { useOnline } from "@/lib/use-online";
|
||||
import { uid } from "@/lib/utils";
|
||||
import { wizardBusy } from "@/lib/wizard-busy";
|
||||
import { CameraIcon, ClockIcon, MusicIcon, PlusIcon, SparklesIcon, XIcon } from "@/components/icons";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { Button } from "@/components/ui";
|
||||
import { LoadingState, type LoadingStage } from "@/components/LoadingState";
|
||||
import { WizardActionBar } from "@/components/WizardActionBar";
|
||||
|
||||
type Phase = "idle" | "loading" | "error";
|
||||
|
||||
type DocItem = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: DocumentKind;
|
||||
status: "extracting" | "ready" | "error";
|
||||
progress?: string;
|
||||
extracted?: ExtractedDocument;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const MAX_DOCUMENTS = 6;
|
||||
const MAX_IMAGES_TO_SEND = 12;
|
||||
const MAX_TEXT_TO_SEND = 100_000;
|
||||
|
||||
/** Episode lengths, in minutes; the prompt turns these into a word target. */
|
||||
const LENGTH_OPTIONS = [2, 4, 7] as const;
|
||||
|
||||
const PODCAST_STAGES: LoadingStage[] = [
|
||||
{ id: "vision", labelKey: "podcast.loading.read1", messageKeys: ["podcast.loading.read1", "podcast.loading.read2"] },
|
||||
{ id: "generation", labelKey: "podcast.loading.write1", messageKeys: ["podcast.loading.write1", "podcast.loading.write2", "podcast.loading.write3"] },
|
||||
];
|
||||
|
||||
const LOADING_TIPS = ["podcast.tip1", "podcast.tip2"];
|
||||
|
||||
/**
|
||||
* Podcast mode: the same notes intake as the deck wizard, but the output is a
|
||||
* spoken episode. One screen — the choices are only "how long" and an optional
|
||||
* topic hint — so it does not need the deck wizard's three steps.
|
||||
*/
|
||||
export function PodcastWizard() {
|
||||
const router = useRouter();
|
||||
const { t, plural } = useI18n();
|
||||
const isPhone = useIsPhone();
|
||||
const online = useOnline();
|
||||
|
||||
const [images, setImages] = useState<PreparedImage[]>([]);
|
||||
const [documents, setDocuments] = useState<DocItem[]>([]);
|
||||
const [text, setText] = useState("");
|
||||
const [focus, setFocus] = useState("");
|
||||
const [minutes, setMinutes] = useState<number>(4);
|
||||
const [phase, setPhase] = useState<Phase>("idle");
|
||||
const [error, setError] = useState<{ message: string; kind?: string } | null>(null);
|
||||
const [fileError, setFileError] = useState<string | null>(null);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
const [progress, setProgress] = useState<GenerateProgress>({ stage: "generation", message: "" });
|
||||
|
||||
const fileInput = useRef<HTMLInputElement>(null);
|
||||
const docInput = useRef<HTMLInputElement>(null);
|
||||
const cameraInput = useRef<HTMLInputElement>(null);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
|
||||
const documentText = useMemo(
|
||||
() =>
|
||||
documents
|
||||
.filter((entry) => entry.status === "ready" && entry.extracted)
|
||||
.map((entry) => entry.extracted?.text ?? "")
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
[documents],
|
||||
);
|
||||
|
||||
const imagesToSend = useMemo(
|
||||
() =>
|
||||
[
|
||||
...images.map((image) => image.dataUrl),
|
||||
...documents.flatMap((entry) => (entry.extracted?.images ?? []).map((image) => image.dataUrl)),
|
||||
].slice(0, MAX_IMAGES_TO_SEND),
|
||||
[images, documents],
|
||||
);
|
||||
|
||||
const mergedText = [text.trim(), documentText.trim()].filter(Boolean).join("\n\n").slice(0, MAX_TEXT_TO_SEND);
|
||||
const canGenerate = online && phase !== "loading" && (imagesToSend.length > 0 || mergedText.length > 0);
|
||||
const extracting = documents.some((entry) => entry.status === "extracting");
|
||||
|
||||
useEffect(() => {
|
||||
wizardBusy.set(phase === "loading");
|
||||
return () => wizardBusy.set(false);
|
||||
}, [phase]);
|
||||
|
||||
const addImages = useCallback(
|
||||
async (files: File[]) => {
|
||||
setFileError(null);
|
||||
const room = MAX_IMAGES - images.length;
|
||||
if (room <= 0) {
|
||||
setFileError(t("upload.tooManyPhotos", { n: MAX_IMAGES }));
|
||||
return;
|
||||
}
|
||||
const accepted = files.slice(0, room);
|
||||
for (const file of accepted) {
|
||||
const failure = describeFileError(file);
|
||||
if (failure) {
|
||||
setFileError(failure);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const prepared = await prepareImage(file);
|
||||
setImages((current) => (current.length >= MAX_IMAGES ? current : [...current, prepared]));
|
||||
} catch (error) {
|
||||
setFileError(error instanceof Error ? error.message : t("upload.couldNotReadFile"));
|
||||
}
|
||||
}
|
||||
if (files.length > room) setFileError(t("upload.tooManyPhotos", { n: MAX_IMAGES }));
|
||||
},
|
||||
[images.length, t],
|
||||
);
|
||||
|
||||
const addDocuments = useCallback(
|
||||
async (files: File[]) => {
|
||||
setFileError(null);
|
||||
for (const file of files) {
|
||||
const kind = documentKindOf(file);
|
||||
if (!kind) {
|
||||
setFileError(describeDocumentError(file));
|
||||
continue;
|
||||
}
|
||||
if (documents.length >= MAX_DOCUMENTS) {
|
||||
setFileError(t("upload.tooManyDocs", { n: MAX_DOCUMENTS }));
|
||||
break;
|
||||
}
|
||||
const id = uid("doc");
|
||||
setDocuments((current) => [
|
||||
...current,
|
||||
{ id, name: file.name, kind, status: "extracting", progress: t("upload.readingDocs") },
|
||||
]);
|
||||
try {
|
||||
const extracted = await extractDocument(file, (update) =>
|
||||
setDocuments((current) =>
|
||||
current.map((entry) => (entry.id === id ? { ...entry, progress: update.message } : entry)),
|
||||
),
|
||||
);
|
||||
setDocuments((current) =>
|
||||
current.map((entry) => (entry.id === id ? { ...entry, status: "ready", extracted } : entry)),
|
||||
);
|
||||
} catch (error) {
|
||||
setDocuments((current) =>
|
||||
current.map((entry) =>
|
||||
entry.id === id
|
||||
? {
|
||||
...entry,
|
||||
status: "error",
|
||||
error: error instanceof Error ? error.message : t("upload.couldNotReadDoc"),
|
||||
}
|
||||
: entry,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
[documents.length, t],
|
||||
);
|
||||
|
||||
/* A share into the app (web share target) lands here as pending notes. */
|
||||
useEffect(() => {
|
||||
const pending = takePendingShare();
|
||||
if (!pending) return;
|
||||
if (pending.text) setText((current) => (current ? `${current}\n\n${pending.text}` : pending.text));
|
||||
if (pending.files.length > 0) void addImages(pending.files.filter(isImageFile));
|
||||
}, [addImages]);
|
||||
|
||||
const generate = async () => {
|
||||
if (!canGenerate) return;
|
||||
setError(null);
|
||||
setPhase("loading");
|
||||
setProgress({
|
||||
stage: imagesToSend.length ? "vision" : "generation",
|
||||
message: imagesToSend.length ? t("podcast.loading.read1") : t("podcast.loading.write1"),
|
||||
});
|
||||
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
try {
|
||||
const { podcast } = await generatePodcast(
|
||||
{
|
||||
images: imagesToSend.length ? imagesToSend : undefined,
|
||||
text: mergedText || undefined,
|
||||
minutes,
|
||||
focus: focus.trim() || undefined,
|
||||
},
|
||||
{ onProgress: setProgress, signal: controller.signal },
|
||||
);
|
||||
|
||||
const now = Date.now();
|
||||
const stored = {
|
||||
...podcast,
|
||||
id: uid("pod"),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
upsertPodcast(stored);
|
||||
router.push(`/podcast/${stored.id}`);
|
||||
} catch (generateError) {
|
||||
setError({
|
||||
message: generateError instanceof ApiError ? generateError.message : t("upload.genericError"),
|
||||
kind: generateError instanceof ApiError ? generateError.kind : undefined,
|
||||
});
|
||||
setPhase("error");
|
||||
} finally {
|
||||
abortRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
if (phase === "loading") {
|
||||
return (
|
||||
<LoadingState
|
||||
stages={PODCAST_STAGES}
|
||||
activeStage={progress.stage}
|
||||
message={progress.message}
|
||||
tipKeys={LOADING_TIPS}
|
||||
onCancel={() => {
|
||||
abortRef.current?.abort();
|
||||
setPhase("idle");
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
const notesReady = imagesToSend.length > 0 || mergedText.length > 0;
|
||||
|
||||
const dropzone = (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => (images.length >= MAX_IMAGES ? cameraInput.current?.click() : fileInput.current?.click())}
|
||||
onDragOver={(event) => {
|
||||
event.preventDefault();
|
||||
setDragging(true);
|
||||
}}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={(event) => {
|
||||
event.preventDefault();
|
||||
setDragging(false);
|
||||
void addImages(imageFilesFromDataTransfer(event.dataTransfer).filter(isImageFile));
|
||||
}}
|
||||
className={cn(
|
||||
"press flex min-h-32 w-full flex-col items-center justify-center gap-2 rounded-3xl border-2 border-dashed px-4 py-6 text-center transition",
|
||||
dragging ? "border-accent-400 bg-accent-400/10" : "border-[var(--glass-border)] bg-[var(--surface-strong)] hover:border-accent-400",
|
||||
)}
|
||||
>
|
||||
<span className="grid size-12 place-items-center rounded-2xl bg-[var(--surface-sunken)] text-accent-600 dark:text-accent-200">
|
||||
<CameraIcon className="size-6" />
|
||||
</span>
|
||||
<span className="text-sm font-semibold text-[var(--text)]">{t("podcast.addNotes")}</span>
|
||||
<span className="text-[12px] text-[var(--text-subtle)]">{t("upload.formats")}</span>
|
||||
</button>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="space-y-5 pb-28 lg:pb-0">
|
||||
<header className="card space-y-1.5 p-4">
|
||||
<h1 className="flex items-center gap-2 text-[17px] font-bold text-[var(--text)]">
|
||||
<MusicIcon className="size-5 text-accent-600 dark:text-accent-200" />
|
||||
{t("podcast.title")}
|
||||
</h1>
|
||||
<p className="text-[13px] leading-relaxed text-[var(--text-muted)]">{t("podcast.subtitle")}</p>
|
||||
</header>
|
||||
|
||||
<section className="card space-y-3 p-4">
|
||||
{dropzone}
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Button variant="secondary" size="sm" onClick={() => fileInput.current?.click()}>
|
||||
<PlusIcon className="size-4" /> {t("podcast.addPhotos")}
|
||||
</Button>
|
||||
<Button variant="secondary" size="sm" onClick={() => docInput.current?.click()}>
|
||||
<PlusIcon className="size-4" /> {t("podcast.addDocs")}
|
||||
</Button>
|
||||
{isPhone ? (
|
||||
<Button variant="secondary" size="sm" onClick={() => cameraInput.current?.click()}>
|
||||
<CameraIcon className="size-4" /> {t("upload.camera")}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{images.length > 0 || documents.length > 0 ? (
|
||||
<ul className="flex flex-wrap items-center gap-2">
|
||||
{images.map((image) => (
|
||||
<li key={image.id} className="relative">
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img src={image.dataUrl} alt="" className="size-14 rounded-xl border border-[var(--hairline)] object-cover" />
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setImages((current) => current.filter((entry) => entry.id !== image.id))}
|
||||
aria-label={t("upload.remove", { name: image.name ?? "" })}
|
||||
className="absolute -right-1.5 -top-1.5 grid size-6 place-items-center rounded-full border border-[var(--hairline)] bg-[var(--surface)] text-[var(--text-muted)]"
|
||||
>
|
||||
<XIcon className="size-3" />
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
{documents.map((entry) => (
|
||||
<li
|
||||
key={entry.id}
|
||||
className={cn(
|
||||
"inline-flex max-w-full items-center gap-1.5 rounded-full border px-2.5 py-1.5",
|
||||
entry.status === "error" ? "border-rose-300/60 bg-rose-500/10" : "border-[var(--hairline)] bg-[var(--surface-strong)]",
|
||||
)}
|
||||
>
|
||||
<span className="rounded-md bg-[var(--surface-sunken)] px-1.5 py-0.5 text-[10px] font-bold uppercase text-[var(--text-muted)]">
|
||||
{KIND_LABELS[entry.kind]}
|
||||
</span>
|
||||
<span className="max-w-32 truncate text-[12.5px] font-medium text-[var(--text)]" title={entry.name}>
|
||||
{entry.name}
|
||||
</span>
|
||||
<span className="text-[11.5px] text-[var(--text-subtle)]">
|
||||
{entry.status === "extracting" ? entry.progress ?? t("upload.readingDocs") : null}
|
||||
{entry.status === "error" ? entry.error : null}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setDocuments((current) => current.filter((item) => item.id !== entry.id))}
|
||||
aria-label={t("upload.remove", { name: entry.name })}
|
||||
className="grid size-5 place-items-center rounded-full text-[var(--text-subtle)] hover:text-[var(--text)]"
|
||||
>
|
||||
<XIcon className="size-3" />
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
|
||||
{fileError ? (
|
||||
<p role="alert" className="text-[12.5px] text-rose-700 dark:text-rose-300">
|
||||
{fileError}
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
<section className="card space-y-4 p-4">
|
||||
<div className="space-y-2">
|
||||
<p className="text-[11px] font-semibold uppercase tracking-wide text-[var(--text-subtle)]">
|
||||
{t("podcast.lengthAria")}
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{LENGTH_OPTIONS.map((option) => (
|
||||
<button
|
||||
key={option}
|
||||
type="button"
|
||||
aria-pressed={minutes === option}
|
||||
onClick={() => setMinutes(option)}
|
||||
className={cn(
|
||||
"chip touch-target inline-flex items-center gap-1.5 px-3 text-[13px]",
|
||||
minutes === option && "border-[var(--nav-active-border)] bg-[var(--nav-active-bg)]",
|
||||
)}
|
||||
>
|
||||
<ClockIcon className="size-4" />
|
||||
{plural(option, "podcast.minutes")}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-[12px] text-[var(--text-subtle)]">{t("podcast.lengthHelp")}</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1.5">
|
||||
<label htmlFor="podcast-focus" className="text-[11px] font-semibold uppercase tracking-wide text-[var(--text-subtle)]">
|
||||
{t("podcast.focusLabel")}
|
||||
</label>
|
||||
<input
|
||||
id="podcast-focus"
|
||||
value={focus}
|
||||
onChange={(event) => setFocus(event.target.value)}
|
||||
placeholder={t("upload.topicPlaceholder")}
|
||||
inputMode="text"
|
||||
enterKeyHint="done"
|
||||
autoComplete="off"
|
||||
autoCapitalize="sentences"
|
||||
spellCheck={false}
|
||||
className="w-full rounded-2xl border border-[var(--hairline)] bg-[var(--surface-strong)] px-4 py-3 text-sm text-[var(--text)] outline-none transition placeholder:text-[var(--text-subtle)] focus:border-accent-400"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1.5">
|
||||
<label htmlFor="podcast-text" className="text-[11px] font-semibold uppercase tracking-wide text-[var(--text-subtle)]">
|
||||
{t("upload.pastedNotes")}
|
||||
</label>
|
||||
<textarea
|
||||
id="podcast-text"
|
||||
value={text}
|
||||
onChange={(event) => setText(event.target.value)}
|
||||
rows={isPhone ? 4 : 6}
|
||||
placeholder={t("upload.pastedPlaceholder")}
|
||||
className="w-full resize-y rounded-2xl border border-[var(--hairline)] bg-[var(--surface-strong)] px-4 py-3 text-sm text-[var(--text)] outline-none transition placeholder:text-[var(--text-subtle)] focus:border-accent-400"
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{error ? (
|
||||
<section className="card space-y-3 border-rose-300/50 p-4">
|
||||
<p role="alert" className="text-[13px] leading-relaxed text-rose-700 dark:text-rose-300">
|
||||
{error.message}
|
||||
</p>
|
||||
<ModelErrorActions kind={error.kind} />
|
||||
</section>
|
||||
) : null}
|
||||
|
||||
<AiDisclaimer />
|
||||
|
||||
{/* Desktop action row; phones get the pinned wizard bar below. */}
|
||||
<div className="hidden sm:block">
|
||||
<Button
|
||||
size="lg"
|
||||
className="w-full touch-target"
|
||||
disabled={!canGenerate || extracting}
|
||||
onClick={() => void generate()}
|
||||
>
|
||||
<SparklesIcon className="size-4" /> {t("podcast.generate")}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<WizardActionBar>
|
||||
<Button
|
||||
size="lg"
|
||||
className="flex-1 touch-target"
|
||||
disabled={!canGenerate || extracting}
|
||||
onClick={() => void generate()}
|
||||
>
|
||||
<SparklesIcon className="size-4" /> {notesReady ? t("podcast.generate") : t("podcast.addFirst")}
|
||||
</Button>
|
||||
</WizardActionBar>
|
||||
|
||||
<input
|
||||
ref={fileInput}
|
||||
type="file"
|
||||
accept={ACCEPTED_TYPES.join(",")}
|
||||
multiple
|
||||
aria-hidden="true"
|
||||
tabIndex={-1}
|
||||
className="sr-only"
|
||||
onChange={(event) => {
|
||||
const files = [...(event.target.files ?? [])];
|
||||
if (files.length) void addImages(files.filter(isImageFile));
|
||||
event.target.value = "";
|
||||
}}
|
||||
/>
|
||||
<input
|
||||
ref={cameraInput}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
capture="environment"
|
||||
aria-hidden="true"
|
||||
tabIndex={-1}
|
||||
className="sr-only"
|
||||
onChange={(event) => {
|
||||
const files = [...(event.target.files ?? [])];
|
||||
if (files.length) void addImages(files.filter(isImageFile));
|
||||
event.target.value = "";
|
||||
}}
|
||||
/>
|
||||
<input
|
||||
ref={docInput}
|
||||
type="file"
|
||||
accept={DOCUMENT_ACCEPT_ATTR}
|
||||
multiple
|
||||
aria-hidden="true"
|
||||
tabIndex={-1}
|
||||
className="sr-only"
|
||||
onChange={(event) => {
|
||||
const files = [...(event.target.files ?? [])];
|
||||
if (files.length) void addDocuments(files.filter(isDocumentFile));
|
||||
event.target.value = "";
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue