Podcast mode: notes read aloud with a lyric-style word highlight
The third segment of the note-powered wizards (Study deck / Homework / Podcast) takes the same intake - photos, documents, pasted text, the share target - and turns it into a spoken episode. - prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown, no visual references), chaptered into 3-8 sections around a length target. - /api/podcast mirrors /api/generate's plumbing (rate limit, model gate, provider keys, heartbeats, the 2x deadline, the optional vision pass). - storage: frezmo.podcasts.v1, zod-validated, included in the backup and the encrypted sync payload; a new library tab lists episodes with listen / rename / delete-with-undo, and the wizard switch deep-links in. - player: the transcript is word-wrapped spans and the word under the voice rides a lyric-style highlight that keeps itself centred. Two clocks: exact onboundary events where the browser reports them, a per-word length timer where it does not (iOS Safari); the first boundary retires the timer. Pause = cancel + re-speak from the word, which makes resume, chapter jumps and tap-any-word-to-seek the same operation; the position saves on pause, page-hide and unmount, so reopening resumes. Speed rebuilds the engine and keeps playing; the Aero soundtrack pauses itself when the voice starts; a browser with no voice gets an honest note and a readable transcript. - the script sits on a glass pane (plain paragraphs over Aero's grass failed contrast; the scan now covers the player too). tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and drives the highlight by hand: 26 checks. All suites green, contrast clean. Version 0.7.3, SW v95.
This commit is contained in:
parent
84a94581dc
commit
d95c5c7151
30 changed files with 2575 additions and 26 deletions
234
app/api/podcast/route.ts
Normal file
234
app/api/podcast/route.ts
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
import { z } from "zod";
|
||||
|
||||
import { extractNotes, generatePodcast, notesToText, type PipelineStage } from "@/lib/ai";
|
||||
import { getServerConfig, resolveModelName } from "@/lib/config";
|
||||
import { INERT_OLLAMA_AUTH, resolveModelTarget } from "@/lib/providers-server";
|
||||
import { friendlyErrorMessage, OllamaError } from "@/lib/ollama";
|
||||
import { rejectCrossOrigin, rejectOversizedBody, requestLocale } from "@/lib/http-guard";
|
||||
import { clientKey, rateLimit, rateLimitResponse } from "@/lib/rate-limit";
|
||||
import { acquireModelGate } from "@/lib/model-guard";
|
||||
import { translate } from "@/lib/i18n-dict";
|
||||
import { dataUrlSchema } from "@/lib/schemas";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
export const maxDuration = 300;
|
||||
|
||||
/** ~8 MB per image after client-side resizing; up to 16 images (photos + PDF pages). */
|
||||
const MAX_IMAGE_CHARS = 8_000_000;
|
||||
|
||||
const requestSchema = z.object({
|
||||
textModel: z.string().min(1).max(200).optional(),
|
||||
visionModel: z.string().min(1).max(200).optional(),
|
||||
images: z.array(dataUrlSchema(MAX_IMAGE_CHARS)).max(16).optional(),
|
||||
text: z.string().max(120_000).optional(),
|
||||
/** Target episode length in minutes. */
|
||||
minutes: z.number().int().min(1).max(15).optional(),
|
||||
focus: z.string().max(300).optional(),
|
||||
/** Language the episode must be written in (usually detected by the vision step). */
|
||||
language: z.string().max(60).optional(),
|
||||
});
|
||||
|
||||
type StreamEvent =
|
||||
| { type: "stage"; stage: PipelineStage; message: string }
|
||||
| { type: "podcast"; podcast: unknown; meta: Record<string, unknown> }
|
||||
| { type: "error"; error: string; kind?: string }
|
||||
| { type: "ping" };
|
||||
|
||||
function encoder() {
|
||||
const enc = new TextEncoder();
|
||||
return (event: StreamEvent) => enc.encode(`${JSON.stringify(event)}\n`);
|
||||
}
|
||||
|
||||
/** Generous but finite: ~16 compressed images plus JSON overhead. */
|
||||
const MAX_BODY_BYTES = 33554432;
|
||||
|
||||
/**
|
||||
* Notes → a spoken episode. Same plumbing as /api/generate (rate limit, model
|
||||
* gate, provider keys, heartbeats, one hard deadline), one model call for the
|
||||
* script after the optional vision pass.
|
||||
*/
|
||||
export async function POST(request: Request) {
|
||||
const locale = requestLocale(request);
|
||||
const crossOrigin = rejectCrossOrigin(request);
|
||||
if (crossOrigin) return crossOrigin;
|
||||
const oversized = rejectOversizedBody(request, MAX_BODY_BYTES);
|
||||
if (oversized) return oversized;
|
||||
|
||||
const config = getServerConfig();
|
||||
|
||||
const limit = rateLimit(`podcast:${clientKey(request)}`, config.rateLimits.generate, config.rateLimits.windowMs);
|
||||
if (!limit.allowed) return rateLimitResponse(limit, locale);
|
||||
|
||||
const resolved = resolveModelTarget(config, request, locale);
|
||||
if ("error" in resolved) {
|
||||
return Response.json({ error: resolved.error.message, kind: resolved.error.kind }, { status: resolved.error.status });
|
||||
}
|
||||
const target = resolved.target;
|
||||
const provider = target.kind === "provider" ? target.target : undefined;
|
||||
const baseAuth = target.kind === "provider" ? INERT_OLLAMA_AUTH : target.auth;
|
||||
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = await request.json();
|
||||
} catch {
|
||||
return Response.json({ error: translate(locale, "server.bodyNotJson") }, { status: 400 });
|
||||
}
|
||||
|
||||
const parsed = requestSchema.safeParse(payload);
|
||||
if (!parsed.success) {
|
||||
return Response.json(
|
||||
{
|
||||
error: translate(locale, "server.invalidRequest", {
|
||||
detail: parsed.error.issues
|
||||
.slice(0, 3)
|
||||
.map((issue) => `${issue.path.join(".") || "body"} ${issue.message}`)
|
||||
.join("; "),
|
||||
}),
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const { images, text, minutes, focus, language } = parsed.data;
|
||||
const imageList = (images ?? []).map(stripDataUrlPrefix).filter(Boolean);
|
||||
|
||||
if (imageList.length === 0 && !text?.trim()) {
|
||||
return Response.json({ error: translate(locale, "server.needContent") }, { status: 400 });
|
||||
}
|
||||
|
||||
const providerTextModel = provider ? parsed.data.textModel ?? parsed.data.visionModel : undefined;
|
||||
const providerVisionModel = provider ? parsed.data.visionModel ?? parsed.data.textModel : undefined;
|
||||
if (provider && !providerTextModel) {
|
||||
return Response.json(
|
||||
{ error: translate(locale, "server.providerModelNeeded"), kind: "bad_provider" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const textModel = provider ? providerTextModel! : config.textModel;
|
||||
const visionModel = provider ? providerVisionModel ?? providerTextModel! : config.visionModel;
|
||||
|
||||
const gate = acquireModelGate(request, config, locale);
|
||||
if ("response" in gate) return gate.response;
|
||||
|
||||
const encode = encoder();
|
||||
const abort = new AbortController();
|
||||
/* One vision pass plus one script: the same 2x deadline as the deck flow. */
|
||||
const globalTimer = setTimeout(() => abort.abort(), config.timeoutMs * 2);
|
||||
const onRequestAbort = () => abort.abort();
|
||||
if (request.signal.aborted) abort.abort();
|
||||
else request.signal.addEventListener("abort", onRequestAbort, { once: true });
|
||||
let heartbeat: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
const send = (event: StreamEvent) => {
|
||||
try {
|
||||
controller.enqueue(encode(event));
|
||||
} catch {
|
||||
/* client went away */
|
||||
}
|
||||
};
|
||||
|
||||
heartbeat = setInterval(() => send({ type: "ping" }), 15_000);
|
||||
try {
|
||||
let sourceText: string;
|
||||
let detectedLanguage: string | undefined = language;
|
||||
let title: string | undefined;
|
||||
let summary: string | undefined;
|
||||
let repaired = false;
|
||||
|
||||
if (imageList.length > 0) {
|
||||
send({ type: "stage", stage: "vision", message: "Reading your notes…" });
|
||||
const extraction = await extractNotes({
|
||||
model: visionModel,
|
||||
repairModel: textModel,
|
||||
auth: baseAuth,
|
||||
provider,
|
||||
images: imageList,
|
||||
text,
|
||||
signal: abort.signal,
|
||||
onProgress: (message) => send({ type: "stage", stage: "vision", message }),
|
||||
});
|
||||
sourceText = notesToText(extraction.notes);
|
||||
title = extraction.notes.title;
|
||||
summary = extraction.notes.summary;
|
||||
detectedLanguage = extraction.notes.language || language;
|
||||
repaired = repaired || extraction.repaired || extraction.languageRepaired;
|
||||
send({ type: "stage", stage: "generation", message: "Writing the script…" });
|
||||
} else {
|
||||
sourceText = text!.trim();
|
||||
send({ type: "stage", stage: "generation", message: "Writing the script…" });
|
||||
}
|
||||
|
||||
const generated = await generatePodcast({
|
||||
model: textModel,
|
||||
auth: baseAuth,
|
||||
provider,
|
||||
notes: sourceText,
|
||||
minutes: minutes ?? 4,
|
||||
language: detectedLanguage,
|
||||
focus,
|
||||
signal: abort.signal,
|
||||
});
|
||||
|
||||
send({
|
||||
type: "podcast",
|
||||
podcast: {
|
||||
title: generated.podcast.title || title || "Your podcast",
|
||||
language: generated.podcast.language || detectedLanguage,
|
||||
summary: generated.podcast.summary ?? summary,
|
||||
sourceText,
|
||||
sections: generated.podcast.sections,
|
||||
},
|
||||
meta: {
|
||||
repaired,
|
||||
language: generated.podcast.language || detectedLanguage || null,
|
||||
visionModel: imageList.length ? (provider ? visionModel : resolveModelName(config.visionModel, baseAuth.directCloud)) : null,
|
||||
textModel: provider ? textModel : resolveModelName(config.textModel, baseAuth.directCloud),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const message = friendlyErrorMessage(error, baseAuth.baseUrl || provider?.baseUrl || "", locale);
|
||||
// Never log the request body: it may contain the user's notes.
|
||||
console.error("[api/podcast]", error instanceof OllamaError ? `${error.kind}: ${error.message}` : error);
|
||||
send({
|
||||
type: "error",
|
||||
error: message,
|
||||
kind: error instanceof OllamaError ? error.kind : "unknown",
|
||||
});
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
clearTimeout(globalTimer);
|
||||
request.signal.removeEventListener("abort", onRequestAbort);
|
||||
gate.gate.release();
|
||||
try {
|
||||
controller.close();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
clearTimeout(globalTimer);
|
||||
request.signal.removeEventListener("abort", onRequestAbort);
|
||||
gate.gate.release();
|
||||
abort.abort();
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Type": "application/x-ndjson; charset=utf-8",
|
||||
"Cache-Control": "no-store, no-transform",
|
||||
Connection: "keep-alive",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function stripDataUrlPrefix(value: string): string {
|
||||
const commaIndex = value.indexOf(",");
|
||||
if (value.startsWith("data:") && commaIndex !== -1) return value.slice(commaIndex + 1);
|
||||
return value;
|
||||
}
|
||||
|
|
@ -1306,6 +1306,23 @@ body:has(.scene-pause) .ice-scene :is(.ice-orb, .ice-bubble) {
|
|||
.chip { min-height: 44px; }
|
||||
}
|
||||
|
||||
/* Podcast transcript: the word under the voice rides a highlight, the way
|
||||
lyrics do in Spotify. Words are plain spans; the player flips one class. */
|
||||
.podcast-word {
|
||||
border-radius: 0.4rem;
|
||||
transition: background-color 140ms ease, color 140ms ease;
|
||||
cursor: pointer;
|
||||
}
|
||||
.podcast-word-active,
|
||||
.podcast-word:hover {
|
||||
background: var(--nav-active-bg);
|
||||
color: var(--text);
|
||||
box-shadow: 0 0 0 2px var(--nav-active-bg);
|
||||
}
|
||||
.podcast-word-active {
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* The AI note under a surface that shows model output -------------------- */
|
||||
.ai-disclaimer {
|
||||
font-size: 12px;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ export default async function HomePage({
|
|||
<AppHeader />
|
||||
<div className="mx-auto w-full max-w-6xl flex-1 px-4 pb-tabbar lg:px-10 lg:pb-16 lg:pt-10">
|
||||
<RouteTransition>
|
||||
<LibraryScreen initialTab={tab === "homework" ? "homework" : "decks"} />
|
||||
<LibraryScreen
|
||||
initialTab={tab === "homework" ? "homework" : tab === "podcasts" ? "podcasts" : "decks"}
|
||||
/>
|
||||
</RouteTransition>
|
||||
</div>
|
||||
</main>
|
||||
|
|
|
|||
72
app/podcast/[id]/page.tsx
Normal file
72
app/podcast/[id]/page.tsx
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
import { useParams, useRouter } from "next/navigation";
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import { ApiKeyGate } from "@/components/ApiKeyGate";
|
||||
import { PodcastPlayer } from "@/components/podcast/PodcastPlayer";
|
||||
import { Button } from "@/components/ui";
|
||||
import { useI18n } from "@/lib/i18n";
|
||||
import { getPodcast } from "@/lib/podcasts";
|
||||
import type { Podcast } from "@/lib/schemas";
|
||||
|
||||
export default function PodcastEpisodePage() {
|
||||
const params = useParams<{ id: string }>();
|
||||
const router = useRouter();
|
||||
const id = typeof params.id === "string" ? params.id : "";
|
||||
const { t } = useI18n();
|
||||
const [episode, setEpisode] = useState<Podcast | null>(null);
|
||||
const [loaded, setLoaded] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!id) {
|
||||
setLoaded(true);
|
||||
return;
|
||||
}
|
||||
setEpisode(getPodcast(id) ?? null);
|
||||
setLoaded(true);
|
||||
}, [id]);
|
||||
|
||||
if (!loaded) {
|
||||
return (
|
||||
<main className="frost-bg grid min-h-dvh place-items-center px-4">
|
||||
<div className="w-full max-w-xl space-y-4" aria-busy="true" aria-label={t("podcast.loadingEpisode")}>
|
||||
<div className="skeleton h-14 rounded-full" />
|
||||
<div className="skeleton h-28 rounded-3xl" />
|
||||
<div className="skeleton h-40 rounded-3xl" />
|
||||
</div>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
if (!episode) {
|
||||
return (
|
||||
<main className="frost-bg grid min-h-dvh place-items-center px-4">
|
||||
<div className="card max-w-md space-y-3 p-6 text-center">
|
||||
<p className="text-4xl" aria-hidden>
|
||||
🎙️
|
||||
</p>
|
||||
<h1 className="text-xl font-extrabold text-[var(--text)]">{t("podcast.notFound")}</h1>
|
||||
<p className="text-sm text-[var(--text-muted)]">{t("podcast.notFoundBody")}</p>
|
||||
<div className="flex justify-center gap-2 pt-1">
|
||||
<Link href="/?tab=podcasts" className="btn-secondary touch-target inline-flex h-11 items-center px-4 text-sm font-semibold">
|
||||
{t("review.done")}
|
||||
</Link>
|
||||
<Button size="lg" className="touch-target" onClick={() => router.push("/podcast")}>
|
||||
{t("podcast.title")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<ApiKeyGate>
|
||||
<main>
|
||||
<PodcastPlayer episode={episode} />
|
||||
</main>
|
||||
</ApiKeyGate>
|
||||
);
|
||||
}
|
||||
34
app/podcast/page.tsx
Normal file
34
app/podcast/page.tsx
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { ApiKeyGate } from "@/components/ApiKeyGate";
|
||||
import { AppHeader } from "@/components/AppHeader";
|
||||
import { RouteTransition } from "@/components/RouteTransition";
|
||||
import { Sidebar, SidebarOffset } from "@/components/Sidebar";
|
||||
import { PodcastWizard } from "@/components/podcast/PodcastWizard";
|
||||
import { WizardModeSwitch } from "@/components/WizardModeSwitch";
|
||||
|
||||
export const metadata = {
|
||||
title: "New podcast",
|
||||
};
|
||||
|
||||
/**
|
||||
* Podcast mode's intake: the same notes as a deck, but the output is a spoken
|
||||
* episode with a synced transcript (see components/podcast).
|
||||
*/
|
||||
export default function PodcastPage() {
|
||||
return (
|
||||
<ApiKeyGate>
|
||||
<div className="flex min-h-dvh">
|
||||
<Sidebar />
|
||||
<SidebarOffset />
|
||||
<main className="flex min-h-dvh min-w-0 flex-1 flex-col">
|
||||
<AppHeader />
|
||||
<div className="mx-auto w-full max-w-3xl flex-1 px-4 pb-tabbar lg:px-10 lg:pb-16 lg:pt-10">
|
||||
<WizardModeSwitch mode="podcast" />
|
||||
<RouteTransition>
|
||||
<PodcastWizard />
|
||||
</RouteTransition>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
</ApiKeyGate>
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue