Podcast mode: notes read aloud with a lyric-style word highlight
The third segment of the note-powered wizards (Study deck / Homework / Podcast) takes the same intake - photos, documents, pasted text, the share target - and turns it into a spoken episode. - prompts: PODCAST_SYSTEM_PROMPT writes for the ear (plain prose, no markdown, no visual references), chaptered into 3-8 sections around a length target. - /api/podcast mirrors /api/generate's plumbing (rate limit, model gate, provider keys, heartbeats, the 2x deadline, the optional vision pass). - storage: frezmo.podcasts.v1, zod-validated, included in the backup and the encrypted sync payload; a new library tab lists episodes with listen / rename / delete-with-undo, and the wizard switch deep-links in. - player: the transcript is word-wrapped spans and the word under the voice rides a lyric-style highlight that keeps itself centred. Two clocks: exact onboundary events where the browser reports them, a per-word length timer where it does not (iOS Safari); the first boundary retires the timer. Pause = cancel + re-speak from the word, which makes resume, chapter jumps and tap-any-word-to-seek the same operation; the position saves on pause, page-hide and unmount, so reopening resumes. Speed rebuilds the engine and keeps playing; the Aero soundtrack pauses itself when the voice starts; a browser with no voice gets an honest note and a readable transcript. - the script sits on a glass pane (plain paragraphs over Aero's grass failed contrast; the scan now covers the player too). tests/podcast-test.mjs (17th suite) stubs the model and the speech engine and drives the highlight by hand: 26 checks. All suites green, contrast clean. Version 0.7.3, SW v95.
This commit is contained in:
parent
84a94581dc
commit
d95c5c7151
30 changed files with 2575 additions and 26 deletions
245
tests/podcast-test.mjs
Normal file
245
tests/podcast-test.mjs
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
/**
|
||||
* Podcast mode: notes → a spoken episode with a synced transcript.
|
||||
*
|
||||
* Runs against a stub Ollama (no model needed) and a stubbed speech engine, so
|
||||
* the word highlight can be driven deterministically: the stub records what
|
||||
* was spoken and fires boundary events on demand.
|
||||
*/
|
||||
import http from "node:http";
|
||||
import { mkdirSync } from "node:fs";
|
||||
|
||||
import {
|
||||
launchBrowser,
|
||||
makeChecker,
|
||||
makePage,
|
||||
portInUse,
|
||||
seed,
|
||||
spawnApp,
|
||||
waitForHttp,
|
||||
} from "./lib.mjs";
|
||||
|
||||
const { check, finish } = makeChecker();
|
||||
|
||||
const STUB_PORT = 3211;
|
||||
const APP_PORT = 3212;
|
||||
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
|
||||
const APP = `http://127.0.0.1:${APP_PORT}`;
|
||||
|
||||
/** Two sections, a handful of words each — enough to watch the highlight walk. */
|
||||
const EPISODE_REPLY = {
|
||||
title: "Why the sky is blue",
|
||||
language: "English",
|
||||
summary: "Rayleigh scattering, explained out loud.",
|
||||
sections: [
|
||||
{ heading: "Daylight", text: "Look up on a clear day. The sky is blue." },
|
||||
{ heading: "Why", text: "Tiny molecules scatter short wavelengths far more than long ones." },
|
||||
],
|
||||
};
|
||||
|
||||
const stub = http.createServer((request, response) => {
|
||||
if (request.url?.startsWith("/api/tags")) {
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
|
||||
return;
|
||||
}
|
||||
let raw = "";
|
||||
request.on("data", (chunk) => (raw += chunk));
|
||||
request.on("end", () => {
|
||||
let content = "{}";
|
||||
try {
|
||||
const body = JSON.parse(raw);
|
||||
const system = String(body.messages?.[0]?.content ?? "");
|
||||
/* The podcast writer answers slowly on purpose: a real script takes a
|
||||
model a while, and the loading screen must be observable. */
|
||||
if (system.includes("podcast writer")) {
|
||||
content = JSON.stringify(EPISODE_REPLY);
|
||||
setTimeout(() => {
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ message: { content } }));
|
||||
}, 900);
|
||||
return;
|
||||
}
|
||||
if (system.includes("study-note transcriber")) {
|
||||
content = JSON.stringify({
|
||||
title: "Sky notes",
|
||||
language: "English",
|
||||
notes: "Rayleigh scattering makes the sky blue.",
|
||||
keyConcepts: ["scattering"],
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
/* keep the default */
|
||||
}
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ message: { content } }));
|
||||
});
|
||||
});
|
||||
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));
|
||||
|
||||
check("podcast app port is free", !(await portInUse(APP_PORT)));
|
||||
const syncDir = new URL("./.tmp/sync-podcast", import.meta.url).pathname;
|
||||
mkdirSync(syncDir, { recursive: true });
|
||||
const app = spawnApp(APP_PORT, {
|
||||
OLLAMA_BASE_URL: STUB_BASE,
|
||||
OLLAMA_AUTH: "local",
|
||||
OLLAMA_VERIFY_KEY: "false",
|
||||
SYNC_DIR: syncDir,
|
||||
});
|
||||
|
||||
const browser = await launchBrowser();
|
||||
|
||||
/** Records utterances and lets the test fire boundaries/ends by hand. */
|
||||
const SPEECH_STUB = `
|
||||
window.__spoken = [];
|
||||
window.__cancelCount = 0;
|
||||
window.__utterances = [];
|
||||
class FakeUtterance {
|
||||
constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.onstart = null; this.onend = null; this.onerror = null; this.onboundary = null; }
|
||||
}
|
||||
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
|
||||
Object.defineProperty(window, "speechSynthesis", {
|
||||
configurable: true,
|
||||
value: {
|
||||
speak(utterance) {
|
||||
window.__spoken.push(utterance.text);
|
||||
window.__utterances.push(utterance);
|
||||
queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 }));
|
||||
},
|
||||
cancel() { window.__cancelCount += 1; },
|
||||
getVoices() { return []; },
|
||||
speaking: false,
|
||||
paused: false,
|
||||
},
|
||||
});
|
||||
`;
|
||||
|
||||
try {
|
||||
const up = await waitForHttp(`${APP}/api/health`);
|
||||
check("podcast app instance is up", up);
|
||||
|
||||
if (up) {
|
||||
/* 1. the wizard: mode switch, intake, generation ---------------- */
|
||||
const { context, page } = await makePage(browser, { width: 1280, height: 900, deviceScaleFactor: 1 });
|
||||
await page.addInitScript(SPEECH_STUB);
|
||||
await seed(page, { decks: [], theme: "dark" });
|
||||
|
||||
await page.goto(`${APP}/create`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(1000);
|
||||
const switchLink = page.getByRole("link", { name: "Podcast" });
|
||||
check("wizard: the mode switch carries a podcast segment", (await switchLink.count()) === 1);
|
||||
await switchLink.click();
|
||||
await page.waitForTimeout(1200);
|
||||
check("wizard: the podcast intake opens", page.url().endsWith("/podcast"));
|
||||
check("wizard: the length options are offered", (await page.getByRole("button", { name: "4 minutes" }).count()) === 1);
|
||||
|
||||
await page.fill("#podcast-text", "Rayleigh scattering makes the sky blue: molecules scatter short wavelengths far more than long ones.");
|
||||
await page.waitForTimeout(200);
|
||||
await page.getByRole("button", { name: "Create the episode" }).first().click();
|
||||
/* Loading screen while the stub answers. */
|
||||
await page.waitForTimeout(600);
|
||||
const loading = (await page.getByText(/Writing the script|Reading your notes/).count()) >= 1;
|
||||
await page.waitForURL(/\/podcast\/pod_/, { timeout: 30_000 }).catch(() => {});
|
||||
await page.waitForTimeout(1200);
|
||||
check("wizard: it lands on the player", /\/podcast\/pod_/.test(page.url()), page.url());
|
||||
check("wizard: the loading screen showed a script stage", loading);
|
||||
|
||||
const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.podcasts.v1") ?? "[]"));
|
||||
check("wizard: the episode is saved with both sections", stored.length === 1 && stored[0].sections.length === 2, JSON.stringify(stored[0]?.sections?.length));
|
||||
check("wizard: the notes ride along for a rewrite", (stored[0]?.sourceText ?? "").includes("Rayleigh"));
|
||||
|
||||
/* 2. the player: transcript, highlight, controls ----------------- */
|
||||
const words = await page.locator("[data-word]").count();
|
||||
/* "Look up on a clear day. The sky is blue." (10) + "Tiny molecules
|
||||
scatter short wavelengths far more than long ones." (10). */
|
||||
check("player: the transcript is word-wrapped", words === 20, String(words));
|
||||
check("player: nothing is highlighted before playing", (await page.locator("[data-word-active]").count()) === 0);
|
||||
|
||||
await page.getByRole("button", { name: "Play the episode" }).click();
|
||||
await page.waitForTimeout(400);
|
||||
const firstSpoken = await page.evaluate(() => window.__spoken[0]);
|
||||
check("player: the first section is what gets spoken", String(firstSpoken).startsWith("Look up on a clear day."), String(firstSpoken).slice(0, 40));
|
||||
check("player: the first word is highlighted", (await page.locator("[data-word-active]").count()) === 1);
|
||||
|
||||
/* The boundary events are the exact clock: jump to a later word. */
|
||||
await page.evaluate(() => {
|
||||
const utterance = window.__utterances.at(-1);
|
||||
utterance.onboundary({ name: "word", charIndex: 12 });
|
||||
});
|
||||
await page.waitForTimeout(300);
|
||||
const active = await page.locator("[data-word-active]").first();
|
||||
check("player: a boundary event moves the highlight", (await active.textContent()) === "clear", String(await active.textContent()));
|
||||
|
||||
/* Pausing stops the voice and keeps the place. */
|
||||
await page.getByRole("button", { name: "Pause the episode" }).click();
|
||||
await page.waitForTimeout(300);
|
||||
const cancelled = await page.evaluate(() => window.__cancelCount);
|
||||
check("player: pause cancels the voice", cancelled >= 1, String(cancelled));
|
||||
|
||||
/* Resuming speaks from the highlighted word, not from the top. */
|
||||
await page.getByRole("button", { name: "Play the episode" }).click();
|
||||
await page.waitForTimeout(400);
|
||||
const resumed = await page.evaluate(() => window.__spoken.at(-1));
|
||||
check("player: resume starts at the current word", String(resumed).startsWith("clear day."), String(resumed).slice(0, 30));
|
||||
|
||||
/* 3. chapters, speed, progress save ------------------------------ */
|
||||
await page.getByRole("button", { name: "Why", exact: true }).click();
|
||||
await page.waitForTimeout(500);
|
||||
const chapterSpoken = await page.evaluate(() => window.__spoken.at(-1));
|
||||
check("player: a chapter jump speaks that section", String(chapterSpoken).startsWith("Tiny molecules"), String(chapterSpoken).slice(0, 30));
|
||||
check(
|
||||
"player: the header names the part",
|
||||
(await page.getByText("Part 2 of 2").count()) === 1,
|
||||
);
|
||||
|
||||
await page.getByRole("button", { name: "Playback speed" }).click();
|
||||
await page.waitForTimeout(500);
|
||||
const rate = await page.evaluate(() => {
|
||||
const utterance = window.__utterances.at(-1);
|
||||
return utterance?.rate;
|
||||
});
|
||||
check("player: the speed applies to the voice", rate === 1.15, String(rate));
|
||||
|
||||
/* Speed changes rebuild the engine mid-play; it keeps speaking. */
|
||||
const spokenCount = await page.evaluate(() => window.__spoken.length);
|
||||
check("player: changing speed keeps playing", spokenCount >= 4, String(spokenCount));
|
||||
|
||||
/* Leaving saves the position for the next visit. */
|
||||
await page.goto(`${APP}/?tab=podcasts`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(900);
|
||||
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.podcasts.v1") ?? "[]")[0]?.progress);
|
||||
check("player: the position is saved on leave", Boolean(saved) && saved.section === 1, JSON.stringify(saved));
|
||||
|
||||
/* 4. the library tab -------------------------------------------- */
|
||||
check("library: the podcasts tab lists the episode", (await page.getByText("Why the sky is blue").count()) === 1);
|
||||
await page.getByRole("link", { name: "Listen" }).click();
|
||||
await page.waitForTimeout(1200);
|
||||
check("library: Listen opens the player", /\/podcast\/pod_/.test(page.url()), page.url());
|
||||
check(
|
||||
"player: reopening resumes at the saved part",
|
||||
(await page.getByText("Part 2 of 2").count()) === 1,
|
||||
);
|
||||
await context.close();
|
||||
|
||||
/* 5. server guards ---------------------------------------------- */
|
||||
const bad = await fetch(`${APP}/api/podcast`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: "" }),
|
||||
});
|
||||
check("api: empty notes are refused", bad.status === 400, String(bad.status));
|
||||
|
||||
const streamed = await fetch(`${APP}/api/podcast`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: "Some notes about the sky.", minutes: 3 }),
|
||||
});
|
||||
const body = await streamed.text();
|
||||
check("api: a script streams back with a podcast event", body.includes('"type":"podcast"') && body.includes('"sections"'), body.slice(0, 120));
|
||||
}
|
||||
} finally {
|
||||
app.stop();
|
||||
stub.close();
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
finish();
|
||||
Loading…
Add table
Add a link
Reference in a new issue