Martin Dvorak
9882d775e7
Safari hands web pages one voice per language, whatever the iPhone has installed, so the picker cannot fix that on iOS. It now carries the way out instead: Accessibility -> Spoken Content -> Voices -> your language, download an Enhanced/Premium voice and make it preferred. The speaker button wears an amber dot while the note is waiting, and the voice-less banner appends the same guidance. Detection covers iPadOS too (maxTouchPoints, since it reports as a Mac). The suite fakes an iPhone user agent, a single Daniel voice and pins the dot and the note's actual steps (37 checks). Version 0.7.5, SW v97.
438 lines
20 KiB
JavaScript
438 lines
20 KiB
JavaScript
/**
|
|
* Podcast mode: notes → a spoken episode with a synced transcript.
|
|
*
|
|
* Runs against a stub Ollama (no model needed) and a stubbed speech engine, so
|
|
* the word highlight can be driven deterministically: the stub records what
|
|
* was spoken and fires boundary events on demand.
|
|
*/
|
|
import http from "node:http";
|
|
import { mkdirSync } from "node:fs";
|
|
|
|
import {
|
|
launchBrowser,
|
|
makeChecker,
|
|
makePage,
|
|
portInUse,
|
|
seed,
|
|
spawnApp,
|
|
waitForHttp,
|
|
} from "./lib.mjs";
|
|
|
|
const { check, finish } = makeChecker();
|
|
|
|
const STUB_PORT = 3211;
|
|
const APP_PORT = 3212;
|
|
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
|
|
const APP = `http://127.0.0.1:${APP_PORT}`;
|
|
|
|
/** Two sections, a handful of words each — enough to watch the highlight walk. */
|
|
const EPISODE_REPLY = {
|
|
title: "Why the sky is blue",
|
|
language: "English",
|
|
summary: "Rayleigh scattering, explained out loud.",
|
|
sections: [
|
|
{ heading: "Daylight", text: "Look up on a clear day. The sky is blue." },
|
|
{ heading: "Why", text: "Tiny molecules scatter short wavelengths far more than long ones." },
|
|
],
|
|
};
|
|
|
|
/** An episode seeded straight into storage, for the voice/scroll sections. */
|
|
const EPISODE_RECORD = {
|
|
id: "pod_local",
|
|
title: "The sky, out loud",
|
|
summary: "Rayleigh scattering.",
|
|
language: "English",
|
|
sourceText: "Rayleigh scattering.",
|
|
createdAt: Date.now(),
|
|
updatedAt: Date.now(),
|
|
sections: [
|
|
{ heading: "Daylight", text: "Look up on a clear day. The sky is blue. Every direction you look, the same blue." },
|
|
{ heading: "Why", text: "Tiny molecules scatter short wavelengths far more than long ones, so blue reaches your eyes from everywhere at once." },
|
|
],
|
|
};
|
|
|
|
/** Long enough to scroll: eight paragraphs of a few lines each. */
|
|
const LONG_RECORD = {
|
|
...EPISODE_RECORD,
|
|
id: "pod_long",
|
|
title: "A long walk through the atmosphere",
|
|
sections: Array.from({ length: 8 }, (_, index) => ({
|
|
heading: `Part ${index + 1}`,
|
|
text: `Paragraph ${index + 1}. ${"The atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones. ".repeat(3)}`,
|
|
})),
|
|
};
|
|
|
|
/** A speech engine that never reports boundaries: the timer clock drives it. */
|
|
const WORD_TIMER_SPEECH = `
|
|
window.__utterances = [];
|
|
class FakeUtterance {
|
|
constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.onstart = null; this.onend = null; this.onerror = null; this.onboundary = null; }
|
|
}
|
|
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
|
|
Object.defineProperty(window, "speechSynthesis", {
|
|
configurable: true,
|
|
value: {
|
|
speak(utterance) { window.__utterances.push(utterance); queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 })); },
|
|
cancel() {}, getVoices() { return []; }, addEventListener() {}, removeEventListener() {}, speaking: false, paused: false,
|
|
},
|
|
});
|
|
`;
|
|
|
|
const stub = http.createServer((request, response) => {
|
|
if (request.url?.startsWith("/api/tags")) {
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
|
|
return;
|
|
}
|
|
let raw = "";
|
|
request.on("data", (chunk) => (raw += chunk));
|
|
request.on("end", () => {
|
|
let content = "{}";
|
|
try {
|
|
const body = JSON.parse(raw);
|
|
const system = String(body.messages?.[0]?.content ?? "");
|
|
/* The podcast writer answers slowly on purpose: a real script takes a
|
|
model a while, and the loading screen must be observable. */
|
|
if (system.includes("podcast writer")) {
|
|
content = JSON.stringify(EPISODE_REPLY);
|
|
setTimeout(() => {
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ message: { content } }));
|
|
}, 900);
|
|
return;
|
|
}
|
|
if (system.includes("study-note transcriber")) {
|
|
content = JSON.stringify({
|
|
title: "Sky notes",
|
|
language: "English",
|
|
notes: "Rayleigh scattering makes the sky blue.",
|
|
keyConcepts: ["scattering"],
|
|
});
|
|
}
|
|
} catch {
|
|
/* keep the default */
|
|
}
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ message: { content } }));
|
|
});
|
|
});
|
|
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));
|
|
|
|
check("podcast app port is free", !(await portInUse(APP_PORT)));
|
|
const syncDir = new URL("./.tmp/sync-podcast", import.meta.url).pathname;
|
|
mkdirSync(syncDir, { recursive: true });
|
|
const app = spawnApp(APP_PORT, {
|
|
OLLAMA_BASE_URL: STUB_BASE,
|
|
OLLAMA_AUTH: "local",
|
|
OLLAMA_VERIFY_KEY: "false",
|
|
SYNC_DIR: syncDir,
|
|
});
|
|
|
|
const browser = await launchBrowser();
|
|
|
|
/** Records utterances and lets the test fire boundaries/ends by hand. */
|
|
const SPEECH_STUB = `
|
|
window.__spoken = [];
|
|
window.__cancelCount = 0;
|
|
window.__utterances = [];
|
|
class FakeUtterance {
|
|
constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.onstart = null; this.onend = null; this.onerror = null; this.onboundary = null; }
|
|
}
|
|
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
|
|
Object.defineProperty(window, "speechSynthesis", {
|
|
configurable: true,
|
|
value: {
|
|
speak(utterance) {
|
|
window.__spoken.push(utterance.text);
|
|
window.__utterances.push(utterance);
|
|
queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 }));
|
|
},
|
|
cancel() { window.__cancelCount += 1; },
|
|
getVoices() { return []; },
|
|
speaking: false,
|
|
paused: false,
|
|
},
|
|
});
|
|
`;
|
|
|
|
try {
|
|
const up = await waitForHttp(`${APP}/api/health`);
|
|
check("podcast app instance is up", up);
|
|
|
|
if (up) {
|
|
/* 1. the wizard: mode switch, intake, generation ---------------- */
|
|
const { context, page } = await makePage(browser, { width: 1280, height: 900, deviceScaleFactor: 1 });
|
|
await page.addInitScript(SPEECH_STUB);
|
|
await seed(page, { decks: [], theme: "dark" });
|
|
|
|
await page.goto(`${APP}/create`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1000);
|
|
const switchLink = page.getByRole("link", { name: "Podcast" });
|
|
check("wizard: the mode switch carries a podcast segment", (await switchLink.count()) === 1);
|
|
await switchLink.click();
|
|
await page.waitForTimeout(1200);
|
|
check("wizard: the podcast intake opens", page.url().endsWith("/podcast"));
|
|
check("wizard: the length options are offered", (await page.getByRole("button", { name: "4 minutes" }).count()) === 1);
|
|
|
|
await page.fill("#podcast-text", "Rayleigh scattering makes the sky blue: molecules scatter short wavelengths far more than long ones.");
|
|
await page.waitForTimeout(200);
|
|
await page.getByRole("button", { name: "Create the episode" }).first().click();
|
|
/* Loading screen while the stub answers. */
|
|
await page.waitForTimeout(600);
|
|
const loading = (await page.getByText(/Writing the script|Reading your notes/).count()) >= 1;
|
|
await page.waitForURL(/\/podcast\/pod_/, { timeout: 30_000 }).catch(() => {});
|
|
await page.waitForTimeout(1200);
|
|
check("wizard: it lands on the player", /\/podcast\/pod_/.test(page.url()), page.url());
|
|
check("wizard: the loading screen showed a script stage", loading);
|
|
|
|
const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.podcasts.v1") ?? "[]"));
|
|
check("wizard: the episode is saved with both sections", stored.length === 1 && stored[0].sections.length === 2, JSON.stringify(stored[0]?.sections?.length));
|
|
check("wizard: the notes ride along for a rewrite", (stored[0]?.sourceText ?? "").includes("Rayleigh"));
|
|
|
|
/* 2. the player: transcript, highlight, controls ----------------- */
|
|
const words = await page.locator("[data-word]").count();
|
|
/* "Look up on a clear day. The sky is blue." (10) + "Tiny molecules
|
|
scatter short wavelengths far more than long ones." (10). */
|
|
check("player: the transcript is word-wrapped", words === 20, String(words));
|
|
check("player: nothing is highlighted before playing", (await page.locator("[data-word-active]").count()) === 0);
|
|
|
|
await page.getByRole("button", { name: "Play the episode" }).click();
|
|
await page.waitForTimeout(400);
|
|
const firstSpoken = await page.evaluate(() => window.__spoken[0]);
|
|
check("player: the first section is what gets spoken", String(firstSpoken).startsWith("Look up on a clear day."), String(firstSpoken).slice(0, 40));
|
|
check("player: the first word is highlighted", (await page.locator("[data-word-active]").count()) === 1);
|
|
|
|
/* The boundary events are the exact clock: jump to a later word. */
|
|
await page.evaluate(() => {
|
|
const utterance = window.__utterances.at(-1);
|
|
utterance.onboundary({ name: "word", charIndex: 12 });
|
|
});
|
|
await page.waitForTimeout(300);
|
|
const active = await page.locator("[data-word-active]").first();
|
|
check("player: a boundary event moves the highlight", (await active.textContent()) === "clear", String(await active.textContent()));
|
|
|
|
/* Pausing stops the voice and keeps the place. */
|
|
await page.getByRole("button", { name: "Pause the episode" }).click();
|
|
await page.waitForTimeout(300);
|
|
const cancelled = await page.evaluate(() => window.__cancelCount);
|
|
check("player: pause cancels the voice", cancelled >= 1, String(cancelled));
|
|
|
|
/* Resuming speaks from the highlighted word, not from the top. */
|
|
await page.getByRole("button", { name: "Play the episode" }).click();
|
|
await page.waitForTimeout(400);
|
|
const resumed = await page.evaluate(() => window.__spoken.at(-1));
|
|
check("player: resume starts at the current word", String(resumed).startsWith("clear day."), String(resumed).slice(0, 30));
|
|
|
|
/* 3. chapters, speed, progress save ------------------------------ */
|
|
await page.getByRole("button", { name: "Why", exact: true }).click();
|
|
await page.waitForTimeout(500);
|
|
const chapterSpoken = await page.evaluate(() => window.__spoken.at(-1));
|
|
check("player: a chapter jump speaks that section", String(chapterSpoken).startsWith("Tiny molecules"), String(chapterSpoken).slice(0, 30));
|
|
check(
|
|
"player: the header names the part",
|
|
(await page.getByText("Part 2 of 2").count()) === 1,
|
|
);
|
|
|
|
await page.getByRole("button", { name: "Playback speed" }).click();
|
|
await page.waitForTimeout(500);
|
|
const rate = await page.evaluate(() => {
|
|
const utterance = window.__utterances.at(-1);
|
|
return utterance?.rate;
|
|
});
|
|
check("player: the speed applies to the voice", rate === 1.15, String(rate));
|
|
|
|
/* Speed changes rebuild the engine mid-play; it keeps speaking. */
|
|
const spokenCount = await page.evaluate(() => window.__spoken.length);
|
|
check("player: changing speed keeps playing", spokenCount >= 4, String(spokenCount));
|
|
|
|
/* Leaving saves the position for the next visit. */
|
|
await page.goto(`${APP}/?tab=podcasts`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(900);
|
|
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.podcasts.v1") ?? "[]")[0]?.progress);
|
|
check("player: the position is saved on leave", Boolean(saved) && saved.section === 1, JSON.stringify(saved));
|
|
|
|
/* 4. the library tab -------------------------------------------- */
|
|
check("library: the podcasts tab lists the episode", (await page.getByText("Why the sky is blue").count()) === 1);
|
|
await page.getByRole("link", { name: "Listen" }).click();
|
|
await page.waitForTimeout(1200);
|
|
check("library: Listen opens the player", /\/podcast\/pod_/.test(page.url()), page.url());
|
|
check(
|
|
"player: reopening resumes at the saved part",
|
|
(await page.getByText("Part 2 of 2").count()) === 1,
|
|
);
|
|
await context.close();
|
|
|
|
/* 4b. voices: the best installed one, and a picker that sticks ------ */
|
|
{
|
|
const { context, page } = await makePage(browser, { width: 390, height: 844, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
|
|
/* A catalogue that looks like a bare Linux box next to Chrome: the robot
|
|
must lose to the natural male voice. */
|
|
await page.addInitScript(() => {
|
|
window.__spoken = [];
|
|
const voices = [
|
|
{ name: "eSpeak English", lang: "en-US", voiceURI: "espeak", localService: true, default: true },
|
|
{ name: "Google US English", lang: "en-US", voiceURI: "g-us", localService: false, default: false },
|
|
{ name: "Google UK English Male", lang: "en-GB", voiceURI: "g-uk-male", localService: false, default: false },
|
|
{ name: "Czech (Czech Republic)", lang: "cs-CZ", voiceURI: "cs-cz", localService: true, default: false },
|
|
];
|
|
class FakeUtterance {
|
|
constructor(text) {
|
|
this.text = String(text);
|
|
this.rate = 1;
|
|
this.lang = "en-US";
|
|
this.voice = null;
|
|
this.onstart = null;
|
|
this.onend = null;
|
|
this.onerror = null;
|
|
this.onboundary = null;
|
|
}
|
|
}
|
|
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
|
|
Object.defineProperty(window, "speechSynthesis", {
|
|
configurable: true,
|
|
value: {
|
|
speak(utterance) {
|
|
window.__spoken.push({ text: utterance.text, voice: utterance.voice?.voiceURI ?? null });
|
|
queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 }));
|
|
},
|
|
cancel() {},
|
|
getVoices: () => voices,
|
|
addEventListener() {},
|
|
removeEventListener() {},
|
|
speaking: false,
|
|
paused: false,
|
|
},
|
|
});
|
|
});
|
|
await seed(page, { decks: [], theme: "dark", extra: { "freezmo.podcasts.v1": JSON.stringify([EPISODE_RECORD]) } });
|
|
await page.goto(`${APP}/podcast/pod_local`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1000);
|
|
await page.getByRole("button", { name: "Play the episode" }).click();
|
|
await page.waitForTimeout(400);
|
|
const firstVoice = await page.evaluate(() => window.__spoken.at(-1)?.voice);
|
|
check("voice: the natural male voice beats the robot", firstVoice === "g-uk-male", String(firstVoice));
|
|
|
|
await page.getByRole("button", { name: "Voice" }).click();
|
|
await page.waitForTimeout(300);
|
|
const listed = await page.evaluate(() =>
|
|
[...document.querySelectorAll("[aria-pressed]")].map((el) => el.textContent?.trim() ?? ""),
|
|
);
|
|
check("voice: the picker lists the voices for the language", listed.some((entry) => entry.includes("Google UK English Male")), JSON.stringify(listed.slice(0, 4)));
|
|
check("voice: other languages are filtered out", !listed.some((entry) => entry.includes("Czech")), JSON.stringify(listed));
|
|
|
|
await page.getByRole("button", { name: /Google US English/ }).click();
|
|
await page.waitForTimeout(500);
|
|
const chosen = await page.evaluate(() => window.__spoken.at(-1)?.voice);
|
|
check("voice: choosing one speaks with it", chosen === "g-us", String(chosen));
|
|
const stored = await page.evaluate(() => localStorage.getItem("freezmo.podcast-voice.v1"));
|
|
check("voice: the choice is remembered", String(stored).includes("g-us"), String(stored));
|
|
await context.close();
|
|
}
|
|
|
|
/* 4b2. an iPhone gets told where the other voices live -------------- */
|
|
{
|
|
const { context, page } = await makePage(browser, { width: 390, height: 844, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
|
|
/* Safari's shape: one voice for the language, whatever is installed. */
|
|
await page.addInitScript(() => {
|
|
Object.defineProperty(navigator, "userAgent", {
|
|
configurable: true,
|
|
get: () => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1",
|
|
});
|
|
window.__spoken = [];
|
|
const voices = [{ name: "Daniel", lang: "en-GB", voiceURI: "daniel", localService: true, default: true }];
|
|
class FakeUtterance {
|
|
constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.voice = null; }
|
|
}
|
|
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
|
|
Object.defineProperty(window, "speechSynthesis", {
|
|
configurable: true,
|
|
value: { speak(u) { queueMicrotask(() => u.onstart?.({ name: "start", charIndex: 0 })); }, cancel() {}, getVoices: () => voices, addEventListener() {}, removeEventListener() {}, speaking: false, paused: false },
|
|
});
|
|
});
|
|
await seed(page, { decks: [], theme: "dark", extra: { "freezmo.podcasts.v1": JSON.stringify([EPISODE_RECORD]) } });
|
|
await page.goto(`${APP}/podcast/pod_local`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1000);
|
|
|
|
const voiceButton = page.getByRole("button", { name: "Voice" });
|
|
const badge = await voiceButton.evaluate((element) => Boolean(element.querySelector("span[aria-hidden]")));
|
|
check("ios: the voice button carries a hint dot", badge);
|
|
await voiceButton.click();
|
|
await page.waitForTimeout(300);
|
|
const note = await page.getByText(/Spoken Content/).count();
|
|
check("ios: the picker explains where to add voices", note === 1, String(note));
|
|
const steps = await page.evaluate(() => {
|
|
const text = [...document.querySelectorAll("p")].map((el) => el.textContent ?? "").find((entry) => entry.includes("Spoken Content")) ?? "";
|
|
return { settings: text.includes("Settings"), accessibility: text.includes("Accessibility"), premium: text.includes("Enhanced") || text.includes("Premium") };
|
|
});
|
|
check("ios: the note names the actual settings path", steps.settings && steps.accessibility && steps.premium, JSON.stringify(steps));
|
|
await context.close();
|
|
}
|
|
|
|
/* 4c. the transcript follows by line, and yields to a manual scroll --- */
|
|
{
|
|
const { context, page } = await makePage(browser, { width: 390, height: 700, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
|
|
await page.addInitScript(WORD_TIMER_SPEECH);
|
|
await seed(page, { decks: [], theme: "dark", extra: { "freezmo.podcasts.v1": JSON.stringify([LONG_RECORD]) } });
|
|
await page.goto(`${APP}/podcast/pod_long`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1000);
|
|
await page.getByRole("button", { name: "Play the episode" }).click();
|
|
await page.waitForTimeout(400);
|
|
|
|
const scrollTop = () => page.evaluate(() => document.querySelector("[data-testid=podcast-transcript]")?.scrollTop ?? -1);
|
|
const boundary = (charIndex) =>
|
|
page.evaluate((index) => {
|
|
const utterance = window.__utterances.at(-1);
|
|
utterance.onboundary({ name: "word", charIndex: index });
|
|
}, charIndex);
|
|
|
|
/* Three words into the same line: the page must not move. */
|
|
const start = await scrollTop();
|
|
await boundary(3);
|
|
await boundary(6);
|
|
await boundary(9);
|
|
await page.waitForTimeout(400);
|
|
check("scroll: words on the same line do not move the page", (await scrollTop()) === start, `${start} -> ${await scrollTop()}`);
|
|
|
|
/* A word far into the script: the page follows it. */
|
|
await boundary(900);
|
|
await page.waitForTimeout(700);
|
|
const followed = await scrollTop();
|
|
check("scroll: a new line is followed", followed > start, `${start} -> ${followed}`);
|
|
|
|
/* The reader scrolls on purpose: following pauses instead of fighting. */
|
|
await page.evaluate(() => {
|
|
const container = document.querySelector("[data-testid=podcast-transcript]");
|
|
container.scrollTop = 0;
|
|
container.dispatchEvent(new WheelEvent("wheel", { bubbles: true }));
|
|
});
|
|
await page.waitForTimeout(200);
|
|
const manual = await scrollTop();
|
|
await boundary(1400);
|
|
await page.waitForTimeout(500);
|
|
check("scroll: a manual scroll is left alone", (await scrollTop()) === manual, `${manual} -> ${await scrollTop()}`);
|
|
await context.close();
|
|
}
|
|
|
|
/* 5. server guards ---------------------------------------------- */
|
|
const bad = await fetch(`${APP}/api/podcast`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ text: "" }),
|
|
});
|
|
check("api: empty notes are refused", bad.status === 400, String(bad.status));
|
|
|
|
const streamed = await fetch(`${APP}/api/podcast`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ text: "Some notes about the sky.", minutes: 3 }),
|
|
});
|
|
const body = await streamed.text();
|
|
check("api: a script streams back with a podcast event", body.includes('"type":"podcast"') && body.includes('"sections"'), body.slice(0, 120));
|
|
}
|
|
} finally {
|
|
app.stop();
|
|
stub.close();
|
|
await browser.close();
|
|
}
|
|
|
|
finish();
|