Podcast speech: the highlight clock learns the engine's pace

"The tts dispatcher thing and the text highlight is completely out of sync on
desktop." Firefox + speech-dispatcher (like iOS) reports no onboundary events,
so the highlight rode a pure estimate - and a 130-word section drifts seconds
away from an engine that speaks at its own pace.

- On platforms without boundaries the system engine now speaks one sentence
  at a time, and every finished utterance re-anchors the clock: measured
  duration vs estimate, smoothed (half old, half new, clamped to 0.25-4x).
  Drift cannot accumulate past a sentence.
- The per-word estimate is punctuation-aware (full stop ~260ms, comma ~130ms).
- Once a boundary does arrive (Chrome), the rest of the section goes back to
  a single utterance with the exact clock - the accurate path is unchanged.
- When a boundary lands mid-section, the remainder is spoken as one utterance
  (it used to skip to the next section).

podcast-test's engine stub finishes utterances now and pins the whole thing:
one sentence first, the next following on its own, the highlight inside the
spoken text, a boundary collapsing the remainder, the highlight advancing.
53 checks. Version 0.7.10, SW v102.
This commit is contained in:
Martin Dvořák 2026-10-16 21:45:18 +00:00
commit 8c532ea6e6
5 changed files with 245 additions and 49 deletions

View file

@ -56,12 +56,28 @@ const LONG_RECORD = {
...EPISODE_RECORD,
id: "pod_long",
title: "A long walk through the atmosphere",
/* One long sentence per part on purpose: with the sentence clock a section
is spoken in one utterance only while it has no sentence break, which is
exactly the shape the scroll checks need (a boundary can land far away). */
sections: Array.from({ length: 8 }, (_, index) => ({
heading: `Part ${index + 1}`,
text: `Paragraph ${index + 1}. ${"The atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones. ".repeat(3)}`,
text: `Part ${index + 1}, ${"the atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones, ".repeat(3).trim()}`,
})),
};
/** Three sentences: the fallback clock must re-anchor at every one. */
const SENTENCES_RECORD = {
...EPISODE_RECORD,
id: "pod_sentences",
sections: [
{
heading: "Daylight",
text: "Look up on a clear day. The sky is blue. Molecules scatter short wavelengths far more than long ones. That is what reaches your eyes.",
},
{ heading: "Why", text: "That scattering is what colours the sky." },
],
};
/** A speech engine that never reports boundaries: the timer clock drives it. */
const WORD_TIMER_SPEECH = `
window.__utterances = [];
@ -566,6 +582,89 @@ try {
await context.close();
}
/* 4d. no boundary events: sentences carry the clock ---------------- */
{
const { context, page } = await makePage(browser, { width: 390, height: 844, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
/* An engine that never reports boundaries (Firefox + speech-dispatcher,
iOS) and *finishes* every utterance, so the sentence clock can be watched
sentence by sentence. */
await page.addInitScript(() => {
window.__utterances = [];
const SENTENCE_MS = 700;
class FakeUtterance {
constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.voice = null; this.onstart = null; this.onend = null; this.onerror = null; this.onboundary = null; }
}
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
Object.defineProperty(window, "speechSynthesis", {
configurable: true,
value: {
_speaking: false,
get speaking() { return this._speaking; },
pending: false,
paused: false,
speak(utterance) {
window.__utterances.push(utterance);
this._speaking = true;
const mine = this;
queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 }));
setTimeout(() => {
mine._speaking = false;
if (utterance.onend) utterance.onend();
}, SENTENCE_MS);
},
cancel() { this._speaking = false; },
resume() {},
getVoices: () => [{ name: "Daniel", lang: "en-GB", voiceURI: "daniel", localService: true, default: true }],
addEventListener() {},
removeEventListener() {},
},
});
});
await seed(page, { decks: [], theme: "dark", extra: { "freezmo.podcasts.v1": JSON.stringify([SENTENCES_RECORD]) } });
await page.goto(`${APP}/podcast/pod_sentences`, { waitUntil: "networkidle" });
await page.waitForTimeout(1000);
await page.getByRole("button", { name: "Play the episode" }).click();
await page.waitForTimeout(500);
const texts = () => page.evaluate(() => window.__utterances.map((u) => u.text));
/** Poll until at least `n` utterances have been spoken. */
const waitFor = async (n, timeout = 4000) => {
const started = Date.now();
while (Date.now() - started < timeout) {
if ((await page.evaluate(() => window.__utterances.length)) >= n) return true;
await page.waitForTimeout(80);
}
return false;
};
check("clock: without boundaries the first utterance is one sentence", (await waitFor(1)) && (await texts())[0] === "Look up on a clear day.", JSON.stringify(await texts()));
check("clock: the next sentence follows on its own", (await waitFor(2)) && (await texts())[1] === "The sky is blue.", JSON.stringify(await texts()));
const active = await page.evaluate(() => document.querySelector("[data-word-active]")?.textContent ?? "");
check("clock: the highlight is inside the spoken text", (await texts()).some((text) => text.includes(active)), JSON.stringify({ active }));
/* A boundary event on the sentence that is playing hands the rest of the
section to the exact clock: the next utterance is no longer one sentence. */
await page.evaluate(() => {
const utterance = window.__utterances[1];
if (utterance && utterance.onboundary) utterance.onboundary({ name: "word", charIndex: 0 });
});
await waitFor(3);
const spoken = await texts();
check(
"clock: a boundary switches the rest of the section to one utterance",
spoken.length >= 3 && (spoken[2].match(/\./g) ?? []).length >= 2,
JSON.stringify(spoken),
);
/* The highlight keeps moving with the voice instead of drifting away. */
const marks = [];
for (let i = 0; i < 5; i += 1) {
marks.push(await page.evaluate(() => document.querySelector("[data-word-active]")?.textContent ?? ""));
await page.waitForTimeout(350);
}
check("clock: the highlight advances over time", new Set(marks.filter(Boolean)).size >= 2, JSON.stringify(marks));
await context.close();}
/* 5. server guards ---------------------------------------------- */
const bad = await fetch(`${APP}/api/podcast`, {
method: "POST",