Podcast speech: the highlight clock learns the engine's pace
"The tts dispatcher thing and the text highlight is completely out of sync on desktop." Firefox + speech-dispatcher (like iOS) reports no onboundary events, so the highlight rode a pure estimate - and a 130-word section drifts seconds away from an engine that speaks at its own pace. - On platforms without boundaries the system engine now speaks one sentence at a time, and every finished utterance re-anchors the clock: measured duration vs estimate, smoothed (half old, half new, clamped to 0.25-4x). Drift cannot accumulate past a sentence. - The per-word estimate is punctuation-aware (full stop ~260ms, comma ~130ms). - Once a boundary does arrive (Chrome), the rest of the section goes back to a single utterance with the exact clock - the accurate path is unchanged. - When a boundary lands mid-section, the remainder is spoken as one utterance (it used to skip to the next section). podcast-test's engine stub finishes utterances now and pins the whole thing: one sentence first, the next following on its own, the highlight inside the spoken text, a boundary collapsing the remainder, the highlight advancing. 53 checks. Version 0.7.10, SW v102.
This commit is contained in:
parent
724a068bec
commit
8c532ea6e6
5 changed files with 245 additions and 49 deletions
|
|
@ -56,12 +56,28 @@ const LONG_RECORD = {
|
|||
...EPISODE_RECORD,
|
||||
id: "pod_long",
|
||||
title: "A long walk through the atmosphere",
|
||||
/* One long sentence per part on purpose: with the sentence clock a section
|
||||
is spoken in one utterance only while it has no sentence break, which is
|
||||
exactly the shape the scroll checks need (a boundary can land far away). */
|
||||
sections: Array.from({ length: 8 }, (_, index) => ({
|
||||
heading: `Part ${index + 1}`,
|
||||
text: `Paragraph ${index + 1}. ${"The atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones. ".repeat(3)}`,
|
||||
text: `Part ${index + 1}, ${"the atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones, ".repeat(3).trim()}`,
|
||||
})),
|
||||
};
|
||||
|
||||
/** Three sentences: the fallback clock must re-anchor at every one. */
|
||||
const SENTENCES_RECORD = {
|
||||
...EPISODE_RECORD,
|
||||
id: "pod_sentences",
|
||||
sections: [
|
||||
{
|
||||
heading: "Daylight",
|
||||
text: "Look up on a clear day. The sky is blue. Molecules scatter short wavelengths far more than long ones. That is what reaches your eyes.",
|
||||
},
|
||||
{ heading: "Why", text: "That scattering is what colours the sky." },
|
||||
],
|
||||
};
|
||||
|
||||
/** A speech engine that never reports boundaries: the timer clock drives it. */
|
||||
const WORD_TIMER_SPEECH = `
|
||||
window.__utterances = [];
|
||||
|
|
@ -566,6 +582,89 @@ try {
|
|||
await context.close();
|
||||
}
|
||||
|
||||
/* 4d. no boundary events: sentences carry the clock ---------------- */
|
||||
{
|
||||
const { context, page } = await makePage(browser, { width: 390, height: 844, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
|
||||
/* An engine that never reports boundaries (Firefox + speech-dispatcher,
|
||||
iOS) and *finishes* every utterance, so the sentence clock can be watched
|
||||
sentence by sentence. */
|
||||
await page.addInitScript(() => {
|
||||
window.__utterances = [];
|
||||
const SENTENCE_MS = 700;
|
||||
class FakeUtterance {
|
||||
constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.voice = null; this.onstart = null; this.onend = null; this.onerror = null; this.onboundary = null; }
|
||||
}
|
||||
Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance });
|
||||
Object.defineProperty(window, "speechSynthesis", {
|
||||
configurable: true,
|
||||
value: {
|
||||
_speaking: false,
|
||||
get speaking() { return this._speaking; },
|
||||
pending: false,
|
||||
paused: false,
|
||||
speak(utterance) {
|
||||
window.__utterances.push(utterance);
|
||||
this._speaking = true;
|
||||
const mine = this;
|
||||
queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 }));
|
||||
setTimeout(() => {
|
||||
mine._speaking = false;
|
||||
if (utterance.onend) utterance.onend();
|
||||
}, SENTENCE_MS);
|
||||
},
|
||||
cancel() { this._speaking = false; },
|
||||
resume() {},
|
||||
getVoices: () => [{ name: "Daniel", lang: "en-GB", voiceURI: "daniel", localService: true, default: true }],
|
||||
addEventListener() {},
|
||||
removeEventListener() {},
|
||||
},
|
||||
});
|
||||
});
|
||||
await seed(page, { decks: [], theme: "dark", extra: { "freezmo.podcasts.v1": JSON.stringify([SENTENCES_RECORD]) } });
|
||||
await page.goto(`${APP}/podcast/pod_sentences`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(1000);
|
||||
await page.getByRole("button", { name: "Play the episode" }).click();
|
||||
await page.waitForTimeout(500);
|
||||
|
||||
const texts = () => page.evaluate(() => window.__utterances.map((u) => u.text));
|
||||
/** Poll until at least `n` utterances have been spoken. */
|
||||
const waitFor = async (n, timeout = 4000) => {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeout) {
|
||||
if ((await page.evaluate(() => window.__utterances.length)) >= n) return true;
|
||||
await page.waitForTimeout(80);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
check("clock: without boundaries the first utterance is one sentence", (await waitFor(1)) && (await texts())[0] === "Look up on a clear day.", JSON.stringify(await texts()));
|
||||
check("clock: the next sentence follows on its own", (await waitFor(2)) && (await texts())[1] === "The sky is blue.", JSON.stringify(await texts()));
|
||||
const active = await page.evaluate(() => document.querySelector("[data-word-active]")?.textContent ?? "");
|
||||
check("clock: the highlight is inside the spoken text", (await texts()).some((text) => text.includes(active)), JSON.stringify({ active }));
|
||||
|
||||
/* A boundary event on the sentence that is playing hands the rest of the
|
||||
section to the exact clock: the next utterance is no longer one sentence. */
|
||||
await page.evaluate(() => {
|
||||
const utterance = window.__utterances[1];
|
||||
if (utterance && utterance.onboundary) utterance.onboundary({ name: "word", charIndex: 0 });
|
||||
});
|
||||
await waitFor(3);
|
||||
const spoken = await texts();
|
||||
check(
|
||||
"clock: a boundary switches the rest of the section to one utterance",
|
||||
spoken.length >= 3 && (spoken[2].match(/\./g) ?? []).length >= 2,
|
||||
JSON.stringify(spoken),
|
||||
);
|
||||
|
||||
/* The highlight keeps moving with the voice instead of drifting away. */
|
||||
const marks = [];
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
marks.push(await page.evaluate(() => document.querySelector("[data-word-active]")?.textContent ?? ""));
|
||||
await page.waitForTimeout(350);
|
||||
}
|
||||
check("clock: the highlight advances over time", new Set(marks.filter(Boolean)).size >= 2, JSON.stringify(marks));
|
||||
await context.close();}
|
||||
|
||||
/* 5. server guards ---------------------------------------------- */
|
||||
const bad = await fetch(`${APP}/api/podcast`, {
|
||||
method: "POST",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue