From 8c532ea6e6dfcdb975ab55c5301d1ea788fa5d39 Mon Sep 17 00:00:00 2001 From: Martin Dvorak Date: Fri, 16 Oct 2026 21:45:18 +0000 Subject: [PATCH] Podcast speech: the highlight clock learns the engine's pace "The tts dispatcher thing and the text highlight is completely out of sync on desktop." Firefox + speech-dispatcher (like iOS) reports no onboundary events, so the highlight rode a pure estimate - and a 130-word section drifts seconds away from an engine that speaks at its own pace. - On platforms without boundaries the system engine now speaks one sentence at a time, and every finished utterance re-anchors the clock: measured duration vs estimate, smoothed (half old, half new, clamped to 0.25-4x). Drift cannot accumulate past a sentence. - The per-word estimate is punctuation-aware (full stop ~260ms, comma ~130ms). - Once a boundary does arrive (Chrome), the rest of the section goes back to a single utterance with the exact clock - the accurate path is unchanged. - When a boundary lands mid-section, the remainder is spoken as one utterance (it used to skip to the next section). podcast-test's engine stub finishes utterances now and pins the whole thing: one sentence first, the next following on its own, the highlight inside the spoken text, a boundary collapsing the remainder, the highlight advancing. 53 checks. Version 0.7.10, SW v102. --- README.md | 18 +++++ lib/podcast-speech.ts | 171 ++++++++++++++++++++++++++++++----------- package.json | 2 +- public/sw.js | 2 +- tests/podcast-test.mjs | 101 +++++++++++++++++++++++- 5 files changed, 245 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index e30fa94..bcfb6d7 100644 --- a/README.md +++ b/README.md @@ -1492,6 +1492,24 @@ Things I chose while building, with the reasoning: into view as you type. No strings, so no i18n; the additive change sits on top of the merged scene-heat rework unchanged. +108. **The clock that learned the engine's pace.** "The tts dispatcher thing and the text highlight is + completely out of sync on desktop" — Firefox + speech-dispatcher (like iOS) reports **no + `onboundary` events**, so the highlight rides the estimated clock, and that estimate drifts + seconds behind or ahead over a 130-word section; engines also speak at their own pace (espeak + is nothing like 150 wpm). The system engine now speaks **one sentence at a time** on those + platforms and re-anchors at every finish: each utterance's *measured* duration is compared + with its estimate, and a smoothed factor (half old, half new, clamped to 0.25–4×) scales the + next stretch — so the clock follows the engine instead of guessing, and the drift can never + accumulate past a sentence. The estimate itself got punctuation-aware too (a full stop costs + ~260 ms, a comma ~130 ms: that is where these engines actually spend their time). Where + boundaries *do* arrive (Chrome), the first one switches the rest of the section back to a + single utterance with the exact clock, so the accurate path stays exactly as it was. Pause, + seek and resume are unchanged: they re-speak from a word, and the sentence lookups are derived + from the same token list. The suite's engine stub now *finishes* utterances and pins all of it: + one sentence first, the next sentence following on its own, the highlight staying inside the + spoken text, a boundary collapsing the rest of the section into one utterance, and the + highlight advancing over time (53 checks). + 107. **The Aero soundtrack button rides above the podcast player.** "The music thing button with the Frutiger Aero doesn't work on the podcasts": the podcast page is a full-screen overlay at `z-45` and the music control sat at `z-40` — visible through the transparent page, but every diff --git a/lib/podcast-speech.ts b/lib/podcast-speech.ts index f1860c1..5904fec 100644 --- a/lib/podcast-speech.ts +++ b/lib/podcast-speech.ts @@ -80,9 +80,32 @@ export function tokenizeSections(sections: PodcastSection[]): Token[] { return tokens; } -/** Estimated spoken time of one word at rate 1 (ms). */ +/** Estimated spoken time of one word at rate 1 (ms). Punctuation is where + * these engines actually spend their time (sentences stop, commas slow). */ function wordDuration(text: string): number { - return 95 + 42 * Math.max(1, text.replace(/[^\p{L}\p{N}]/gu, "").length); + const letters = Math.max(1, text.replace(/[^\p{L}\p{N}]/gu, "").length); + let ms = 80 + 42 * letters; + if (/[.!?…]["')\]]*$/.test(text)) ms += 260; + else if (/[,;:]["')\]]*$/.test(text)) ms += 130; + return ms; +} + +/** One sentence of a section, in word indices (inclusive). */ +type SentenceRange = { from: number; to: number }; + +/** Sentence spans for a section's words, split on end punctuation. */ +function sentenceRanges(sectionTokens: Token[]): SentenceRange[] { + const ranges: SentenceRange[] = []; + let from = 0; + sectionTokens.forEach((token, index) => { + const endsSentence = /[.!?…]["')\]]*$/.test(token.text); + const last = index === sectionTokens.length - 1; + if (endsSentence || last) { + ranges.push({ from, to: index }); + from = index + 1; + } + }); + return ranges.length > 0 ? ranges : [{ from: 0, to: Math.max(0, sectionTokens.length - 1) }]; } export type PodcastSpeech = { @@ -105,6 +128,18 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech /** A cancel in the last moments: some engines drop a speak() that follows * one immediately, so it gets a tick of air. */ let lastCancelAt = 0; + /** When the current utterance actually began (calibration). */ + let startedAt = 0; + /** A breath between sentences: narration needs it, and it keeps the + * boundary pops of the system engines down. */ + const SENTENCE_BEAT_MS = 90; + /** + * Measured ÷ estimated speech time, learned from every finished utterance. + * Engines without boundary events are estimated word by word; without this + * factor the highlight drifts seconds behind on a long section (espeak and + * speech-dispatcher speak at their own pace, nothing like 150 wpm). + */ + let calibration = 1; const clearTimer = () => { if (timer !== null) { @@ -120,35 +155,33 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech const wordsIn = (section: number) => tokens.filter((token) => token.section === section); - /** Speak the section from `word` on. Empty word list → the next section. */ - const speakFrom = (target: PodcastVoice) => { - if (!speechAvailable()) { - events.onError?.("unsupported"); - return; - } - const section = events.sections[target.section]; + /** + * Speak one stretch of a section: `to` is the last word index included. + * Without boundary events the stretches are sentences (so the clock is + * re-calibrated every few seconds and the highlight cannot drift away); + * once boundaries are proven, the rest of the section goes in one go. + */ + const speakRange = (sectionIndex: number, fromWord: number, toWord: number) => { + const section = events.sections[sectionIndex]; if (!section) { position = null; finishPass(); events.onEnd(); return; } - const sectionTokens = wordsIn(target.section); + const sectionTokens = wordsIn(sectionIndex); if (sectionTokens.length === 0) { - speakFrom({ section: target.section + 1, word: 0 }); + speakRange(sectionIndex + 1, 0, 0); return; } - const word = Math.min(Math.max(0, target.word), sectionTokens.length - 1); - position = { section: target.section, word }; + const from = Math.min(Math.max(0, fromWord), sectionTokens.length - 1); + const to = Math.min(Math.max(from, toWord), sectionTokens.length - 1); + position = { section: sectionIndex, word: from }; - const from = sectionTokens[word].start; - /* Spoken text hygiene: engines click on control characters, stray - double spaces and unicode ellipses, and some pause oddly around them. */ - const text = normaliseSpoken(section.text.slice(from)); + const base = sectionTokens[from].start; + const text = normaliseSpoken(section.text.slice(base, sectionTokens[to].end)); const current = ++generation; try { - /* Only cancel when something is actually pending: an idle cancel makes - several engines pop. */ const synth = window.speechSynthesis; const cancelNeeded = synth.speaking || synth.pending; const cancelledRecently = Date.now() - lastCancelAt < 400; @@ -160,22 +193,26 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech pauses it on its own): without this resume the next speak() queues silently and the player looks dead after one listen. */ if (synth.paused) synth.resume(); + const utterance = new SpeechSynthesisUtterance(text); utterance.lang = events.locale; - utterance.rate = Math.min(3, Math.max(0.3, events.getRate?.() ?? 1)); + const rate = Math.min(3, Math.max(0.3, events.getRate?.() ?? 1)); + utterance.rate = rate; const voice = events.getVoice?.() ?? null; if (voice) utterance.voice = voice; - const offsetFor = (charIndex: number) => { - /* First token whose span ends after the reported character. */ - let found = word; - for (let i = word; i < sectionTokens.length; i += 1) { - const token = sectionTokens[i]; - if (token.end > from + charIndex) { - found = i; - break; - } + /** Estimated time of the words from..to at the current calibration. */ + const estimate = () => { + let ms = 0; + for (let i = from; i <= to; i += 1) ms += wordDuration(sectionTokens[i].text); + return ms / rate; + }; + + const indexAt = (charIndex: number) => { + let found = from; + for (let i = from; i <= to; i += 1) { found = i; + if (sectionTokens[i].end > base + charIndex) break; } return found; }; @@ -185,44 +222,72 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech if (event.name && event.name !== "word") return; boundariesWork = true; clearTimer(); - const index = offsetFor(event.charIndex); - position = { section: target.section, word: index }; + const index = indexAt(event.charIndex); + position = { section: sectionIndex, word: index }; events.onWord(position); }; utterance.onstart = () => { if (generation !== current) return; playing = true; + startedAt = Date.now(); events.onPlay?.(); - events.onWord(position ?? { section: target.section, word }); - /* The fallback clock: only while this browser has never reported a - * boundary. Each tick schedules the next word by its length. */ + events.onWord(position ?? { section: sectionIndex, word: from }); + /* The fallback clock: word by word, at the pace this engine has been + measured to speak. Retired for good once a boundary arrives. */ if (boundariesWork !== true) { - let index = word; + let index = from; const tick = () => { if (generation !== current) return; index += 1; - if (index >= sectionTokens.length) { - /* Let onend take over; a long word list can outlast the audio - * by a hair, and stopping the tick avoids double-advancing. */ + if (index > to) { timer = null; return; } - position = { section: target.section, word: index }; + position = { section: sectionIndex, word: index }; events.onWord(position); - timer = window.setTimeout(tick, wordDuration(sectionTokens[index].text) / (events.getRate?.() ?? 1)); + timer = window.setTimeout(tick, (wordDuration(sectionTokens[index].text) / rate) * calibration); }; - timer = window.setTimeout(tick, wordDuration(sectionTokens[word].text) / (events.getRate?.() ?? 1)); + timer = window.setTimeout(tick, (wordDuration(sectionTokens[from].text) / rate) * calibration); } }; utterance.onend = () => { if (generation !== current) return; clearTimer(); + /* Learn this engine's real pace from what just played. */ + if (startedAt > 0) { + const measured = Date.now() - startedAt; + const expected = estimate(); + if (measured > 400 && expected > 200) { + const ratio = Math.min(4, Math.max(0.25, measured / expected)); + calibration = calibration * 0.5 + ratio * 0.5; + } + startedAt = 0; + } /* An utterance that ended without a single boundary means the browser - does not report them: the fallback clock was the right call. */ + does not report them: sentence mode stays. */ if (boundariesWork === null) boundariesWork = false; - if (target.section + 1 >= events.sections.length) { + + const lastWord = sectionTokens.length - 1; + if (to < lastWord) { + /* There is more of this section to speak. With boundaries proven the + rest goes in one utterance (the exact clock handles it); without + them it is the next sentence. */ + const next = boundariesWork === true + ? { from: to + 1, to: lastWord } + : (() => { + const ranges = sentenceRanges(sectionTokens); + const found = ranges.find((range) => range.from > to); + return found ?? { from: to + 1, to: lastWord }; + })(); + window.setTimeout(() => { + if (generation !== current) return; + speakRange(sectionIndex, next.from, next.to); + }, SENTENCE_BEAT_MS); + return; + } + if (sectionIndex + 1 >= events.sections.length) { position = null; finishPass(); events.onEnd(); @@ -232,8 +297,8 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech what makes the boundary pop on several platforms. */ window.setTimeout(() => { if (generation !== current) return; - speakFrom({ section: target.section + 1, word: 0 }); - }, 140); + speakRange(sectionIndex + 1, 0, Number.MAX_SAFE_INTEGER); + }, SENTENCE_BEAT_MS); }; utterance.onerror = (event) => { @@ -269,7 +334,21 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech return { play(from) { - speakFrom(from ?? position ?? { section: 0, word: 0 }); + if (!speechAvailable()) { + events.onError?.("unsupported"); + return; + } + const target = from ?? position ?? { section: 0, word: 0 }; + const sectionTokens = wordsIn(target.section); + if (boundariesWork === true || sectionTokens.length === 0) { + /* Boundaries are exact: the rest of the section in one utterance. */ + speakRange(target.section, target.word, Number.MAX_SAFE_INTEGER); + return; + } + /* No boundaries yet: speak the sentence the listener is in. */ + const ranges = sentenceRanges(sectionTokens); + const range = ranges.find((entry) => entry.to >= target.word) ?? ranges[ranges.length - 1]; + speakRange(target.section, Math.max(range.from, Math.min(target.word, range.to)), range.to); }, pause() { generation += 1; diff --git a/package.json b/package.json index 31e1cd2..1b22433 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "freezmo", - "version": "0.7.9", + "version": "0.8.0", "private": true, "license": "GPL-3.0-only", "scripts": { diff --git a/public/sw.js b/public/sw.js index 76cf046..a639475 100644 --- a/public/sw.js +++ b/public/sw.js @@ -9,7 +9,7 @@ * * Bumping CACHE_VERSION is how a new deploy invalidates old assets. */ -const CACHE_VERSION = "freezmo-v101"; +const CACHE_VERSION = "freezmo-v102"; const SHELL_CACHE = `${CACHE_VERSION}-shell`; const ASSET_CACHE = `${CACHE_VERSION}-assets`; diff --git a/tests/podcast-test.mjs b/tests/podcast-test.mjs index 69f79bf..b18a004 100644 --- a/tests/podcast-test.mjs +++ b/tests/podcast-test.mjs @@ -56,12 +56,28 @@ const LONG_RECORD = { ...EPISODE_RECORD, id: "pod_long", title: "A long walk through the atmosphere", + /* One long sentence per part on purpose: with the sentence clock a section + is spoken in one utterance only while it has no sentence break, which is + exactly the shape the scroll checks need (a boundary can land far away). */ sections: Array.from({ length: 8 }, (_, index) => ({ heading: `Part ${index + 1}`, - text: `Paragraph ${index + 1}. ${"The atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones. ".repeat(3)}`, + text: `Part ${index + 1}, ${"the atmosphere scatters light in every direction, and the shorter wavelengths bounce far more often than the longer ones, ".repeat(3).trim()}`, })), }; +/** Three sentences: the fallback clock must re-anchor at every one. */ +const SENTENCES_RECORD = { + ...EPISODE_RECORD, + id: "pod_sentences", + sections: [ + { + heading: "Daylight", + text: "Look up on a clear day. The sky is blue. Molecules scatter short wavelengths far more than long ones. That is what reaches your eyes.", + }, + { heading: "Why", text: "That scattering is what colours the sky." }, + ], +}; + /** A speech engine that never reports boundaries: the timer clock drives it. */ const WORD_TIMER_SPEECH = ` window.__utterances = []; @@ -566,6 +582,89 @@ try { await context.close(); } +/* 4d. no boundary events: sentences carry the clock ---------------- */ +{ + const { context, page } = await makePage(browser, { width: 390, height: 844, deviceScaleFactor: 1, isMobile: true, hasTouch: true }); + /* An engine that never reports boundaries (Firefox + speech-dispatcher, + iOS) and *finishes* every utterance, so the sentence clock can be watched + sentence by sentence. */ + await page.addInitScript(() => { + window.__utterances = []; + const SENTENCE_MS = 700; + class FakeUtterance { + constructor(text) { this.text = String(text); this.rate = 1; this.lang = "en-US"; this.voice = null; this.onstart = null; this.onend = null; this.onerror = null; this.onboundary = null; } + } + Object.defineProperty(window, "SpeechSynthesisUtterance", { configurable: true, value: FakeUtterance }); + Object.defineProperty(window, "speechSynthesis", { + configurable: true, + value: { + _speaking: false, + get speaking() { return this._speaking; }, + pending: false, + paused: false, + speak(utterance) { + window.__utterances.push(utterance); + this._speaking = true; + const mine = this; + queueMicrotask(() => utterance.onstart && utterance.onstart({ name: "start", charIndex: 0 })); + setTimeout(() => { + mine._speaking = false; + if (utterance.onend) utterance.onend(); + }, SENTENCE_MS); + }, + cancel() { this._speaking = false; }, + resume() {}, + getVoices: () => [{ name: "Daniel", lang: "en-GB", voiceURI: "daniel", localService: true, default: true }], + addEventListener() {}, + removeEventListener() {}, + }, + }); + }); + await seed(page, { decks: [], theme: "dark", extra: { "freezmo.podcasts.v1": JSON.stringify([SENTENCES_RECORD]) } }); + await page.goto(`${APP}/podcast/pod_sentences`, { waitUntil: "networkidle" }); + await page.waitForTimeout(1000); + await page.getByRole("button", { name: "Play the episode" }).click(); + await page.waitForTimeout(500); + + const texts = () => page.evaluate(() => window.__utterances.map((u) => u.text)); + /** Poll until at least `n` utterances have been spoken. */ + const waitFor = async (n, timeout = 4000) => { + const started = Date.now(); + while (Date.now() - started < timeout) { + if ((await page.evaluate(() => window.__utterances.length)) >= n) return true; + await page.waitForTimeout(80); + } + return false; + }; + + check("clock: without boundaries the first utterance is one sentence", (await waitFor(1)) && (await texts())[0] === "Look up on a clear day.", JSON.stringify(await texts())); + check("clock: the next sentence follows on its own", (await waitFor(2)) && (await texts())[1] === "The sky is blue.", JSON.stringify(await texts())); + const active = await page.evaluate(() => document.querySelector("[data-word-active]")?.textContent ?? ""); + check("clock: the highlight is inside the spoken text", (await texts()).some((text) => text.includes(active)), JSON.stringify({ active })); + + /* A boundary event on the sentence that is playing hands the rest of the + section to the exact clock: the next utterance is no longer one sentence. */ + await page.evaluate(() => { + const utterance = window.__utterances[1]; + if (utterance && utterance.onboundary) utterance.onboundary({ name: "word", charIndex: 0 }); + }); + await waitFor(3); + const spoken = await texts(); + check( + "clock: a boundary switches the rest of the section to one utterance", + spoken.length >= 3 && (spoken[2].match(/\./g) ?? []).length >= 2, + JSON.stringify(spoken), + ); + + /* The highlight keeps moving with the voice instead of drifting away. */ + const marks = []; + for (let i = 0; i < 5; i += 1) { + marks.push(await page.evaluate(() => document.querySelector("[data-word-active]")?.textContent ?? "")); + await page.waitForTimeout(350); + } + check("clock: the highlight advances over time", new Set(marks.filter(Boolean)).size >= 2, JSON.stringify(marks)); + await context.close();} + /* 5. server guards ---------------------------------------------- */ const bad = await fetch(`${APP}/api/podcast`, { method: "POST",