dvorakmortonxd/freezmo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Podcast speech: the highlight clock learns the engine's pace

Browse source 
"The tts dispatcher thing and the text highlight is completely out of sync on
desktop." Firefox + speech-dispatcher (like iOS) reports no onboundary events,
so the highlight rode a pure estimate - and a 130-word section drifts seconds
away from an engine that speaks at its own pace.

- On platforms without boundaries the system engine now speaks one sentence
  at a time, and every finished utterance re-anchors the clock: measured
  duration vs estimate, smoothed (half old, half new, clamped to 0.25-4x).
  Drift cannot accumulate past a sentence.
- The per-word estimate is punctuation-aware (full stop ~260ms, comma ~130ms).
- Once a boundary does arrive (Chrome), the rest of the section goes back to
  a single utterance with the exact clock - the accurate path is unchanged.
- When a boundary lands mid-section, the remainder is spoken as one utterance
  (it used to skip to the next section).

podcast-test's engine stub finishes utterances now and pins the whole thing:
one sentence first, the next following on its own, the highlight inside the
spoken text, a boundary collapsing the remainder, the highlight advancing.
53 checks. Version 0.7.10, SW v102.
This commit is contained in:
Martin Dvořák 2026-10-16 21:45:18 +00:00
commit 8c532ea6e6
5 changed files with 245 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

171
lib/podcast-speech.ts
View file

@ -80,9 +80,32 @@ export function tokenizeSections(sections: PodcastSection[]): Token[] {
return tokens;
}
/** Estimated spoken time of one word at rate 1 (ms). */
/** Estimated spoken time of one word at rate 1 (ms). Punctuation is where
* these engines actually spend their time (sentences stop, commas slow). */
function wordDuration(text: string): number {
return 95 + 42 * Math.max(1, text.replace(/[^\p{L}\p{N}]/gu, "").length);
const letters = Math.max(1, text.replace(/[^\p{L}\p{N}]/gu, "").length);
let ms = 80 + 42 * letters;
if (/[.!?…]["')\]]*$/.test(text)) ms += 260;
else if (/[,;:]["')\]]*$/.test(text)) ms += 130;
return ms;
}
/** One sentence of a section, in word indices (inclusive). */
type SentenceRange = { from: number; to: number };
/** Sentence spans for a section's words, split on end punctuation. */
function sentenceRanges(sectionTokens: Token[]): SentenceRange[] {
const ranges: SentenceRange[] = [];
let from = 0;
sectionTokens.forEach((token, index) => {
const endsSentence = /[.!?…]["')\]]*$/.test(token.text);
const last = index === sectionTokens.length - 1;
if (endsSentence || last) {
ranges.push({ from, to: index });
from = index + 1;
}
});
return ranges.length > 0 ? ranges : [{ from: 0, to: Math.max(0, sectionTokens.length - 1) }];
}
export type PodcastSpeech = {
@ -105,6 +128,18 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
/** A cancel in the last moments: some engines drop a speak() that follows
* one immediately, so it gets a tick of air. */
let lastCancelAt = 0;
/** When the current utterance actually began (calibration). */
let startedAt = 0;
/** A breath between sentences: narration needs it, and it keeps the
* boundary pops of the system engines down. */
const SENTENCE_BEAT_MS = 90;
/**
* Measured ÷ estimated speech time, learned from every finished utterance.
* Engines without boundary events are estimated word by word; without this
* factor the highlight drifts seconds behind on a long section (espeak and
* speech-dispatcher speak at their own pace, nothing like 150 wpm).
*/
let calibration = 1;
const clearTimer = () => {
if (timer !== null) {
@ -120,35 +155,33 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
const wordsIn = (section: number) => tokens.filter((token) => token.section === section);
/** Speak the section from `word` on. Empty word list → the next section. */
const speakFrom = (target: PodcastVoice) => {
if (!speechAvailable()) {
events.onError?.("unsupported");
return;
}
const section = events.sections[target.section];
/**
* Speak one stretch of a section: `to` is the last word index included.
* Without boundary events the stretches are sentences (so the clock is
* re-calibrated every few seconds and the highlight cannot drift away);
* once boundaries are proven, the rest of the section goes in one go.
*/
const speakRange = (sectionIndex: number, fromWord: number, toWord: number) => {
const section = events.sections[sectionIndex];
if (!section) {
position = null;
finishPass();
events.onEnd();
return;
}
const sectionTokens = wordsIn(target.section);
const sectionTokens = wordsIn(sectionIndex);
if (sectionTokens.length === 0) {
speakFrom({ section: target.section + 1, word: 0 });
speakRange(sectionIndex + 1, 0, 0);
return;
}
const word = Math.min(Math.max(0, target.word), sectionTokens.length - 1);
position = { section: target.section, word };
const from = Math.min(Math.max(0, fromWord), sectionTokens.length - 1);
const to = Math.min(Math.max(from, toWord), sectionTokens.length - 1);
position = { section: sectionIndex, word: from };
const from = sectionTokens[word].start;
/* Spoken text hygiene: engines click on control characters, stray
double spaces and unicode ellipses, and some pause oddly around them. */
const text = normaliseSpoken(section.text.slice(from));
const base = sectionTokens[from].start;
const text = normaliseSpoken(section.text.slice(base, sectionTokens[to].end));
const current = ++generation;
try {
/* Only cancel when something is actually pending: an idle cancel makes
several engines pop. */
const synth = window.speechSynthesis;
const cancelNeeded = synth.speaking || synth.pending;
const cancelledRecently = Date.now() - lastCancelAt < 400;
@ -160,22 +193,26 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
pauses it on its own): without this resume the next speak() queues
silently and the player looks dead after one listen. */
if (synth.paused) synth.resume();
const utterance = new SpeechSynthesisUtterance(text);
utterance.lang = events.locale;
utterance.rate = Math.min(3, Math.max(0.3, events.getRate?.() ?? 1));
const rate = Math.min(3, Math.max(0.3, events.getRate?.() ?? 1));
utterance.rate = rate;
const voice = events.getVoice?.() ?? null;
if (voice) utterance.voice = voice;
const offsetFor = (charIndex: number) => {
/* First token whose span ends after the reported character. */
let found = word;
for (let i = word; i < sectionTokens.length; i += 1) {
const token = sectionTokens[i];
if (token.end > from + charIndex) {
found = i;
break;
}
/** Estimated time of the words from..to at the current calibration. */
const estimate = () => {
let ms = 0;
for (let i = from; i <= to; i += 1) ms += wordDuration(sectionTokens[i].text);
return ms / rate;
};
const indexAt = (charIndex: number) => {
let found = from;
for (let i = from; i <= to; i += 1) {
found = i;
if (sectionTokens[i].end > base + charIndex) break;
}
return found;
};
@ -185,44 +222,72 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
if (event.name && event.name !== "word") return;
boundariesWork = true;
clearTimer();
const index = offsetFor(event.charIndex);
position = { section: target.section, word: index };
const index = indexAt(event.charIndex);
position = { section: sectionIndex, word: index };
events.onWord(position);
};
utterance.onstart = () => {
if (generation !== current) return;
playing = true;
startedAt = Date.now();
events.onPlay?.();
events.onWord(position ?? { section: target.section, word });
/* The fallback clock: only while this browser has never reported a
* boundary. Each tick schedules the next word by its length. */
events.onWord(position ?? { section: sectionIndex, word: from });
/* The fallback clock: word by word, at the pace this engine has been
measured to speak. Retired for good once a boundary arrives. */
if (boundariesWork !== true) {
let index = word;
let index = from;
const tick = () => {
if (generation !== current) return;
index += 1;
if (index >= sectionTokens.length) {
/* Let onend take over; a long word list can outlast the audio
* by a hair, and stopping the tick avoids double-advancing. */
if (index > to) {
timer = null;
return;
}
position = { section: target.section, word: index };
position = { section: sectionIndex, word: index };
events.onWord(position);
timer = window.setTimeout(tick, wordDuration(sectionTokens[index].text) / (events.getRate?.() ?? 1));
timer = window.setTimeout(tick, (wordDuration(sectionTokens[index].text) / rate) * calibration);
};
timer = window.setTimeout(tick, wordDuration(sectionTokens[word].text) / (events.getRate?.() ?? 1));
timer = window.setTimeout(tick, (wordDuration(sectionTokens[from].text) / rate) * calibration);
}
};
utterance.onend = () => {
if (generation !== current) return;
clearTimer();
/* Learn this engine's real pace from what just played. */
if (startedAt > 0) {
const measured = Date.now() - startedAt;
const expected = estimate();
if (measured > 400 && expected > 200) {
const ratio = Math.min(4, Math.max(0.25, measured / expected));
calibration = calibration * 0.5 + ratio * 0.5;
}
startedAt = 0;
}
/* An utterance that ended without a single boundary means the browser
does not report them: the fallback clock was the right call. */
does not report them: sentence mode stays. */
if (boundariesWork === null) boundariesWork = false;
if (target.section + 1 >= events.sections.length) {
const lastWord = sectionTokens.length - 1;
if (to < lastWord) {
/* There is more of this section to speak. With boundaries proven the
rest goes in one utterance (the exact clock handles it); without
them it is the next sentence. */
const next = boundariesWork === true
? { from: to + 1, to: lastWord }
: (() => {
const ranges = sentenceRanges(sectionTokens);
const found = ranges.find((range) => range.from > to);
return found ?? { from: to + 1, to: lastWord };
})();
window.setTimeout(() => {
if (generation !== current) return;
speakRange(sectionIndex, next.from, next.to);
}, SENTENCE_BEAT_MS);
return;
}
if (sectionIndex + 1 >= events.sections.length) {
position = null;
finishPass();
events.onEnd();
@ -232,8 +297,8 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
what makes the boundary pop on several platforms. */
window.setTimeout(() => {
if (generation !== current) return;
speakFrom({ section: target.section + 1, word: 0 });
}, 140);
speakRange(sectionIndex + 1, 0, Number.MAX_SAFE_INTEGER);
}, SENTENCE_BEAT_MS);
};
utterance.onerror = (event) => {
@ -269,7 +334,21 @@ export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech
return {
play(from) {
speakFrom(from ?? position ?? { section: 0, word: 0 });
if (!speechAvailable()) {
events.onError?.("unsupported");
return;
}
const target = from ?? position ?? { section: 0, word: 0 };
const sectionTokens = wordsIn(target.section);
if (boundariesWork === true || sectionTokens.length === 0) {
/* Boundaries are exact: the rest of the section in one utterance. */
speakRange(target.section, target.word, Number.MAX_SAFE_INTEGER);
return;
}
/* No boundaries yet: speak the sentence the listener is in. */
const ranges = sentenceRanges(sectionTokens);
const range = ranges.find((entry) => entry.to >= target.word) ?? ranges[ranges.length - 1];
speakRange(target.section, Math.max(range.from, Math.min(target.word, range.to)), range.to);
},
pause() {
generation += 1;