freezmo/lib/podcast-speech.ts

"use client";

/**
 * The podcast player's voice: one section at a time, with the spoken word
 * reported back so the transcript can highlight it (the Spotify-lyrics bit).
 *
 * Two clocks drive the highlight. Where the browser fires `onboundary`
 * (desktop Chrome and friends) that is exact — the character index maps onto
 * the word list. Where it does not (iOS Safari, several Android engines) a
 * timer estimates each word from its length and the playback rate, so the
 * highlight still moves at roughly speaking speed. The first boundary seen
 * retires the timer for the rest of the session.
 *
 * Pause/seek are implemented by cancelling and re-speaking from a word: the
 * platform `pause()` is unreliable across browsers, and re-speaking also makes
 * "jump to a chapter" and "resume where you left off" the same operation.
 */

import { speechAvailable } from "./speech";
import type { PodcastSection } from "./schemas";

export type PodcastVoice = { section: number; word: number };

/** Tidy text for the speech engine (see the note where it is used). */
export function normaliseSpoken(raw: string): string {
  return raw
    .replace(/[\u0000-\u001f\u007f-\u009f\u200b-\u200f\ufeff]/g, " ")
    .replace(/…/g, ", ")
    .replace(/\s*—\s*/g, " — ")
    .replace(/\s+/g, " ")
    .trim();
}

export type PodcastSpeechEvents = {
  sections: PodcastSection[];
  locale: string;
  /** Read live, so speed and voice changes never rebuild a speaking engine. */
  getRate?: () => number;
  getVoice?: () => SpeechSynthesisVoice | null;
  /** Global word position changed (word = index inside its section). */
  onWord: (position: PodcastVoice) => void;
  /** Playback actually began (after the engine accepted it). */
  onPlay?: () => void;
  onPause?: () => void;
  onEnd: () => void;
  onError?: (message: string) => void;
};

export type Token = {
  section: number;
  /** Index of the word inside its section. */
  index: number;
  /** Index across the whole episode (the transcript's data-word). */
  global: number;
  start: number;
  end: number;
  text: string;
};

/** Split into words, keeping each one's character span inside its section. */
export function tokenizeSections(sections: PodcastSection[]): Token[] {
  const tokens: Token[] = [];
  sections.forEach((section, sectionIndex) => {
    const text = section.text;
    const pattern = /\S+/g;
    let index = 0;
    let match: RegExpExecArray | null;
    while ((match = pattern.exec(text)) !== null) {
      tokens.push({
        section: sectionIndex,
        index,
        global: tokens.length,
        start: match.index,
        end: match.index + match[0].length,
        text: match[0],
      });
      index += 1;
    }
  });
  return tokens;
}

/** Estimated spoken time of one word at rate 1 (ms). Punctuation is where
 *  these engines actually spend their time (sentences stop, commas slow). */
function wordDuration(text: string): number {
  const letters = Math.max(1, text.replace(/[^\p{L}\p{N}]/gu, "").length);
  let ms = 80 + 42 * letters;
  if (/[.!?…]["')\]]*$/.test(text)) ms += 260;
  else if (/[,;:]["')\]]*$/.test(text)) ms += 130;
  return ms;
}

/** One sentence of a section, in word indices (inclusive). */
type SentenceRange = { from: number; to: number };

/** Sentence spans for a section's words, split on end punctuation. */
function sentenceRanges(sectionTokens: Token[]): SentenceRange[] {
  const ranges: SentenceRange[] = [];
  let from = 0;
  sectionTokens.forEach((token, index) => {
    const endsSentence = /[.!?…]["')\]]*$/.test(token.text);
    const last = index === sectionTokens.length - 1;
    if (endsSentence || last) {
      ranges.push({ from, to: index });
      from = index + 1;
    }
  });
  return ranges.length > 0 ? ranges : [{ from: 0, to: Math.max(0, sectionTokens.length - 1) }];
}

export type PodcastSpeech = {
  play: (from?: PodcastVoice) => void;
  pause: () => void;
  stop: () => void;
  playing: () => boolean;
  position: () => PodcastVoice | null;
  words: Token[];
};

export function createPodcastSpeech(events: PodcastSpeechEvents): PodcastSpeech {
  const tokens = tokenizeSections(events.sections);
  let playing = false;
  let position: PodcastVoice | null = null;
  let boundariesWork: boolean | null = null;
  let timer: number | null = null;
  /** Cleared by cancel(): stops stale timers/events from a previous utterance. */
  let generation = 0;
  /** A cancel in the last moments: some engines drop a speak() that follows
   *  one immediately, so it gets a tick of air. */
  let lastCancelAt = 0;
  /** When the current utterance actually began (calibration). */
  let startedAt = 0;
  /** A breath between sentences: narration needs it, and it keeps the
   *  boundary pops of the system engines down. */
  const SENTENCE_BEAT_MS = 90;
  /**
   * Measured ÷ estimated speech time, learned from every finished utterance.
   * Engines without boundary events are estimated word by word; without this
   * factor the highlight drifts seconds behind on a long section (espeak and
   * speech-dispatcher speak at their own pace, nothing like 150 wpm).
   */
  let calibration = 1;

  const clearTimer = () => {
    if (timer !== null) {
      window.clearTimeout(timer);
      timer = null;
    }
  };

  const finishPass = () => {
    playing = false;
    events.onPause?.();
  };

  const wordsIn = (section: number) => tokens.filter((token) => token.section === section);

  /**
   * Speak one stretch of a section: `to` is the last word index included.
   * Without boundary events the stretches are sentences (so the clock is
   * re-calibrated every few seconds and the highlight cannot drift away);
   * once boundaries are proven, the rest of the section goes in one go.
   */
  const speakRange = (sectionIndex: number, fromWord: number, toWord: number) => {
    const section = events.sections[sectionIndex];
    if (!section) {
      position = null;
      finishPass();
      events.onEnd();
      return;
    }
    const sectionTokens = wordsIn(sectionIndex);
    if (sectionTokens.length === 0) {
      speakRange(sectionIndex + 1, 0, 0);
      return;
    }
    const from = Math.min(Math.max(0, fromWord), sectionTokens.length - 1);
    const to = Math.min(Math.max(from, toWord), sectionTokens.length - 1);
    position = { section: sectionIndex, word: from };

    const base = sectionTokens[from].start;
    const text = normaliseSpoken(section.text.slice(base, sectionTokens[to].end));
    const current = ++generation;
    try {
      const synth = window.speechSynthesis;
      const cancelNeeded = synth.speaking || synth.pending;
      const cancelledRecently = Date.now() - lastCancelAt < 400;
      if (cancelNeeded) {
        synth.cancel();
        lastCancelAt = Date.now();
      }
      /* WebKit leaves the synthesis paused after a cancel (and sometimes
         pauses it on its own): without this resume the next speak() queues
         silently and the player looks dead after one listen. */
      if (synth.paused) synth.resume();

      const utterance = new SpeechSynthesisUtterance(text);
      utterance.lang = events.locale;
      const rate = Math.min(3, Math.max(0.3, events.getRate?.() ?? 1));
      utterance.rate = rate;
      const voice = events.getVoice?.() ?? null;
      if (voice) utterance.voice = voice;

      /** Estimated time of the words from..to at the current calibration. */
      const estimate = () => {
        let ms = 0;
        for (let i = from; i <= to; i += 1) ms += wordDuration(sectionTokens[i].text);
        return ms / rate;
      };

      const indexAt = (charIndex: number) => {
        let found = from;
        for (let i = from; i <= to; i += 1) {
          found = i;
          if (sectionTokens[i].end > base + charIndex) break;
        }
        return found;
      };

      utterance.onboundary = (event) => {
        if (generation !== current) return;
        if (event.name && event.name !== "word") return;
        boundariesWork = true;
        clearTimer();
        const index = indexAt(event.charIndex);
        position = { section: sectionIndex, word: index };
        events.onWord(position);
      };

      utterance.onstart = () => {
        if (generation !== current) return;
        playing = true;
        startedAt = Date.now();
        events.onPlay?.();
        events.onWord(position ?? { section: sectionIndex, word: from });
        /* The fallback clock: word by word, at the pace this engine has been
           measured to speak. Retired for good once a boundary arrives. */
        if (boundariesWork !== true) {
          let index = from;
          const tick = () => {
            if (generation !== current) return;
            index += 1;
            if (index > to) {
              timer = null;
              return;
            }
            position = { section: sectionIndex, word: index };
            events.onWord(position);
            timer = window.setTimeout(tick, (wordDuration(sectionTokens[index].text) / rate) * calibration);
          };
          timer = window.setTimeout(tick, (wordDuration(sectionTokens[from].text) / rate) * calibration);
        }
      };

      utterance.onend = () => {
        if (generation !== current) return;
        clearTimer();
        /* Learn this engine's real pace from what just played. */
        if (startedAt > 0) {
          const measured = Date.now() - startedAt;
          const expected = estimate();
          if (measured > 400 && expected > 200) {
            const ratio = Math.min(4, Math.max(0.25, measured / expected));
            calibration = calibration * 0.5 + ratio * 0.5;
          }
          startedAt = 0;
        }
        /* An utterance that ended without a single boundary means the browser
           does not report them: sentence mode stays. */
        if (boundariesWork === null) boundariesWork = false;

        const lastWord = sectionTokens.length - 1;
        if (to < lastWord) {
          /* There is more of this section to speak. With boundaries proven the
             rest goes in one utterance (the exact clock handles it); without
             them it is the next sentence. */
          const next = boundariesWork === true
            ? { from: to + 1, to: lastWord }
            : (() => {
                const ranges = sentenceRanges(sectionTokens);
                const found = ranges.find((range) => range.from > to);
                return found ?? { from: to + 1, to: lastWord };
              })();
          window.setTimeout(() => {
            if (generation !== current) return;
            speakRange(sectionIndex, next.from, next.to);
          }, SENTENCE_BEAT_MS);
          return;
        }
        if (sectionIndex + 1 >= events.sections.length) {
          position = null;
          finishPass();
          events.onEnd();
          return;
        }
        /* A short beat between parts: speaking into the engine's tail is
           what makes the boundary pop on several platforms. */
        window.setTimeout(() => {
          if (generation !== current) return;
          speakRange(sectionIndex + 1, 0, Number.MAX_SAFE_INTEGER);
        }, SENTENCE_BEAT_MS);
      };

      utterance.onerror = (event) => {
        if (generation !== current) return;
        clearTimer();
        playing = false;
        /* "interrupted"/"canceled" are our own stops, not failures. */
        const reason = (event as SpeechSynthesisErrorEvent).error;
        if (reason !== "interrupted" && reason !== "canceled") {
          events.onError?.(String(reason ?? "error"));
          events.onPause?.();
        }
      };

      const sayIt = () => {
        if (generation !== current) return;
        try {
          window.speechSynthesis.speak(utterance);
        } catch {
          playing = false;
          events.onError?.("error");
        }
      };
      /* A speak() that follows a cancel immediately is dropped by several
         engines (Android Chrome, WebKit); a short beat avoids it. */
      if (cancelNeeded || cancelledRecently) window.setTimeout(sayIt, 90);
      else sayIt();
    } catch {
      playing = false;
      events.onError?.("error");
    }
  };

  return {
    play(from) {
      if (!speechAvailable()) {
        events.onError?.("unsupported");
        return;
      }
      const target = from ?? position ?? { section: 0, word: 0 };
      const sectionTokens = wordsIn(target.section);
      if (boundariesWork === true || sectionTokens.length === 0) {
        /* Boundaries are exact: the rest of the section in one utterance. */
        speakRange(target.section, target.word, Number.MAX_SAFE_INTEGER);
        return;
      }
      /* No boundaries yet: speak the sentence the listener is in. */
      const ranges = sentenceRanges(sectionTokens);
      const range = ranges.find((entry) => entry.to >= target.word) ?? ranges[ranges.length - 1];
      speakRange(target.section, Math.max(range.from, Math.min(target.word, range.to)), range.to);
    },
    pause() {
      generation += 1;
      clearTimer();
      try {
        const synth = window.speechSynthesis;
        if (synth.speaking || synth.pending) {
          synth.cancel();
          lastCancelAt = Date.now();
        }
        /* Cancel leaves some engines (WebKit especially) paused; clear that
           here so the next play is not a silent no-op. */
        if (synth.paused) synth.resume();
      } catch {
        /* already gone */
      }
      playing = false;
      events.onPause?.();
    },
    stop() {
      this.pause();
      position = null;
    },
    playing: () => playing,
    position: () => position,
    words: tokens,
  };
}