freezmo/lib/language.ts

/**
 * Cheap, dependency-free language checks used to protect the notes' language.
 *
 * The vision model is instructed never to translate, but a small model can
 * still slip into English on hard handwriting. When that happens we re-run the
 * transcription through the (much stronger, multilingual) text model.
 *
 * These helpers are heuristic on purpose: they are only used to decide whether
 * one extra repair call is worth it, and a false positive costs a single call
 * while a false negative would hand the student an English deck.
 */

export type Script =
  | "latin"
  | "cyrillic"
  | "greek"
  | "han"
  | "kana"
  | "hangul"
  | "arabic"
  | "hebrew"
  | "devanagari"
  | "thai"
  | "unknown";

const SCRIPT_RANGES: Array<{ script: Script; test: RegExp }> = [
  { script: "cyrillic", test: /[\u0400-\u04FF]/ },
  { script: "greek", test: /[\u0370-\u03FF\u1F00-\u1FFF]/ },
  { script: "hangul", test: /[\uAC00-\uD7AF\u1100-\u11FF]/ },
  { script: "kana", test: /[\u3040-\u30FF]/ },
  { script: "han", test: /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF]/ },
  { script: "arabic", test: /[\u0600-\u06FF\u0750-\u077F]/ },
  { script: "hebrew", test: /[\u0590-\u05FF]/ },
  { script: "devanagari", test: /[\u0900-\u097F]/ },
  { script: "thai", test: /[\u0E00-\u0E7F]/ },
  { script: "latin", test: /[A-Za-z\u00C0-\u024F]/ },
];

/** Language names/codes we know, mapped to the script they are written in. */
const LANGUAGE_SCRIPTS: Record<string, Script> = {
  english: "latin", en: "latin",
  czech: "latin", cs: "latin",
  slovak: "latin", sk: "latin",
  polish: "latin", pl: "latin",
  german: "latin", de: "latin",
  french: "latin", fr: "latin",
  spanish: "latin", es: "latin",
  italian: "latin", it: "latin",
  portuguese: "latin", pt: "latin",
  dutch: "latin", nl: "latin",
  swedish: "latin", sv: "latin",
  norwegian: "latin", no: "latin",
  danish: "latin", da: "latin",
  finnish: "latin", fi: "latin",
  hungarian: "latin", hu: "latin",
  romanian: "latin", ro: "latin",
  turkish: "latin", tr: "latin",
  croatian: "latin", hr: "latin",
  slovenian: "latin", sl: "latin",
  serbian: "cyrillic", sr: "cyrillic",
  russian: "cyrillic", ru: "cyrillic",
  ukrainian: "cyrillic", uk: "cyrillic",
  bulgarian: "cyrillic", bg: "cyrillic",
  belarusian: "cyrillic", be: "cyrillic",
  macedonian: "cyrillic", mk: "cyrillic",
  greek: "greek", el: "greek",
  chinese: "han", mandarin: "han", cantonese: "han", zh: "han",
  japanese: "kana", ja: "kana",
  korean: "hangul", ko: "hangul",
  arabic: "arabic", ar: "arabic",
  persian: "arabic", farsi: "arabic", urdu: "arabic",
  hebrew: "hebrew", he: "hebrew",
  hindi: "devanagari", marathi: "devanagari", nepali: "devanagari",
  thai: "thai",
};

/** Characters that practically always appear in real text of these languages. */
const DISTINCTIVE_CHARS: Record<string, RegExp> = {
  czech: /[ěščřžýáíéúůťďňĚŠČŘŽÝÁÍÉÚŮŤĎŇ]/,
  slovak: /[ľščťžýáíéúôäňĺ༊ȍŽÝÁÍÉ]/,
  polish: /[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]/,
  hungarian: /[őűáéíóöüŐŰÁÉÍÓÖÜ]/,
  romanian: /[ăâîșțĂÂÎȘȚ]/,
  turkish: /[ğışöüçĞİŞÖÜÇ]/,
  croatian: /[čćđšžČĆĐŠŽ]/,
  slovenian: /[蚞ȊŽ]/,
  german: /[äöüßÄÖÜ]/,
  french: /[éèêëàâçîïôûùüÿœæÉÈÊËÀÂÇÎÏÔÛÙŸŒÆ]/,
  spanish: /[áéíóúñü¡¿ÁÉÍÓÚÑÜ]/,
  portuguese: /[áâãàçéêíóôõúüÁÂÃÀÇÉÊÍÓÔÕÚ]/,
  italian: /[àèéìòùÀÈÉÌÒÙ]/,
  swedish: /[åäöÅÄÖ]/,
  norwegian: /[æøåÆØÅ]/,
  danish: /[æøåÆØÅ]/,
  finnish: /[äöåÄÖÅ]/,
  dutch: /[ëïéèöüËÏÉÈÖÜ]/,
  vietnamese: /[ăâđêôơưàảãáạằẳẵắặầẩẫấậèẻẽéẹềểễếệ]/,
  latvian: /[āčēģīķļņšūž]/,
  lithuanian: /[ąčęėįšųūž]/,
  estonian: /[õäöüšžÕÄÖÜŠŽ]/,
  icelandic: /[áéíóúýþæöðÁÉÍÓÚÝÞÆÖÐ]/,
};

const ENGLISH_STOPWORDS =
  /\b(the|and|that|with|from|this|which|are|was|were|have|has|for|not|but|you|your|they|their|into|than|then)\b/gi;

/** Normalise "Czech", "cs-CZ", "česky" to a lookup key. */
function languageKey(language: string | undefined): string {
  if (!language) return "";
  return language.trim().toLowerCase().replace(/[_.]/g, "-").split("-")[0].replace(/[^a-z]/g, "");
}

export function expectedScript(language: string | undefined): Script | null {
  const key = languageKey(language);
  if (!key) return null;
  return LANGUAGE_SCRIPTS[key] ?? null;
}

/** The script that most non-whitespace characters belong to. */
export function dominantScript(text: string): Script {
  const counts = new Map<Script, number>();
  for (const { script, test } of SCRIPT_RANGES) {
    const matches = text.match(new RegExp(test.source, "g"));
    if (matches) counts.set(script, matches.length);
  }
  let best: Script = "unknown";
  let bestCount = 0;
  for (const [script, count] of counts) {
    if (count > bestCount) {
      best = script;
      bestCount = count;
    }
  }
  return best;
}

function looksEnglish(text: string, words: number): boolean {
  const hits = (text.match(ENGLISH_STOPWORDS) ?? []).length;
  return words >= 40 && hits >= Math.max(4, Math.round(words / 30));
}

export type LanguageCheck = { suspicious: boolean; reason?: string };

/**
 * Decides whether the transcription probably got translated away from the
 * language the vision model reported.
 */
export function needsLanguageRepair(input: { language?: string; text: string }): LanguageCheck {
  const { language, text } = input;
  const key = languageKey(language);
  const words = text.split(/\s+/).filter(Boolean).length;
  if (words < 40) return { suspicious: false }; // too short to judge reliably
  if (key === "english" || key === "en") return { suspicious: false };

  const expected = expectedScript(language);
  const actual = dominantScript(text);

  if (expected && expected !== "unknown" && actual !== "unknown" && expected !== actual) {
    return {
      suspicious: true,
      reason: `notes were declared as ${language} (${expected}) but are written in ${actual}`,
    };
  }

  const distinctive = DISTINCTIVE_CHARS[key];
  if (expected === "latin" && distinctive && !distinctive.test(text) && looksEnglish(text, words)) {
    return {
      suspicious: true,
      reason: `notes were declared as ${language} but read like English`,
    };
  }

  return { suspicious: false };
}