/** * Cheap, dependency-free language checks used to protect the notes' language. * * The vision model is instructed never to translate, but a small model can * still slip into English on hard handwriting. When that happens we re-run the * transcription through the (much stronger, multilingual) text model. * * These helpers are heuristic on purpose: they are only used to decide whether * one extra repair call is worth it, and a false positive costs a single call * while a false negative would hand the student an English deck. */ export type Script = | "latin" | "cyrillic" | "greek" | "han" | "kana" | "hangul" | "arabic" | "hebrew" | "devanagari" | "thai" | "unknown"; const SCRIPT_RANGES: Array<{ script: Script; test: RegExp }> = [ { script: "cyrillic", test: /[\u0400-\u04FF]/ }, { script: "greek", test: /[\u0370-\u03FF\u1F00-\u1FFF]/ }, { script: "hangul", test: /[\uAC00-\uD7AF\u1100-\u11FF]/ }, { script: "kana", test: /[\u3040-\u30FF]/ }, { script: "han", test: /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF]/ }, { script: "arabic", test: /[\u0600-\u06FF\u0750-\u077F]/ }, { script: "hebrew", test: /[\u0590-\u05FF]/ }, { script: "devanagari", test: /[\u0900-\u097F]/ }, { script: "thai", test: /[\u0E00-\u0E7F]/ }, { script: "latin", test: /[A-Za-z\u00C0-\u024F]/ }, ]; /** Language names/codes we know, mapped to the script they are written in. */ const LANGUAGE_SCRIPTS: Record = { english: "latin", en: "latin", czech: "latin", cs: "latin", slovak: "latin", sk: "latin", polish: "latin", pl: "latin", german: "latin", de: "latin", french: "latin", fr: "latin", spanish: "latin", es: "latin", italian: "latin", it: "latin", portuguese: "latin", pt: "latin", dutch: "latin", nl: "latin", swedish: "latin", sv: "latin", norwegian: "latin", no: "latin", danish: "latin", da: "latin", finnish: "latin", fi: "latin", hungarian: "latin", hu: "latin", romanian: "latin", ro: "latin", turkish: "latin", tr: "latin", croatian: "latin", hr: "latin", slovenian: "latin", sl: "latin", serbian: "cyrillic", sr: "cyrillic", russian: "cyrillic", ru: "cyrillic", ukrainian: "cyrillic", uk: "cyrillic", bulgarian: "cyrillic", bg: "cyrillic", belarusian: "cyrillic", be: "cyrillic", macedonian: "cyrillic", mk: "cyrillic", greek: "greek", el: "greek", chinese: "han", mandarin: "han", cantonese: "han", zh: "han", japanese: "kana", ja: "kana", korean: "hangul", ko: "hangul", arabic: "arabic", ar: "arabic", persian: "arabic", farsi: "arabic", urdu: "arabic", hebrew: "hebrew", he: "hebrew", hindi: "devanagari", marathi: "devanagari", nepali: "devanagari", thai: "thai", }; /** Characters that practically always appear in real text of these languages. */ const DISTINCTIVE_CHARS: Record = { czech: /[ěščřžýáíéúůťďňĚŠČŘŽÝÁÍÉÚŮŤĎŇ]/, slovak: /[ľščťžýáíéúôäňĺ༊ȍŽÝÁÍÉ]/, polish: /[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]/, hungarian: /[őűáéíóöüŐŰÁÉÍÓÖÜ]/, romanian: /[ăâîșțĂÂÎȘȚ]/, turkish: /[ğışöüçĞİŞÖÜÇ]/, croatian: /[čćđšžČĆĐŠŽ]/, slovenian: /[蚞ȊŽ]/, german: /[äöüßÄÖÜ]/, french: /[éèêëàâçîïôûùüÿœæÉÈÊËÀÂÇÎÏÔÛÙŸŒÆ]/, spanish: /[áéíóúñü¡¿ÁÉÍÓÚÑÜ]/, portuguese: /[áâãàçéêíóôõúüÁÂÃÀÇÉÊÍÓÔÕÚ]/, italian: /[àèéìòùÀÈÉÌÒÙ]/, swedish: /[åäöÅÄÖ]/, norwegian: /[æøåÆØÅ]/, danish: /[æøåÆØÅ]/, finnish: /[äöåÄÖÅ]/, dutch: /[ëïéèöüËÏÉÈÖÜ]/, vietnamese: /[ăâđêôơưàảãáạằẳẵắặầẩẫấậèẻẽéẹềểễếệ]/, latvian: /[āčēģīķļņšūž]/, lithuanian: /[ąčęėįšųūž]/, estonian: /[õäöüšžÕÄÖÜŠŽ]/, icelandic: /[áéíóúýþæöðÁÉÍÓÚÝÞÆÖÐ]/, }; const ENGLISH_STOPWORDS = /\b(the|and|that|with|from|this|which|are|was|were|have|has|for|not|but|you|your|they|their|into|than|then)\b/gi; /** Normalise "Czech", "cs-CZ", "česky" to a lookup key. */ function languageKey(language: string | undefined): string { if (!language) return ""; return language.trim().toLowerCase().replace(/[_.]/g, "-").split("-")[0].replace(/[^a-z]/g, ""); } export function expectedScript(language: string | undefined): Script | null { const key = languageKey(language); if (!key) return null; return LANGUAGE_SCRIPTS[key] ?? null; } /** The script that most non-whitespace characters belong to. */ export function dominantScript(text: string): Script { const counts = new Map(); for (const { script, test } of SCRIPT_RANGES) { const matches = text.match(new RegExp(test.source, "g")); if (matches) counts.set(script, matches.length); } let best: Script = "unknown"; let bestCount = 0; for (const [script, count] of counts) { if (count > bestCount) { best = script; bestCount = count; } } return best; } function looksEnglish(text: string, words: number): boolean { const hits = (text.match(ENGLISH_STOPWORDS) ?? []).length; return words >= 40 && hits >= Math.max(4, Math.round(words / 30)); } export type LanguageCheck = { suspicious: boolean; reason?: string }; /** * Decides whether the transcription probably got translated away from the * language the vision model reported. */ export function needsLanguageRepair(input: { language?: string; text: string }): LanguageCheck { const { language, text } = input; const key = languageKey(language); const words = text.split(/\s+/).filter(Boolean).length; if (words < 40) return { suspicious: false }; // too short to judge reliably if (key === "english" || key === "en") return { suspicious: false }; const expected = expectedScript(language); const actual = dominantScript(text); if (expected && expected !== "unknown" && actual !== "unknown" && expected !== actual) { return { suspicious: true, reason: `notes were declared as ${language} (${expected}) but are written in ${actual}`, }; } const distinctive = DISTINCTIVE_CHARS[key]; if (expected === "latin" && distinctive && !distinctive.test(text) && looksEnglish(text, words)) { return { suspicious: true, reason: `notes were declared as ${language} but read like English`, }; } return { suspicious: false }; }