/** * Server-side AI pipeline: vision extraction, activity generation and * short-answer grading. Every step asks for JSON, validates with zod and * retries once with the validation error fed back to the model. */ import { OllamaError, ollamaChat, type OllamaMessage } from "./ollama"; import { needsLanguageRepair } from "./language"; import { buildGenerationUserPrompt, buildGradingUserPrompt, buildRepairPrompt, GENERATION_SYSTEM_PROMPT, GRADING_SYSTEM_PROMPT, VISION_SYSTEM_PROMPT, VISION_USER_INSTRUCTION, } from "./prompts"; import { extractedNotesSchema, generatedActivitySchema, gradeResultSchema, type Difficulty, type ExtractedNotes, type GeneratedActivity, type GradeResult, } from "./schemas"; import { parseJsonLoose } from "./utils"; import type { OllamaAuth } from "./config"; export type PipelineStage = "vision" | "generation" | "done"; type JsonStepOptions = { model: string; auth: OllamaAuth; system: string; user: string; images?: string[]; /** Validates and can also "tighten" the parsed value. */ validate: (value: unknown) => { ok: true; value: T } | { ok: false; error: string }; shapeHint: string; label: string; signal?: AbortSignal; temperature?: number; }; type JsonStepResult = { value: T; /** true when the first attempt failed and the repair round was used. */ repaired: boolean; attempts: number; }; async function runJsonStep({ model, auth, system, user, images, validate, shapeHint, label, signal, temperature, }: JsonStepOptions): Promise> { const messages: OllamaMessage[] = [ { role: "system", content: system }, { role: "user", content: user, ...(images && images.length ? { images } : {}) }, ]; let lastRaw = ""; let lastError = ""; for (let attempt = 1; attempt <= 2; attempt++) { const result = await ollamaChat({ model, auth, messages, json: true, signal, temperature: temperature ?? 0.4, }); lastRaw = result.content; let parsed: unknown; try { parsed = parseJsonLoose(result.content); } catch (error) { lastError = `Response was not valid JSON: ${error instanceof Error ? error.message : String(error)}`; if (attempt === 1) { messages.push({ role: "assistant", content: result.content }); messages.push({ role: "user", content: buildRepairPrompt(lastRaw, lastError, shapeHint), }); continue; } throw new OllamaError("bad_response", `${label} failed: ${lastError}`); } const checked = validate(parsed); if (checked.ok) { return { value: checked.value, repaired: attempt > 1, attempts: attempt }; } lastError = checked.error; if (attempt === 1) { messages.push({ role: "assistant", content: result.content }); messages.push({ role: "user", content: buildRepairPrompt(lastRaw, lastError, shapeHint), }); } } throw new OllamaError( "bad_response", `${label} returned data that did not match the expected format: ${lastError}`, ); } /* ------------------------------------------------------------------ */ /* Step 1 - vision */ /* ------------------------------------------------------------------ */ const VISION_SHAPE_HINT = `{"title":"...","subject":"...","language":"English name of the notes' language","summary":"...","notes":"plain text transcription","keyConcepts":["..."],"definitions":[{"term":"...","definition":"..."}],"formulas":[{"name":"...","formula":"..."}],"facts":["..."]}`; /** * Read the uploaded images (and any pasted text) into structured notes. * Any pasted text is passed through verbatim as additional note content. * * A small vision model can accidentally translate on hard handwriting, so the * result is checked and — when it clearly came back in another language — * restored by the (stronger, multilingual) text model. */ export async function extractNotes(input: { model: string; /** Model used to restore the language if the transcription was translated. */ repairModel: string; auth: OllamaAuth; images: string[]; text?: string; signal?: AbortSignal; onProgress?: (message: string) => void; }): Promise<{ notes: ExtractedNotes; repaired: boolean; languageRepaired: boolean }> { const user = input.text?.trim() ? `${VISION_USER_INSTRUCTION}\n\nADDITIONAL TEXT THE STUDENT PASTED (treat as notes too):\n${input.text.slice(0, 12_000)}` : VISION_USER_INSTRUCTION; const { value, repaired } = await runJsonStep({ model: input.model, auth: input.auth, system: VISION_SYSTEM_PROMPT, user, images: input.images, validate: (parsed) => { const result = extractedNotesSchema.safeParse(parsed); if (!result.success) return { ok: false, error: summariseZodError(result.error) }; if (!result.data.notes.trim()) return { ok: false, error: "The notes field is empty." }; return { ok: true, value: result.data }; }, shapeHint: VISION_SHAPE_HINT, label: "Reading your notes", signal: input.signal, temperature: 0.2, }); const check = needsLanguageRepair({ language: value.language, text: value.notes }); if (!check.suspicious) return { notes: value, repaired, languageRepaired: false }; console.warn(`[freezmo] restoring the notes' language: ${check.reason}`); input.onProgress?.(`Restoring the notes' language (${value.language ?? "original"})…`); try { const restored = await restoreLanguage({ model: input.repairModel, auth: input.auth, language: value.language ?? "the original language of the notes", extraction: value, signal: input.signal, }); return { notes: restored, repaired, languageRepaired: true }; } catch (error) { // The guard is best-effort: a failure must never break generation. console.warn("[freezmo] language restore failed, using the original transcription", error); return { notes: value, repaired, languageRepaired: false }; } } const RESTORE_SYSTEM_PROMPT = `You are Freezmo's language restorer. A student's study notes were wrongly translated out of their original language (usually into English) before reaching you. Your job: rewrite the JSON you receive back into the ORIGINAL language, keeping every fact, number, formula, name and definition exactly as it is. Rules: - Translate ONLY what was wrongly translated. Do not add, remove, summarise or explain anything. - Keep the same JSON keys and the same array structure. Keep the same number of items in every array. - Keep technical terms, units and formulas as the original notes used them. - Keep proper nouns as they are unless the original language would inflect them. - Output JSON only, no markdown fences, no commentary.`; /** * Ask the text model to put a translated transcription back into the language * the vision model reported. Returns the original value if the repair is * unusable. */ export async function restoreLanguage(input: { model: string; auth: OllamaAuth; language: string; extraction: ExtractedNotes; signal?: AbortSignal; }): Promise { const value = await runJsonStep({ model: input.model, auth: input.auth, system: RESTORE_SYSTEM_PROMPT, user: [ `ORIGINAL LANGUAGE: ${input.language}`, "", "JSON TO RESTORE:", JSON.stringify(input.extraction).slice(0, 40_000), "", "Return the same JSON with the text fields written in the original language.", ].join("\n"), validate: (parsed) => { const result = extractedNotesSchema.safeParse(parsed); if (!result.success) return { ok: false, error: summariseZodError(result.error) }; if (!result.data.notes.trim()) return { ok: false, error: "The notes field is empty." }; return { ok: true, value: result.data }; }, shapeHint: VISION_SHAPE_HINT, label: "Restoring the notes' language", signal: input.signal, temperature: 0.1, }); return value.value; } /* ------------------------------------------------------------------ */ /* Step 2 - generation */ /* ------------------------------------------------------------------ */ const GENERATION_SHAPE_HINT = `{"title":"...","subject":"...","language":"English name of the language used","summary":"...","activities":[{"type":"multiple_choice","topic":"...","question":"...","options":["...","...","...","..."],"correctIndex":0,"explanation":"..."}, ...]}`; const VALID_TYPES = [ "flashcard", "multiple_choice", "true_false", "fill_blank", "matching", "ordering", "short_answer", ] as const; /** * Be forgiving: validate each activity on its own and keep the good ones. * Only if too few survive does the step get retried. */ function validateDeck(parsed: unknown, minimum: number, requested: number) { if (!parsed || typeof parsed !== "object") { return { ok: false as const, error: "Expected a JSON object at the top level." }; } const record = parsed as Record; const title = typeof record.title === "string" && record.title.trim() ? record.title.trim() : "Untitled deck"; const subject = typeof record.subject === "string" ? record.subject.trim() : undefined; const summary = typeof record.summary === "string" ? record.summary.trim() : undefined; const language = typeof record.language === "string" && record.language.trim() ? record.language.trim() : undefined; const rawActivities = Array.isArray(record.activities) ? record.activities : Array.isArray(record.items) ? record.items : null; if (!rawActivities) { return { ok: false as const, error: `Missing "activities" array. Expected an array of ${requested} activity objects.` }; } const activities: GeneratedActivity[] = []; const rejected: string[] = []; rawActivities.slice(0, 24).forEach((item, index) => { const result = generatedActivitySchema.safeParse(item); if (result.success) { activities.push(sanitiseActivity(result.data)); } else { const type = (item as { type?: unknown } | null)?.type; rejected.push(`#${index + 1} (${typeof type === "string" ? type : "no type"}): ${summariseZodError(result.error)}`); } }); if (activities.length < minimum) { return { ok: false as const, error: `Only ${activities.length} valid activities were found, at least ${minimum} are needed. Rejected: ${rejected.slice(0, 3).join("; ") || "none"}. Valid types are: ${VALID_TYPES.join(", ")}.`, }; } return { ok: true as const, value: { title, subject, summary, language, activities } }; } export async function generateActivities(input: { model: string; auth: OllamaAuth; notes: string; difficulty: Difficulty; count: number; language?: string; existing?: { title: string; types: string[]; count: number }; focus?: string; signal?: AbortSignal; }): Promise<{ deck: { title: string; subject?: string; summary?: string; language?: string; activities: GeneratedActivity[] }; requested: number; }> { const requested = Math.min(Math.max(input.count, 4), 18); const { value } = await runJsonStep<{ title: string; subject?: string; summary?: string; language?: string; activities: GeneratedActivity[]; }>({ model: input.model, auth: input.auth, system: GENERATION_SYSTEM_PROMPT, user: buildGenerationUserPrompt({ notes: input.notes, difficulty: input.difficulty, count: requested, language: input.language, existing: input.existing, focus: input.focus, }), validate: (parsed) => validateDeck(parsed, Math.max(4, Math.ceil(requested * 0.5)), requested), shapeHint: GENERATION_SHAPE_HINT, label: "Building your activities", signal: input.signal, temperature: 0.7, }); return { deck: value, requested }; } /* ------------------------------------------------------------------ */ /* Grading */ /* ------------------------------------------------------------------ */ const GRADE_SHAPE_HINT = `{"correct":true,"score":85,"feedback":"short feedback","missing":["key point missed"]}`; export async function gradeShortAnswer(input: { model: string; auth: OllamaAuth; question: string; referenceAnswer: string; keyPoints: string[]; userAnswer: string; language?: string; signal?: AbortSignal; }): Promise { const { value } = await runJsonStep({ model: input.model, auth: input.auth, system: GRADING_SYSTEM_PROMPT, user: buildGradingUserPrompt(input), validate: (parsed) => { const result = gradeResultSchema.safeParse(parsed); if (!result.success) return { ok: false, error: summariseZodError(result.error) }; return { ok: true, value: result.data }; }, shapeHint: GRADE_SHAPE_HINT, label: "Grading your answer", signal: input.signal, temperature: 0.1, }); return value; } /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ /** * Small fixes so the UI can trust the data: ordering items must be unique * (they are used as React keys and are ambiguous otherwise), fill_blank * sentences must actually contain a blank, and word banks must contain * the answer. */ function sanitiseActivity(activity: GeneratedActivity): GeneratedActivity { switch (activity.type) { case "ordering": { const seen = new Set(); const items = activity.items.filter((item) => { const key = item.trim().toLowerCase(); if (seen.has(key)) return false; seen.add(key); return true; }); return items.length === activity.items.length ? activity : { ...activity, items }; } case "fill_blank": { if (/_{2,}/.test(activity.sentence)) return activity; return { ...activity, sentence: `${activity.sentence.trim()} ____` }; } case "matching": { const seen = new Set(); const pairs = activity.pairs.filter((pair) => { const key = pair.term.trim().toLowerCase(); if (seen.has(key)) return false; seen.add(key); return true; }); return pairs.length === activity.pairs.length ? activity : { ...activity, pairs }; } default: return activity; } } /** Compact, human-readable zod error for the repair prompt. */ function summariseZodError(error: { issues: Array<{ path: PropertyKey[]; message: string }> }): string { return error.issues .slice(0, 6) .map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`) .join("; "); } /** Turn the structured extraction into the plain text the generator reads. */ export function notesToText(notes: ExtractedNotes): string { const parts: string[] = []; parts.push(`TITLE: ${notes.title}`); if (notes.subject) parts.push(`SUBJECT: ${notes.subject}`); if (notes.language) parts.push(`LANGUAGE: ${notes.language}`); if (notes.summary) parts.push(`SUMMARY: ${notes.summary}`); parts.push("", "TRANSCRIPTION:", notes.notes); if (notes.keyConcepts.length) parts.push("", "KEY CONCEPTS:", ...notes.keyConcepts.map((c) => `- ${c}`)); if (notes.definitions.length) parts.push("", "DEFINITIONS:", ...notes.definitions.map((d) => `- ${d.term}: ${d.definition}`)); if (notes.formulas.length) parts.push("", "FORMULAS:", ...notes.formulas.map((f) => `- ${f.name}: ${f.formula}`)); if (notes.facts.length) parts.push("", "FACTS:", ...notes.facts.map((f) => `- ${f}`)); return parts.join("\n").slice(0, 48_000); }