Freezmo: self-hosted notes-to-activities study app
Next.js 15 app that turns photos, PDFs, DOCX/PPTX and text into a swipeable feed of AI-generated activities (flashcards, multiple choice, true/false, fill-in-the-blank, matching, ordering, short answer) using the user's own Ollama models. Highlights: - All AI calls run in server route handlers; the browser never talks to Ollama. - Bring-your-own-key mode (default): the key lives only in the browser and is sent per-request; server, local-daemon and shared-key modes also supported. - Client-side document extraction (pdf.js, mammoth, jszip) with scanned-PDF page rendering for the vision model; notes stay in their original language. - ChatGPT-style shell: collapsible sidebar / mobile drawer, neutral token-based light + dark themes, folder organisation, PWA (manifest, SW, offline page). - Strict JSON validation with zod + auto-repair, rate limiting, and request guards (cross-origin, body-size, API-key sanitising). - setup.sh bootstraps deps, .env.local and a production build. Gitignore excludes .env*, the generated pdf.js worker and local scratch files.
This commit is contained in:
parent
3baa0fa8ce
commit
3ea2cea100
83 changed files with 10909 additions and 165 deletions
442
lib/ai.ts
Normal file
442
lib/ai.ts
Normal file
|
|
@ -0,0 +1,442 @@
|
|||
/**
|
||||
* Server-side AI pipeline: vision extraction, activity generation and
|
||||
* short-answer grading. Every step asks for JSON, validates with zod and
|
||||
* retries once with the validation error fed back to the model.
|
||||
*/
|
||||
|
||||
import { OllamaError, ollamaChat, type OllamaMessage } from "./ollama";
|
||||
import { needsLanguageRepair } from "./language";
|
||||
import {
|
||||
buildGenerationUserPrompt,
|
||||
buildGradingUserPrompt,
|
||||
buildRepairPrompt,
|
||||
GENERATION_SYSTEM_PROMPT,
|
||||
GRADING_SYSTEM_PROMPT,
|
||||
VISION_SYSTEM_PROMPT,
|
||||
VISION_USER_INSTRUCTION,
|
||||
} from "./prompts";
|
||||
import {
|
||||
extractedNotesSchema,
|
||||
generatedActivitySchema,
|
||||
gradeResultSchema,
|
||||
type Difficulty,
|
||||
type ExtractedNotes,
|
||||
type GeneratedActivity,
|
||||
type GradeResult,
|
||||
} from "./schemas";
|
||||
import { parseJsonLoose } from "./utils";
|
||||
import type { OllamaAuth } from "./config";
|
||||
|
||||
export type PipelineStage = "vision" | "generation" | "done";
|
||||
|
||||
type JsonStepOptions<T> = {
|
||||
model: string;
|
||||
auth: OllamaAuth;
|
||||
system: string;
|
||||
user: string;
|
||||
images?: string[];
|
||||
/** Validates and can also "tighten" the parsed value. */
|
||||
validate: (value: unknown) => { ok: true; value: T } | { ok: false; error: string };
|
||||
shapeHint: string;
|
||||
label: string;
|
||||
signal?: AbortSignal;
|
||||
temperature?: number;
|
||||
};
|
||||
|
||||
type JsonStepResult<T> = {
|
||||
value: T;
|
||||
/** true when the first attempt failed and the repair round was used. */
|
||||
repaired: boolean;
|
||||
attempts: number;
|
||||
};
|
||||
|
||||
async function runJsonStep<T>({
|
||||
model,
|
||||
auth,
|
||||
system,
|
||||
user,
|
||||
images,
|
||||
validate,
|
||||
shapeHint,
|
||||
label,
|
||||
signal,
|
||||
temperature,
|
||||
}: JsonStepOptions<T>): Promise<JsonStepResult<T>> {
|
||||
const messages: OllamaMessage[] = [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: user, ...(images && images.length ? { images } : {}) },
|
||||
];
|
||||
|
||||
let lastRaw = "";
|
||||
let lastError = "";
|
||||
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
const result = await ollamaChat({
|
||||
model,
|
||||
auth,
|
||||
messages,
|
||||
json: true,
|
||||
signal,
|
||||
temperature: temperature ?? 0.4,
|
||||
});
|
||||
lastRaw = result.content;
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = parseJsonLoose(result.content);
|
||||
} catch (error) {
|
||||
lastError = `Response was not valid JSON: ${error instanceof Error ? error.message : String(error)}`;
|
||||
if (attempt === 1) {
|
||||
messages.push({ role: "assistant", content: result.content });
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: buildRepairPrompt(lastRaw, lastError, shapeHint),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
throw new OllamaError("bad_response", `${label} failed: ${lastError}`);
|
||||
}
|
||||
|
||||
const checked = validate(parsed);
|
||||
if (checked.ok) {
|
||||
return { value: checked.value, repaired: attempt > 1, attempts: attempt };
|
||||
}
|
||||
|
||||
lastError = checked.error;
|
||||
if (attempt === 1) {
|
||||
messages.push({ role: "assistant", content: result.content });
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: buildRepairPrompt(lastRaw, lastError, shapeHint),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
throw new OllamaError(
|
||||
"bad_response",
|
||||
`${label} returned data that did not match the expected format: ${lastError}`,
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Step 1 - vision */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const VISION_SHAPE_HINT = `{"title":"...","subject":"...","language":"English name of the notes' language","summary":"...","notes":"plain text transcription","keyConcepts":["..."],"definitions":[{"term":"...","definition":"..."}],"formulas":[{"name":"...","formula":"..."}],"facts":["..."]}`;
|
||||
|
||||
/**
|
||||
* Read the uploaded images (and any pasted text) into structured notes.
|
||||
* Any pasted text is passed through verbatim as additional note content.
|
||||
*
|
||||
* A small vision model can accidentally translate on hard handwriting, so the
|
||||
* result is checked and — when it clearly came back in another language —
|
||||
* restored by the (stronger, multilingual) text model.
|
||||
*/
|
||||
export async function extractNotes(input: {
|
||||
model: string;
|
||||
/** Model used to restore the language if the transcription was translated. */
|
||||
repairModel: string;
|
||||
auth: OllamaAuth;
|
||||
images: string[];
|
||||
text?: string;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (message: string) => void;
|
||||
}): Promise<{ notes: ExtractedNotes; repaired: boolean; languageRepaired: boolean }> {
|
||||
const user = input.text?.trim()
|
||||
? `${VISION_USER_INSTRUCTION}\n\nADDITIONAL TEXT THE STUDENT PASTED (treat as notes too):\n${input.text.slice(0, 12_000)}`
|
||||
: VISION_USER_INSTRUCTION;
|
||||
|
||||
const { value, repaired } = await runJsonStep<ExtractedNotes>({
|
||||
model: input.model,
|
||||
auth: input.auth,
|
||||
system: VISION_SYSTEM_PROMPT,
|
||||
user,
|
||||
images: input.images,
|
||||
validate: (parsed) => {
|
||||
const result = extractedNotesSchema.safeParse(parsed);
|
||||
if (!result.success) return { ok: false, error: summariseZodError(result.error) };
|
||||
if (!result.data.notes.trim()) return { ok: false, error: "The notes field is empty." };
|
||||
return { ok: true, value: result.data };
|
||||
},
|
||||
shapeHint: VISION_SHAPE_HINT,
|
||||
label: "Reading your notes",
|
||||
signal: input.signal,
|
||||
temperature: 0.2,
|
||||
});
|
||||
|
||||
const check = needsLanguageRepair({ language: value.language, text: value.notes });
|
||||
if (!check.suspicious) return { notes: value, repaired, languageRepaired: false };
|
||||
|
||||
console.warn(`[freezmo] restoring the notes' language: ${check.reason}`);
|
||||
input.onProgress?.(`Restoring the notes' language (${value.language ?? "original"})…`);
|
||||
try {
|
||||
const restored = await restoreLanguage({
|
||||
model: input.repairModel,
|
||||
auth: input.auth,
|
||||
language: value.language ?? "the original language of the notes",
|
||||
extraction: value,
|
||||
signal: input.signal,
|
||||
});
|
||||
return { notes: restored, repaired, languageRepaired: true };
|
||||
} catch (error) {
|
||||
// The guard is best-effort: a failure must never break generation.
|
||||
console.warn("[freezmo] language restore failed, using the original transcription", error);
|
||||
return { notes: value, repaired, languageRepaired: false };
|
||||
}
|
||||
}
|
||||
|
||||
const RESTORE_SYSTEM_PROMPT = `You are Freezmo's language restorer. A student's study notes were wrongly translated out of their original language (usually into English) before reaching you.
|
||||
|
||||
Your job: rewrite the JSON you receive back into the ORIGINAL language, keeping every fact, number, formula, name and definition exactly as it is.
|
||||
|
||||
Rules:
|
||||
- Translate ONLY what was wrongly translated. Do not add, remove, summarise or explain anything.
|
||||
- Keep the same JSON keys and the same array structure. Keep the same number of items in every array.
|
||||
- Keep technical terms, units and formulas as the original notes used them.
|
||||
- Keep proper nouns as they are unless the original language would inflect them.
|
||||
- Output JSON only, no markdown fences, no commentary.`;
|
||||
|
||||
/**
|
||||
* Ask the text model to put a translated transcription back into the language
|
||||
* the vision model reported. Returns the original value if the repair is
|
||||
* unusable.
|
||||
*/
|
||||
export async function restoreLanguage(input: {
|
||||
model: string;
|
||||
auth: OllamaAuth;
|
||||
language: string;
|
||||
extraction: ExtractedNotes;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ExtractedNotes> {
|
||||
const value = await runJsonStep<ExtractedNotes>({
|
||||
model: input.model,
|
||||
auth: input.auth,
|
||||
system: RESTORE_SYSTEM_PROMPT,
|
||||
user: [
|
||||
`ORIGINAL LANGUAGE: ${input.language}`,
|
||||
"",
|
||||
"JSON TO RESTORE:",
|
||||
JSON.stringify(input.extraction).slice(0, 40_000),
|
||||
"",
|
||||
"Return the same JSON with the text fields written in the original language.",
|
||||
].join("\n"),
|
||||
validate: (parsed) => {
|
||||
const result = extractedNotesSchema.safeParse(parsed);
|
||||
if (!result.success) return { ok: false, error: summariseZodError(result.error) };
|
||||
if (!result.data.notes.trim()) return { ok: false, error: "The notes field is empty." };
|
||||
return { ok: true, value: result.data };
|
||||
},
|
||||
shapeHint: VISION_SHAPE_HINT,
|
||||
label: "Restoring the notes' language",
|
||||
signal: input.signal,
|
||||
temperature: 0.1,
|
||||
});
|
||||
return value.value;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Step 2 - generation */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const GENERATION_SHAPE_HINT = `{"title":"...","subject":"...","language":"English name of the language used","summary":"...","activities":[{"type":"multiple_choice","topic":"...","question":"...","options":["...","...","...","..."],"correctIndex":0,"explanation":"..."}, ...]}`;
|
||||
|
||||
const VALID_TYPES = [
|
||||
"flashcard",
|
||||
"multiple_choice",
|
||||
"true_false",
|
||||
"fill_blank",
|
||||
"matching",
|
||||
"ordering",
|
||||
"short_answer",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Be forgiving: validate each activity on its own and keep the good ones.
|
||||
* Only if too few survive does the step get retried.
|
||||
*/
|
||||
function validateDeck(parsed: unknown, minimum: number, requested: number) {
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return { ok: false as const, error: "Expected a JSON object at the top level." };
|
||||
}
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const title = typeof record.title === "string" && record.title.trim() ? record.title.trim() : "Untitled deck";
|
||||
const subject = typeof record.subject === "string" ? record.subject.trim() : undefined;
|
||||
const summary = typeof record.summary === "string" ? record.summary.trim() : undefined;
|
||||
const language = typeof record.language === "string" && record.language.trim() ? record.language.trim() : undefined;
|
||||
|
||||
const rawActivities = Array.isArray(record.activities)
|
||||
? record.activities
|
||||
: Array.isArray(record.items)
|
||||
? record.items
|
||||
: null;
|
||||
|
||||
if (!rawActivities) {
|
||||
return { ok: false as const, error: `Missing "activities" array. Expected an array of ${requested} activity objects.` };
|
||||
}
|
||||
|
||||
const activities: GeneratedActivity[] = [];
|
||||
const rejected: string[] = [];
|
||||
|
||||
rawActivities.slice(0, 24).forEach((item, index) => {
|
||||
const result = generatedActivitySchema.safeParse(item);
|
||||
if (result.success) {
|
||||
activities.push(sanitiseActivity(result.data));
|
||||
} else {
|
||||
const type = (item as { type?: unknown } | null)?.type;
|
||||
rejected.push(`#${index + 1} (${typeof type === "string" ? type : "no type"}): ${summariseZodError(result.error)}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (activities.length < minimum) {
|
||||
return {
|
||||
ok: false as const,
|
||||
error: `Only ${activities.length} valid activities were found, at least ${minimum} are needed. Rejected: ${rejected.slice(0, 3).join("; ") || "none"}. Valid types are: ${VALID_TYPES.join(", ")}.`,
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true as const, value: { title, subject, summary, language, activities } };
|
||||
}
|
||||
|
||||
export async function generateActivities(input: {
|
||||
model: string;
|
||||
auth: OllamaAuth;
|
||||
notes: string;
|
||||
difficulty: Difficulty;
|
||||
count: number;
|
||||
language?: string;
|
||||
existing?: { title: string; types: string[]; count: number };
|
||||
focus?: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<{
|
||||
deck: { title: string; subject?: string; summary?: string; language?: string; activities: GeneratedActivity[] };
|
||||
requested: number;
|
||||
}> {
|
||||
const requested = Math.min(Math.max(input.count, 4), 18);
|
||||
|
||||
const { value } = await runJsonStep<{
|
||||
title: string;
|
||||
subject?: string;
|
||||
summary?: string;
|
||||
language?: string;
|
||||
activities: GeneratedActivity[];
|
||||
}>({
|
||||
model: input.model,
|
||||
auth: input.auth,
|
||||
system: GENERATION_SYSTEM_PROMPT,
|
||||
user: buildGenerationUserPrompt({
|
||||
notes: input.notes,
|
||||
difficulty: input.difficulty,
|
||||
count: requested,
|
||||
language: input.language,
|
||||
existing: input.existing,
|
||||
focus: input.focus,
|
||||
}),
|
||||
validate: (parsed) => validateDeck(parsed, Math.max(4, Math.ceil(requested * 0.5)), requested),
|
||||
shapeHint: GENERATION_SHAPE_HINT,
|
||||
label: "Building your activities",
|
||||
signal: input.signal,
|
||||
temperature: 0.7,
|
||||
});
|
||||
|
||||
return { deck: value, requested };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Grading */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const GRADE_SHAPE_HINT = `{"correct":true,"score":85,"feedback":"short feedback","missing":["key point missed"]}`;
|
||||
|
||||
export async function gradeShortAnswer(input: {
|
||||
model: string;
|
||||
auth: OllamaAuth;
|
||||
question: string;
|
||||
referenceAnswer: string;
|
||||
keyPoints: string[];
|
||||
userAnswer: string;
|
||||
language?: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<GradeResult> {
|
||||
const { value } = await runJsonStep<GradeResult>({
|
||||
model: input.model,
|
||||
auth: input.auth,
|
||||
system: GRADING_SYSTEM_PROMPT,
|
||||
user: buildGradingUserPrompt(input),
|
||||
validate: (parsed) => {
|
||||
const result = gradeResultSchema.safeParse(parsed);
|
||||
if (!result.success) return { ok: false, error: summariseZodError(result.error) };
|
||||
return { ok: true, value: result.data };
|
||||
},
|
||||
shapeHint: GRADE_SHAPE_HINT,
|
||||
label: "Grading your answer",
|
||||
signal: input.signal,
|
||||
temperature: 0.1,
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* Small fixes so the UI can trust the data: ordering items must be unique
|
||||
* (they are used as React keys and are ambiguous otherwise), fill_blank
|
||||
* sentences must actually contain a blank, and word banks must contain
|
||||
* the answer.
|
||||
*/
|
||||
function sanitiseActivity(activity: GeneratedActivity): GeneratedActivity {
|
||||
switch (activity.type) {
|
||||
case "ordering": {
|
||||
const seen = new Set<string>();
|
||||
const items = activity.items.filter((item) => {
|
||||
const key = item.trim().toLowerCase();
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
return items.length === activity.items.length ? activity : { ...activity, items };
|
||||
}
|
||||
case "fill_blank": {
|
||||
if (/_{2,}/.test(activity.sentence)) return activity;
|
||||
return { ...activity, sentence: `${activity.sentence.trim()} ____` };
|
||||
}
|
||||
case "matching": {
|
||||
const seen = new Set<string>();
|
||||
const pairs = activity.pairs.filter((pair) => {
|
||||
const key = pair.term.trim().toLowerCase();
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
return pairs.length === activity.pairs.length ? activity : { ...activity, pairs };
|
||||
}
|
||||
default:
|
||||
return activity;
|
||||
}
|
||||
}
|
||||
|
||||
/** Compact, human-readable zod error for the repair prompt. */
|
||||
function summariseZodError(error: { issues: Array<{ path: PropertyKey[]; message: string }> }): string {
|
||||
return error.issues
|
||||
.slice(0, 6)
|
||||
.map((issue) => `${issue.path.map(String).join(".") || "(root)"}: ${issue.message}`)
|
||||
.join("; ");
|
||||
}
|
||||
|
||||
/** Turn the structured extraction into the plain text the generator reads. */
|
||||
export function notesToText(notes: ExtractedNotes): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`TITLE: ${notes.title}`);
|
||||
if (notes.subject) parts.push(`SUBJECT: ${notes.subject}`);
|
||||
if (notes.language) parts.push(`LANGUAGE: ${notes.language}`);
|
||||
if (notes.summary) parts.push(`SUMMARY: ${notes.summary}`);
|
||||
parts.push("", "TRANSCRIPTION:", notes.notes);
|
||||
if (notes.keyConcepts.length) parts.push("", "KEY CONCEPTS:", ...notes.keyConcepts.map((c) => `- ${c}`));
|
||||
if (notes.definitions.length)
|
||||
parts.push("", "DEFINITIONS:", ...notes.definitions.map((d) => `- ${d.term}: ${d.definition}`));
|
||||
if (notes.formulas.length)
|
||||
parts.push("", "FORMULAS:", ...notes.formulas.map((f) => `- ${f.name}: ${f.formula}`));
|
||||
if (notes.facts.length) parts.push("", "FACTS:", ...notes.facts.map((f) => `- ${f}`));
|
||||
return parts.join("\n").slice(0, 48_000);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue