Freezmo: self-hosted notes-to-activities study app
Next.js 15 app that turns photos, PDFs, DOCX/PPTX and text into a swipeable feed of AI-generated activities (flashcards, multiple choice, true/false, fill-in-the-blank, matching, ordering, short answer) using the user's own Ollama models. Highlights: - All AI calls run in server route handlers; the browser never talks to Ollama. - Bring-your-own-key mode (default): the key lives only in the browser and is sent per-request; server, local-daemon and shared-key modes also supported. - Client-side document extraction (pdf.js, mammoth, jszip) with scanned-PDF page rendering for the vision model; notes stay in their original language. - ChatGPT-style shell: collapsible sidebar / mobile drawer, neutral token-based light + dark themes, folder organisation, PWA (manifest, SW, offline page). - Strict JSON validation with zod + auto-repair, rate limiting, and request guards (cross-origin, body-size, API-key sanitising). - setup.sh bootstraps deps, .env.local and a production build. Gitignore excludes .env*, the generated pdf.js worker and local scratch files.
This commit is contained in:
parent
3baa0fa8ce
commit
3ea2cea100
83 changed files with 10909 additions and 165 deletions
476
lib/documents.ts
Normal file
476
lib/documents.ts
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
"use client";
|
||||
|
||||
/**
|
||||
* Client-side extraction for the note formats people actually use:
|
||||
*
|
||||
* .pdf text layer, or rendered pages when scanned
|
||||
* .docx text + embedded images
|
||||
* .pptx slide text, speaker notes + embedded images
|
||||
* .odt / .odp / .ods LibreOffice documents (content.xml)
|
||||
* .txt .md .csv .rtf-lite plain text
|
||||
*
|
||||
* Everything runs in the browser: the original file never leaves the device,
|
||||
* only the extracted text (and any page images for the vision model) is sent
|
||||
* to the API routes. Photos uploaded as images keep working exactly as before.
|
||||
*/
|
||||
|
||||
import { prepareImageFromBlob, type PreparedImage } from "./image";
|
||||
|
||||
export const MAX_DOCUMENT_BYTES = 40 * 1024 * 1024; // 40 MB per file
|
||||
/** How many rendered page images one document may contribute. */
|
||||
export const MAX_DOCUMENT_IMAGES = 10;
|
||||
/** Characters kept per document (long books get truncated). */
|
||||
export const MAX_DOCUMENT_CHARS = 60_000;
|
||||
/** Pages below this much text are treated as scanned and rendered as images. */
|
||||
const SCANNED_PAGE_CHAR_THRESHOLD = 60;
|
||||
const PDF_RENDER_SCALE = 2;
|
||||
|
||||
export const ACCEPTED_DOCUMENT_EXTENSIONS = [
|
||||
"pdf",
|
||||
"docx",
|
||||
"pptx",
|
||||
"odt",
|
||||
"odp",
|
||||
"ods",
|
||||
"txt",
|
||||
"md",
|
||||
"markdown",
|
||||
"csv",
|
||||
"tsv",
|
||||
"log",
|
||||
] as const;
|
||||
|
||||
export const DOCUMENT_ACCEPT_ATTR = [
|
||||
...ACCEPTED_DOCUMENT_EXTENSIONS.map((extension) => `.${extension}`),
|
||||
"application/pdf",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"text/plain",
|
||||
"text/markdown",
|
||||
"text/csv",
|
||||
].join(",");
|
||||
|
||||
export type DocumentKind = "pdf" | "docx" | "pptx" | "opendocument" | "text";
|
||||
|
||||
export type ExtractedDocument = {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: DocumentKind;
|
||||
/** Extracted text, already truncated to MAX_DOCUMENT_CHARS. */
|
||||
text: string;
|
||||
/** Page-rendered or embedded images (JPEG data URLs, ready to send). */
|
||||
images: PreparedImage[];
|
||||
words: number;
|
||||
characters: number;
|
||||
pages?: number;
|
||||
truncated: boolean;
|
||||
imagesTruncated: boolean;
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export type DocumentProgress = {
|
||||
/** Human readable stage, e.g. "Reading page 4 of 12". */
|
||||
message: string;
|
||||
/** 0..1 where known. */
|
||||
progress?: number;
|
||||
};
|
||||
|
||||
function extensionOf(name: string): string {
|
||||
const index = name.lastIndexOf(".");
|
||||
return index === -1 ? "" : name.slice(index + 1).toLowerCase();
|
||||
}
|
||||
|
||||
export function documentKindOf(file: File): DocumentKind | null {
|
||||
const extension = extensionOf(file.name);
|
||||
switch (extension) {
|
||||
case "pdf":
|
||||
return "pdf";
|
||||
case "docx":
|
||||
return "docx";
|
||||
case "pptx":
|
||||
return "pptx";
|
||||
case "odt":
|
||||
case "odp":
|
||||
case "ods":
|
||||
return "opendocument";
|
||||
case "txt":
|
||||
case "md":
|
||||
case "markdown":
|
||||
case "csv":
|
||||
case "tsv":
|
||||
case "log":
|
||||
return "text";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function isDocumentFile(file: File): boolean {
|
||||
return documentKindOf(file) !== null;
|
||||
}
|
||||
|
||||
export function describeDocumentError(file: File): string | null {
|
||||
if (!isDocumentFile(file)) {
|
||||
return `"${file.name}" is not a supported document (use PDF, DOCX, PPTX, ODT/ODP or text).`;
|
||||
}
|
||||
if (file.size > MAX_DOCUMENT_BYTES) {
|
||||
return `"${file.name}" is larger than 40 MB.`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export const KIND_LABELS: Record<DocumentKind, string> = {
|
||||
pdf: "PDF",
|
||||
docx: "Word",
|
||||
pptx: "PowerPoint",
|
||||
opendocument: "OpenDocument",
|
||||
text: "Text",
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Shared helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function cleanText(raw: string): string {
|
||||
return raw
|
||||
.replace(/\r\n?/g, "\n")
|
||||
.replace(/\u00a0/g, " ")
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.replace(/[ \t]{2,}/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function countWords(text: string): number {
|
||||
return text.split(/\s+/).filter(Boolean).length;
|
||||
}
|
||||
|
||||
function withLimits(
|
||||
name: string,
|
||||
kind: DocumentKind,
|
||||
text: string,
|
||||
images: PreparedImage[],
|
||||
extra: { pages?: number; warnings?: string[]; imagesTruncated?: boolean },
|
||||
): ExtractedDocument {
|
||||
const cleaned = cleanText(text);
|
||||
const truncated = cleaned.length > MAX_DOCUMENT_CHARS;
|
||||
const finalText = truncated ? cleaned.slice(0, MAX_DOCUMENT_CHARS) : cleaned;
|
||||
return {
|
||||
id: `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
name,
|
||||
kind,
|
||||
text: finalText,
|
||||
images,
|
||||
words: countWords(finalText),
|
||||
characters: finalText.length,
|
||||
pages: extra.pages,
|
||||
truncated,
|
||||
imagesTruncated: extra.imagesTruncated ?? false,
|
||||
warnings: extra.warnings ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* PDF */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
type PdfPage = {
|
||||
getTextContent: () => Promise<{ items: Array<{ str?: string; transform?: number[] }> }>;
|
||||
getViewport: (options: { scale: number }) => { width: number; height: number };
|
||||
render: (options: { canvasContext: CanvasRenderingContext2D; viewport: unknown }) => { promise: Promise<void> };
|
||||
cleanup?: () => void;
|
||||
};
|
||||
|
||||
/** Join pdf.js text items, inserting line breaks when the baseline jumps. */
|
||||
function textFromPdfItems(items: Array<{ str?: string; transform?: number[] }>): string {
|
||||
let output = "";
|
||||
let lastY: number | null = null;
|
||||
for (const item of items) {
|
||||
const value = item.str ?? "";
|
||||
const y = item.transform?.[5];
|
||||
if (lastY !== null && typeof y === "number" && Math.abs(y - lastY) > 2) {
|
||||
output += "\n";
|
||||
}
|
||||
output += value;
|
||||
if (value && !value.endsWith(" ")) output += " ";
|
||||
if (typeof y === "number") lastY = y;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
async function extractPdf(
|
||||
file: File,
|
||||
onProgress?: (update: DocumentProgress) => void,
|
||||
): Promise<ExtractedDocument> {
|
||||
// Loaded lazily so pdf.js (and its worker) never hits the initial bundle.
|
||||
const pdfjs = await import("pdfjs-dist");
|
||||
pdfjs.GlobalWorkerOptions.workerSrc = "/pdf.worker.min.mjs";
|
||||
|
||||
const data = new Uint8Array(await file.arrayBuffer());
|
||||
const document = await pdfjs.getDocument({ data }).promise;
|
||||
|
||||
const pages: string[] = [];
|
||||
const images: PreparedImage[] = [];
|
||||
const warnings: string[] = [];
|
||||
let imagesTruncated = false;
|
||||
|
||||
const totalPages = Math.min(document.numPages, 80);
|
||||
if (document.numPages > totalPages) {
|
||||
warnings.push(`Only the first ${totalPages} of ${document.numPages} pages were read.`);
|
||||
}
|
||||
|
||||
for (let pageNumber = 1; pageNumber <= totalPages; pageNumber++) {
|
||||
onProgress?.({
|
||||
message: `Reading page ${pageNumber} of ${totalPages}…`,
|
||||
progress: pageNumber / totalPages,
|
||||
});
|
||||
|
||||
const page = (await document.getPage(pageNumber)) as unknown as PdfPage;
|
||||
let pageText = "";
|
||||
try {
|
||||
const content = await page.getTextContent();
|
||||
pageText = textFromPdfItems(content.items);
|
||||
} catch {
|
||||
warnings.push(`Page ${pageNumber} text could not be extracted.`);
|
||||
}
|
||||
|
||||
if (pageText.trim().length >= SCANNED_PAGE_CHAR_THRESHOLD) {
|
||||
pages.push(`--- page ${pageNumber} ---\n${pageText.trim()}`);
|
||||
} else if (images.length < MAX_DOCUMENT_IMAGES) {
|
||||
// Scanned or image-only page: render it so the vision model can read it.
|
||||
try {
|
||||
const viewport = page.getViewport({ scale: PDF_RENDER_SCALE });
|
||||
const canvas = window.document.createElement("canvas");
|
||||
canvas.width = Math.floor(viewport.width);
|
||||
canvas.height = Math.floor(viewport.height);
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("no 2d context");
|
||||
context.fillStyle = "#ffffff";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
await page.render({ canvasContext: context, viewport }).promise;
|
||||
const blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/jpeg", 0.9));
|
||||
if (blob) {
|
||||
images.push(await prepareImageFromBlob(blob, `${file.name} p${pageNumber}`));
|
||||
}
|
||||
} catch {
|
||||
warnings.push(`Page ${pageNumber} looks scanned but could not be rendered.`);
|
||||
}
|
||||
} else {
|
||||
imagesTruncated = true;
|
||||
}
|
||||
|
||||
if (pageText.trim().length < SCANNED_PAGE_CHAR_THRESHOLD) {
|
||||
// Nothing to add to the text; the image carries the content.
|
||||
imagesTruncated = images.length >= MAX_DOCUMENT_IMAGES;
|
||||
}
|
||||
|
||||
page.cleanup?.();
|
||||
}
|
||||
|
||||
if (images.length > 0) {
|
||||
warnings.push(
|
||||
images.length === 1
|
||||
? "1 scanned page was turned into an image for the vision model."
|
||||
: `${images.length} scanned pages were turned into images for the vision model.`,
|
||||
);
|
||||
}
|
||||
|
||||
return withLimits(file.name, "pdf", pages.join("\n\n"), images, {
|
||||
pages: document.numPages,
|
||||
warnings,
|
||||
imagesTruncated,
|
||||
});
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* DOCX */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
async function extractDocx(file: File): Promise<ExtractedDocument> {
|
||||
const mammoth = await import("mammoth");
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const warnings: string[] = [];
|
||||
const images: PreparedImage[] = [];
|
||||
|
||||
const textResult = await mammoth.extractRawText({ arrayBuffer });
|
||||
for (const message of textResult.messages.slice(0, 3)) {
|
||||
if (message.type === "warning") warnings.push(message.message);
|
||||
}
|
||||
|
||||
// Pull out embedded images (diagrams, pasted screenshots) for the vision step.
|
||||
try {
|
||||
const htmlResult = await mammoth.convertToHtml(
|
||||
{ arrayBuffer },
|
||||
{
|
||||
convertImage: mammoth.images.imgElement(async (image: { read: (encoding: string) => Promise<string>; contentType: string }) => {
|
||||
if (images.length >= MAX_DOCUMENT_IMAGES) return { src: "" };
|
||||
try {
|
||||
const base64 = await image.read("base64");
|
||||
const binary = atob(base64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
||||
const blob = new Blob([bytes], { type: image.contentType || "image/png" });
|
||||
images.push(await prepareImageFromBlob(blob, `${file.name} image ${images.length + 1}`));
|
||||
} catch {
|
||||
/* skip unreadable images */
|
||||
}
|
||||
return { src: "" };
|
||||
}),
|
||||
},
|
||||
);
|
||||
void htmlResult;
|
||||
} catch {
|
||||
warnings.push("Embedded images could not be extracted (text was still read).");
|
||||
}
|
||||
|
||||
const imagesTruncated = images.length >= MAX_DOCUMENT_IMAGES;
|
||||
if (images.length > 0) {
|
||||
warnings.push(`${images.length} embedded image${images.length === 1 ? "" : "s"} sent to the vision model.`);
|
||||
}
|
||||
|
||||
return withLimits(file.name, "docx", textResult.value, images, { warnings, imagesTruncated });
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* PPTX */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const XML_TEXT_RUN = /<a:t[^>]*>([\s\S]*?)<\/a:t>/g;
|
||||
const XML_PARAGRAPH = /<\/a:p>/g;
|
||||
|
||||
function decodeXmlEntities(value: string): string {
|
||||
return value
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, code: string) => String.fromCodePoint(Number.parseInt(code, 16)))
|
||||
.replace(/&/g, "&");
|
||||
}
|
||||
|
||||
function textFromSlideXml(xml: string): string {
|
||||
const withBreaks = xml.replace(XML_PARAGRAPH, "\n");
|
||||
const parts: string[] = [];
|
||||
for (const match of withBreaks.matchAll(XML_TEXT_RUN)) {
|
||||
parts.push(decodeXmlEntities(match[1]));
|
||||
}
|
||||
return cleanText(parts.join(" "));
|
||||
}
|
||||
|
||||
async function extractOoxmlPackage(
|
||||
file: File,
|
||||
kind: "pptx" | "opendocument",
|
||||
): Promise<ExtractedDocument> {
|
||||
const JSZip = (await import("jszip")).default;
|
||||
const zip = await JSZip.loadAsync(await file.arrayBuffer());
|
||||
const warnings: string[] = [];
|
||||
const images: PreparedImage[] = [];
|
||||
const sections: string[] = [];
|
||||
|
||||
if (kind === "pptx") {
|
||||
const slideNames = Object.keys(zip.files)
|
||||
.filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
|
||||
.sort((a, b) => {
|
||||
const na = Number(a.match(/(\d+)/)?.[1] ?? 0);
|
||||
const nb = Number(b.match(/(\d+)/)?.[1] ?? 0);
|
||||
return na - nb;
|
||||
});
|
||||
|
||||
for (const [index, name] of slideNames.entries()) {
|
||||
const xml = await zip.file(name)?.async("string");
|
||||
if (!xml) continue;
|
||||
const body = textFromSlideXml(xml);
|
||||
const notesName = `ppt/notesSlides/notesSlide${index + 1}.xml`;
|
||||
const notesXml = zip.file(notesName) ? await zip.file(notesName)?.async("string") : null;
|
||||
const notes = notesXml ? textFromSlideXml(notesXml) : "";
|
||||
const chunk = [`--- slide ${index + 1} ---`, body || "(no text on this slide)"];
|
||||
if (notes) chunk.push(`Speaker notes: ${notes}`);
|
||||
sections.push(chunk.join("\n"));
|
||||
}
|
||||
|
||||
const media = Object.keys(zip.files).filter(
|
||||
(name) => /^ppt\/media\/.+\.(png|jpe?g|gif|webp|bmp)$/i.test(name) && !name.includes("/_rels/"),
|
||||
);
|
||||
for (const name of media) {
|
||||
if (images.length >= MAX_DOCUMENT_IMAGES) break;
|
||||
try {
|
||||
const blob = await zip.file(name)?.async("blob");
|
||||
if (blob) images.push(await prepareImageFromBlob(blob, `${file.name} ${name.split("/").pop()}`));
|
||||
} catch {
|
||||
/* skip */
|
||||
}
|
||||
}
|
||||
if (media.length > images.length) {
|
||||
warnings.push(`Only ${images.length} of ${media.length} slide images were kept.`);
|
||||
}
|
||||
} else {
|
||||
const content = await zip.file("content.xml")?.async("string");
|
||||
if (!content) throw new Error("This OpenDocument file has no readable content.");
|
||||
const text = decodeXmlEntities(
|
||||
content
|
||||
.replace(/<text:p[^>]*>/g, "\n")
|
||||
.replace(/<text:h[^>]*>/g, "\n")
|
||||
.replace(/<\/text:p>/g, "\n")
|
||||
.replace(/<\/text:h>/g, "\n")
|
||||
.replace(/<[^>]+>/g, ""),
|
||||
);
|
||||
sections.push(text);
|
||||
}
|
||||
|
||||
return withLimits(file.name, kind, sections.join("\n\n"), images, {
|
||||
pages: kind === "pptx" ? sections.length : undefined,
|
||||
warnings,
|
||||
imagesTruncated: images.length >= MAX_DOCUMENT_IMAGES,
|
||||
});
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Plain text */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
async function extractPlainText(file: File): Promise<ExtractedDocument> {
|
||||
const text = await file.text();
|
||||
return withLimits(file.name, "text", text, [], {});
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Dispatcher */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export async function extractDocument(
|
||||
file: File,
|
||||
onProgress?: (update: DocumentProgress) => void,
|
||||
): Promise<ExtractedDocument> {
|
||||
const kind = documentKindOf(file);
|
||||
if (!kind) throw new Error(`"${file.name}" is not a supported document.`);
|
||||
|
||||
switch (kind) {
|
||||
case "pdf":
|
||||
return extractPdf(file, onProgress);
|
||||
case "docx":
|
||||
onProgress?.({ message: "Reading the Word document…" });
|
||||
return extractDocx(file);
|
||||
case "pptx":
|
||||
onProgress?.({ message: "Reading the slides…" });
|
||||
return extractOoxmlPackage(file, "pptx");
|
||||
case "opendocument":
|
||||
onProgress?.({ message: "Reading the OpenDocument file…" });
|
||||
return extractOoxmlPackage(file, "opendocument");
|
||||
case "text":
|
||||
onProgress?.({ message: "Reading the text file…" });
|
||||
return extractPlainText(file);
|
||||
}
|
||||
}
|
||||
|
||||
/** Merge several documents into one text block with file headers. */
|
||||
export function documentsToText(documents: ExtractedDocument[]): string {
|
||||
return documents
|
||||
.filter((document) => document.text.trim().length > 0)
|
||||
.map((document) => `### ${document.name}\n${document.text.trim()}`)
|
||||
.join("\n\n")
|
||||
.slice(0, MAX_DOCUMENT_CHARS * 2);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue