Martin Dvorak
abacfb6018
Sync was push-on-demand only: a deck frozen on the laptop stayed in dirty
until someone opened Settings and pressed Sync now. The two stores whose
writes mean "the user just made something" now schedule one debounced
background syncNow("auto") through lib/sync-auto: debounced 2.5s so a burst
of writes is one push, one run at a time, silent on failure (the data stays
dirty and Settings still offers the manual sync), retried on the browser's
online event, and imported dynamically so the sync code stays out of the
first load with no import cycle.
markSyncDirty() now reports whether it actually marked, which keeps pull and
backup-import paths quiet while a real import pushes like any other local
change. Answers and chat keep the existing manual behaviour on purpose —
they change every few seconds while studying.
Tests: phase-1 (a deck write pushes itself, then Sync now is a no-op — 50
checks) and phase-3 (a check run pushes the new homework record — 30 checks).
README: decision 59. SW -> freezmo-v46.
308 lines
15 KiB
JavaScript
308 lines
15 KiB
JavaScript
/**
|
|
* Phase 3 verification: "Check my work" (server mapping + phone and desktop
|
|
* UI), follow-ups (shorter / more detail / explain a step differently) and
|
|
* "Turn into a study deck" pre-filling the New deck flow.
|
|
* Runs against a stub Ollama so no model is needed.
|
|
*/
|
|
import http from "node:http";
|
|
import { mkdirSync } from "node:fs";
|
|
|
|
import { launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";
|
|
|
|
const { check, finish } = makeChecker();
|
|
|
|
const STUB_PORT = 11998;
|
|
const APP_PORT = 3208;
|
|
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
|
|
const APP = `http://127.0.0.1:${APP_PORT}`;
|
|
|
|
const CHECK_REPLY = {
|
|
title: "Equation check",
|
|
language: "English",
|
|
verdict: "One mistake: the sign in the second line.",
|
|
correct: ["The setup is right."],
|
|
wrong: [{ issue: "x = 3", why: "4 - 2 is 2, not 3.", fix: "x = 2" }],
|
|
corrected: "x + 2 = 4\nx = 4 - 2\nx = 2",
|
|
};
|
|
const GUIDE_REPLY = {
|
|
title: "Equation guide",
|
|
language: "English",
|
|
steps: [
|
|
{
|
|
title: "Move the number",
|
|
explanation: "Subtract 2 from both sides so x stands alone.",
|
|
hints: ["Think about the opposite operation.", "Subtract 2 from both sides.", "x = 4 - 2"],
|
|
answer: "x = 4 - 2",
|
|
},
|
|
{
|
|
title: "Finish it",
|
|
explanation: "Now do the subtraction to get the value of x.",
|
|
hints: ["Just arithmetic now.", "4 minus 2.", "x = 2"],
|
|
answer: "x = 2",
|
|
},
|
|
],
|
|
};
|
|
|
|
const stub = http.createServer((request, response) => {
|
|
if (request.url?.startsWith("/api/tags")) {
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
|
|
return;
|
|
}
|
|
let raw = "";
|
|
request.on("data", (chunk) => (raw += chunk));
|
|
request.on("end", () => {
|
|
let content = "{}";
|
|
try {
|
|
const body = JSON.parse(raw);
|
|
const system = String(body.messages?.[0]?.content ?? "");
|
|
const user = String(body.messages?.[1]?.content ?? "");
|
|
if (system.includes("study-note transcriber")) {
|
|
content = JSON.stringify({
|
|
title: "Attempt",
|
|
language: "English",
|
|
notes: "x = 3 (from the photo)",
|
|
keyConcepts: ["linear equation"],
|
|
});
|
|
} else if (system.includes("homework checker")) content = JSON.stringify(CHECK_REPLY);
|
|
else if (system.includes("homework writer")) content = JSON.stringify({ title: "Equation", language: "English", answer: "x = 2" });
|
|
else if (system.includes("homework coach")) content = JSON.stringify(GUIDE_REPLY);
|
|
else if (system.includes("homework assistant")) {
|
|
content = user.includes("SHORTER")
|
|
? "SHORTER: x = 2"
|
|
: user.includes("MORE DETAIL")
|
|
? "MORE DETAIL: x + 2 = 4, so subtract 2: x = 2"
|
|
: "EXPLAINED DIFFERENTLY: picture a balance scale.";
|
|
}
|
|
} catch {
|
|
/* keep the default */
|
|
}
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ message: { content } }));
|
|
});
|
|
});
|
|
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));
|
|
|
|
check("stub app port is free", !(await portInUse(APP_PORT)));
|
|
const syncDir = new URL("./.tmp/sync-phase3", import.meta.url).pathname;
|
|
mkdirSync(syncDir, { recursive: true });
|
|
const app = spawnApp(APP_PORT, {
|
|
OLLAMA_BASE_URL: STUB_BASE,
|
|
OLLAMA_AUTH: "local",
|
|
OLLAMA_VERIFY_KEY: "false",
|
|
SYNC_DIR: syncDir,
|
|
});
|
|
|
|
const browser = await launchBrowser();
|
|
|
|
try {
|
|
const up = await waitForHttp(`${APP}/api/health`);
|
|
check("stub app instance is up", up);
|
|
|
|
if (up) {
|
|
/* 1. server: check mode ---------------------------------------- */
|
|
const apiCheck = await fetch(`${APP}/api/homework`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4", solution: "x = 3" }),
|
|
});
|
|
const events = (await apiCheck.text())
|
|
.trim()
|
|
.split("\n")
|
|
.map((line) => JSON.parse(line));
|
|
const homeworkEvent = events.find((event) => event.type === "homework");
|
|
check("api: check mode returns the feedback payload", homeworkEvent?.homework?.mode === "check");
|
|
check("api: verdict and corrected version", Boolean(homeworkEvent?.homework?.verdict) && homeworkEvent?.homework?.corrected.includes("x = 2"));
|
|
check("api: wrong entry has issue/why/fix", homeworkEvent?.homework?.wrong?.[0]?.issue === "x = 3" && Boolean(homeworkEvent?.homework?.wrong?.[0]?.fix));
|
|
|
|
const missingSolution = await fetch(`${APP}/api/homework`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4" }),
|
|
});
|
|
check("api: check without a solution → 400", missingSolution.status === 400, String(missingSolution.status));
|
|
|
|
/* Photos of the attempt: transcribed by the vision model first, then the
|
|
check runs on typed text + transcription. */
|
|
const photoCheck = await fetch(`${APP}/api/homework`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
mode: "check",
|
|
text: "Solve x + 2 = 4",
|
|
solutionImages: [`data:image/jpeg;base64,${"A".repeat(2000)}`],
|
|
}),
|
|
});
|
|
const photoEvents = (await photoCheck.text())
|
|
.trim()
|
|
.split("\n")
|
|
.map((line) => JSON.parse(line));
|
|
const photoHomework = photoEvents.find((event) => event.type === "homework");
|
|
check(
|
|
"api: solution photos are transcribed into the attempt",
|
|
photoHomework?.meta?.solutionText?.includes("from the photo") && photoHomework?.homework?.mode === "check",
|
|
JSON.stringify(photoHomework?.meta ?? {}).slice(0, 90),
|
|
);
|
|
check(
|
|
"api: vision ran before the check",
|
|
photoEvents.some((event) => event.type === "stage" && event.message.includes("solution")),
|
|
JSON.stringify(photoEvents.filter((event) => event.type === "stage").map((event) => event.stage)),
|
|
);
|
|
|
|
/* 2. server: refine -------------------------------------------- */
|
|
const refine = await fetch(`${APP}/api/homework/refine`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ instruction: "shorter", text: "x + 2 = 4, therefore x = 2" }),
|
|
});
|
|
const refineBody = await refine.json();
|
|
check("api: refine returns rewritten text", refine.status === 200 && refineBody.text.includes("SHORTER"), JSON.stringify(refineBody).slice(0, 60));
|
|
const badRefine = await fetch(`${APP}/api/homework/refine`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ instruction: "wrong", text: "x" }),
|
|
});
|
|
check("api: refine validates the instruction", badRefine.status === 400, String(badRefine.status));
|
|
|
|
/* 3. phone UI: check my work + follow-ups + to-deck -------------- */
|
|
const phone = await makePage(browser, { width: 390 });
|
|
await seed(phone.page, { decks: [] });
|
|
await phone.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
|
await phone.page.waitForTimeout(900);
|
|
await phone.page.getByRole("button", { name: "Paste text instead" }).click();
|
|
await phone.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
|
await phone.page.getByRole("button", { name: "Next" }).click();
|
|
await phone.page.waitForTimeout(400);
|
|
check("phone: AI disclaimer near the start cards", await phone.page.getByText("Freezmo uses AI and can make mistakes").isVisible());
|
|
await phone.page.getByRole("button", { name: /Check my work/ }).click();
|
|
await phone.page.waitForTimeout(400);
|
|
check("phone: solution field revealed", await phone.page.locator("#homework-solution").isVisible());
|
|
await phone.page.locator("#homework-solution").fill("x = 3");
|
|
await phone.page.getByRole("button", { name: "Check my work", exact: true }).click();
|
|
await phone.page.waitForTimeout(2500);
|
|
check("phone: verdict shown", await phone.page.getByText("One mistake: the sign in the second line.").isVisible());
|
|
check("phone: what's right listed", await phone.page.getByText("The setup is right.").isVisible());
|
|
check("phone: what needs fixing listed", await phone.page.getByText("4 - 2 is 2, not 3.").isVisible());
|
|
check("phone: corrected version shown", await phone.page.getByText("x = 4 - 2").isVisible());
|
|
check("phone: AI disclaimer on the result", await phone.page.getByText("Freezmo uses AI and can make mistakes").isVisible());
|
|
|
|
await phone.page.getByRole("button", { name: "Shorter" }).click();
|
|
await phone.page.waitForTimeout(1500);
|
|
check("phone: shorter follow-up replaces the text", await phone.page.getByText("SHORTER: x = 2").isVisible());
|
|
check("phone: original can be restored", await phone.page.getByRole("button", { name: "Original" }).isVisible());
|
|
|
|
await phone.page.getByRole("button", { name: "Turn into a study deck" }).click();
|
|
await phone.page.waitForTimeout(1500);
|
|
check("phone: landed on New deck", phone.page.url().includes("/create"));
|
|
const seededText = await phone.page.locator("textarea").first().inputValue().catch(() => "");
|
|
check("phone: deck wizard pre-filled with the content", seededText.includes("x = 4 - 2") || seededText.includes("SHORTER"), seededText.slice(0, 40));
|
|
await phone.context.close();
|
|
|
|
/* 4. desktop UI + guide explain ---------------------------------- */
|
|
const desktop = await makePage(browser, { width: 1024, height: 900 });
|
|
await seed(desktop.page, { decks: [] });
|
|
const checkBodies = [];
|
|
desktop.page.on("request", (request) => {
|
|
if (request.method() === "POST" && request.url().includes("/api/homework")) {
|
|
checkBodies.push(request.postData() ?? "");
|
|
}
|
|
});
|
|
await desktop.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
|
await desktop.page.waitForTimeout(900);
|
|
await desktop.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
|
await desktop.page.getByRole("button", { name: /Check my work/ }).click();
|
|
await desktop.page.waitForTimeout(300);
|
|
check("desktop: solution field appears with the mode", await desktop.page.locator("#homework-solution").isVisible());
|
|
await desktop.page.locator("#homework-solution").fill("x = 3");
|
|
/* A photo of the work travels with the attempt (handwriting, scans). */
|
|
await desktop.page.locator('#homework-solution ~ input[type="file"]').first().setInputFiles({
|
|
name: "attempt.png",
|
|
mimeType: "image/png",
|
|
// 1x1 PNG
|
|
buffer: Buffer.from(
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
|
|
"base64",
|
|
),
|
|
});
|
|
await desktop.page.waitForTimeout(1200);
|
|
check("desktop: attached photo gets a thumbnail", (await desktop.page.locator('img[src^="data:image"]').count()) >= 1);
|
|
await desktop.page.getByRole("button", { name: "Check my work", exact: true }).click();
|
|
await desktop.page.waitForTimeout(3000);
|
|
check("desktop: check result renders", await desktop.page.getByText("Corrected version").isVisible());
|
|
check("desktop: the attempt travelled with the photo", checkBodies.some((body) => body.includes("solutionImages")));
|
|
check("desktop: AI disclaimer on the result", await desktop.page.getByText("Freezmo uses AI and can make mistakes").isVisible());
|
|
await desktop.context.close();
|
|
|
|
/* 4b. a new homework pushes itself when sync is on ---------------- */
|
|
const synced = await makePage(browser, { width: 390 });
|
|
await seed(synced.page, { decks: [] });
|
|
await synced.page.goto(`${APP}/`, { waitUntil: "networkidle" });
|
|
await synced.page.waitForTimeout(800);
|
|
await synced.page.locator("nav.tabbar").getByRole("button").click();
|
|
await synced.page.getByRole("dialog").waitFor({ timeout: 6000 });
|
|
await synced.page.waitForTimeout(500);
|
|
await synced.page.getByRole("button", { name: "Turn on sync" }).click();
|
|
await synced.page.getByPlaceholder("Sync passphrase").fill("correct horse battery");
|
|
await synced.page.getByPlaceholder("Repeat the passphrase").fill("correct horse battery");
|
|
await synced.page.getByRole("button", { name: "Start syncing" }).click();
|
|
await synced.page.waitForTimeout(2500);
|
|
await synced.page.keyboard.press("Escape");
|
|
await synced.page.waitForTimeout(500);
|
|
|
|
const blob = () =>
|
|
synced.page.evaluate(async () => {
|
|
const record = JSON.parse(localStorage.getItem("freezmo.sync.v1"));
|
|
const response = await fetch(`/api/sync?id=${record.id}`, { headers: { "x-ollama-key": "test-key" } });
|
|
const body = response.ok ? await response.json() : null;
|
|
return { id: record.id, dirtyAt: record.dirtyAt, updatedAt: body?.updatedAt ?? null };
|
|
});
|
|
const beforeHomework = await blob();
|
|
check("sync: enabled on this device", /^[a-f0-9]{32}$/.test(beforeHomework.id) && beforeHomework.updatedAt !== null, JSON.stringify(beforeHomework));
|
|
|
|
// Run a check: saving the homework record must push itself (no Sync now).
|
|
await synced.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
|
await synced.page.waitForTimeout(900);
|
|
await synced.page.getByRole("button", { name: "Paste text instead" }).click();
|
|
await synced.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
|
await synced.page.getByRole("button", { name: "Next" }).click();
|
|
await synced.page.waitForTimeout(400);
|
|
await synced.page.getByRole("button", { name: /Check my work/ }).click();
|
|
await synced.page.waitForTimeout(400);
|
|
await synced.page.locator("#homework-solution").fill("x = 3");
|
|
await synced.page.getByRole("button", { name: "Check my work", exact: true }).click();
|
|
// The check run + the debounced auto-push (see lib/sync-auto, 2.5s).
|
|
await synced.page.waitForTimeout(7500);
|
|
const pushed = await blob();
|
|
check(
|
|
"sync: the new homework pushed itself",
|
|
pushed.dirtyAt === null && pushed.updatedAt !== null && pushed.updatedAt !== beforeHomework.updatedAt,
|
|
JSON.stringify({ before: beforeHomework.updatedAt, after: pushed.updatedAt }),
|
|
);
|
|
await synced.context.close();
|
|
|
|
const guide = await makePage(browser, { width: 390 });
|
|
await seed(guide.page, { decks: [] });
|
|
await guide.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
|
await guide.page.waitForTimeout(900);
|
|
await guide.page.getByRole("button", { name: "Paste text instead" }).click();
|
|
await guide.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
|
await guide.page.getByRole("button", { name: "Next" }).click();
|
|
await guide.page.waitForTimeout(400);
|
|
await guide.page.getByRole("button", { name: /Explain how to make this homework/ }).click();
|
|
await guide.page.waitForTimeout(2500);
|
|
const explainButton = guide.page.getByRole("button", { name: /Explain step 1 differently/ });
|
|
check("guide: explain action present on the step", await explainButton.isVisible().catch(() => false));
|
|
if (await explainButton.isVisible().catch(() => false)) {
|
|
await explainButton.click();
|
|
await guide.page.waitForTimeout(1500);
|
|
check("guide: explanation replaced", await guide.page.getByText("EXPLAINED DIFFERENTLY: picture a balance scale.").isVisible());
|
|
}
|
|
await guide.context.close();
|
|
}
|
|
} finally {
|
|
app.stop();
|
|
stub.close();
|
|
await browser.close();
|
|
}
|
|
|
|
finish();
|