/** * Phase 3 verification: "Check my work" (server mapping + phone and desktop * UI), follow-ups (shorter / more detail / explain a step differently) and * "Turn into a study deck" pre-filling the New deck flow. * Runs against a stub Ollama so no model is needed. */ import http from "node:http"; import { launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs"; const { check, finish } = makeChecker(); const STUB_PORT = 11998; const APP_PORT = 3208; const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`; const APP = `http://127.0.0.1:${APP_PORT}`; const CHECK_REPLY = { title: "Equation check", language: "English", verdict: "One mistake: the sign in the second line.", correct: ["The setup is right."], wrong: [{ issue: "x = 3", why: "4 - 2 is 2, not 3.", fix: "x = 2" }], corrected: "x + 2 = 4\nx = 4 - 2\nx = 2", }; const GUIDE_REPLY = { title: "Equation guide", language: "English", steps: [ { title: "Move the number", explanation: "Subtract 2 from both sides so x stands alone.", hints: ["Think about the opposite operation.", "Subtract 2 from both sides.", "x = 4 - 2"], answer: "x = 4 - 2", }, { title: "Finish it", explanation: "Now do the subtraction to get the value of x.", hints: ["Just arithmetic now.", "4 minus 2.", "x = 2"], answer: "x = 2", }, ], }; const stub = http.createServer((request, response) => { if (request.url?.startsWith("/api/tags")) { response.writeHead(200, { "Content-Type": "application/json" }); response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] })); return; } let raw = ""; request.on("data", (chunk) => (raw += chunk)); request.on("end", () => { let content = "{}"; try { const body = JSON.parse(raw); const system = String(body.messages?.[0]?.content ?? ""); const user = String(body.messages?.[1]?.content ?? ""); if (system.includes("homework checker")) content = JSON.stringify(CHECK_REPLY); else if (system.includes("homework writer")) content = JSON.stringify({ title: "Equation", language: "English", answer: "x = 2" }); else if (system.includes("homework coach")) content = JSON.stringify(GUIDE_REPLY); else if (system.includes("homework assistant")) { content = user.includes("SHORTER") ? "SHORTER: x = 2" : user.includes("MORE DETAIL") ? "MORE DETAIL: x + 2 = 4, so subtract 2: x = 2" : "EXPLAINED DIFFERENTLY: picture a balance scale."; } } catch { /* keep the default */ } response.writeHead(200, { "Content-Type": "application/json" }); response.end(JSON.stringify({ message: { content } })); }); }); await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve)); check("stub app port is free", !(await portInUse(APP_PORT))); const app = spawnApp(APP_PORT, { OLLAMA_BASE_URL: STUB_BASE, OLLAMA_AUTH: "local", OLLAMA_VERIFY_KEY: "false" }); const browser = await launchBrowser(); try { const up = await waitForHttp(`${APP}/api/health`); check("stub app instance is up", up); if (up) { /* 1. server: check mode ---------------------------------------- */ const apiCheck = await fetch(`${APP}/api/homework`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4", solution: "x = 3" }), }); const events = (await apiCheck.text()) .trim() .split("\n") .map((line) => JSON.parse(line)); const homeworkEvent = events.find((event) => event.type === "homework"); check("api: check mode returns the feedback payload", homeworkEvent?.homework?.mode === "check"); check("api: verdict and corrected version", Boolean(homeworkEvent?.homework?.verdict) && homeworkEvent?.homework?.corrected.includes("x = 2")); check("api: wrong entry has issue/why/fix", homeworkEvent?.homework?.wrong?.[0]?.issue === "x = 3" && Boolean(homeworkEvent?.homework?.wrong?.[0]?.fix)); const missingSolution = await fetch(`${APP}/api/homework`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4" }), }); check("api: check without a solution → 400", missingSolution.status === 400, String(missingSolution.status)); /* 2. server: refine -------------------------------------------- */ const refine = await fetch(`${APP}/api/homework/refine`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ instruction: "shorter", text: "x + 2 = 4, therefore x = 2" }), }); const refineBody = await refine.json(); check("api: refine returns rewritten text", refine.status === 200 && refineBody.text.includes("SHORTER"), JSON.stringify(refineBody).slice(0, 60)); const badRefine = await fetch(`${APP}/api/homework/refine`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ instruction: "wrong", text: "x" }), }); check("api: refine validates the instruction", badRefine.status === 400, String(badRefine.status)); /* 3. phone UI: check my work + follow-ups + to-deck -------------- */ const phone = await makePage(browser, { width: 390 }); await seed(phone.page, { decks: [] }); await phone.page.goto(`${APP}/homework`, { waitUntil: "networkidle" }); await phone.page.waitForTimeout(900); await phone.page.getByRole("button", { name: "Paste text instead" }).click(); await phone.page.locator("textarea").first().fill("Solve x + 2 = 4"); await phone.page.getByRole("button", { name: "Next" }).click(); await phone.page.waitForTimeout(400); check("phone: AI disclaimer near the start cards", await phone.page.getByText("Freezmo uses AI and can make mistakes").isVisible()); await phone.page.getByRole("button", { name: /Check my work/ }).click(); await phone.page.waitForTimeout(400); check("phone: solution field revealed", await phone.page.locator("#homework-solution").isVisible()); await phone.page.locator("#homework-solution").fill("x = 3"); await phone.page.getByRole("button", { name: "Check my work", exact: true }).click(); await phone.page.waitForTimeout(2500); check("phone: verdict shown", await phone.page.getByText("One mistake: the sign in the second line.").isVisible()); check("phone: what's right listed", await phone.page.getByText("The setup is right.").isVisible()); check("phone: what needs fixing listed", await phone.page.getByText("4 - 2 is 2, not 3.").isVisible()); check("phone: corrected version shown", await phone.page.getByText("x = 4 - 2").isVisible()); check("phone: AI disclaimer on the result", await phone.page.getByText("Freezmo uses AI and can make mistakes").isVisible()); await phone.page.getByRole("button", { name: "Shorter" }).click(); await phone.page.waitForTimeout(1500); check("phone: shorter follow-up replaces the text", await phone.page.getByText("SHORTER: x = 2").isVisible()); check("phone: original can be restored", await phone.page.getByRole("button", { name: "Original" }).isVisible()); await phone.page.getByRole("button", { name: "Turn into a study deck" }).click(); await phone.page.waitForTimeout(1500); check("phone: landed on New deck", phone.page.url().includes("/create")); const seededText = await phone.page.locator("textarea").first().inputValue().catch(() => ""); check("phone: deck wizard pre-filled with the content", seededText.includes("x = 4 - 2") || seededText.includes("SHORTER"), seededText.slice(0, 40)); await phone.context.close(); /* 4. desktop UI + guide explain ---------------------------------- */ const desktop = await makePage(browser, { width: 1024, height: 900 }); await seed(desktop.page, { decks: [] }); await desktop.page.goto(`${APP}/homework`, { waitUntil: "networkidle" }); await desktop.page.waitForTimeout(900); await desktop.page.locator("textarea").first().fill("Solve x + 2 = 4"); await desktop.page.getByRole("button", { name: /Check my work/ }).click(); await desktop.page.waitForTimeout(300); check("desktop: solution field appears with the mode", await desktop.page.locator("#homework-solution").isVisible()); await desktop.page.locator("#homework-solution").fill("x = 3"); await desktop.page.getByRole("button", { name: "Check my work", exact: true }).click(); await desktop.page.waitForTimeout(2500); check("desktop: check result renders", await desktop.page.getByText("Corrected version").isVisible()); check("desktop: AI disclaimer on the result", await desktop.page.getByText("Freezmo uses AI and can make mistakes").isVisible()); await desktop.context.close(); const guide = await makePage(browser, { width: 390 }); await seed(guide.page, { decks: [] }); await guide.page.goto(`${APP}/homework`, { waitUntil: "networkidle" }); await guide.page.waitForTimeout(900); await guide.page.getByRole("button", { name: "Paste text instead" }).click(); await guide.page.locator("textarea").first().fill("Solve x + 2 = 4"); await guide.page.getByRole("button", { name: "Next" }).click(); await guide.page.waitForTimeout(400); await guide.page.getByRole("button", { name: /Explain how to make this homework/ }).click(); await guide.page.waitForTimeout(2500); const explainButton = guide.page.getByRole("button", { name: /Explain step 1 differently/ }); check("guide: explain action present on the step", await explainButton.isVisible().catch(() => false)); if (await explainButton.isVisible().catch(() => false)) { await explainButton.click(); await guide.page.waitForTimeout(1500); check("guide: explanation replaced", await guide.page.getByText("EXPLAINED DIFFERENTLY: picture a balance scale.").isVisible()); } await guide.context.close(); } } finally { app.stop(); stub.close(); await browser.close(); } finish();