freezmo/tests/phase3-test.mjs

/**
 * Phase 3 verification: "Check my work" (server mapping + phone and desktop
 * UI), follow-ups (shorter / more detail / explain a step differently) and
 * "Turn into a study deck" pre-filling the New deck flow.
 * Runs against a stub Ollama so no model is needed.
 */
import http from "node:http";
import { mkdirSync } from "node:fs";

import { launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";

const { check, finish } = makeChecker();

const STUB_PORT = 11998;
const APP_PORT = 3208;
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
const APP = `http://127.0.0.1:${APP_PORT}`;

const CHECK_REPLY = {
  title: "Equation check",
  language: "English",
  verdict: "One mistake: the sign in the second line.",
  correct: ["The setup is right."],
  wrong: [{ issue: "x = 3", why: "4 - 2 is 2, not 3.", fix: "x = 2" }],
  corrected: "x + 2 = 4\nx = 4 - 2\nx = 2",
};
const GUIDE_REPLY = {
  title: "Equation guide",
  language: "English",
  steps: [
    {
      title: "Move the number",
      explanation: "Subtract 2 from both sides so x stands alone.",
      hints: ["Think about the opposite operation.", "Subtract 2 from both sides.", "x = 4 - 2"],
      answer: "x = 4 - 2",
    },
    {
      title: "Finish it",
      explanation: "Now do the subtraction to get the value of x.",
      hints: ["Just arithmetic now.", "4 minus 2.", "x = 2"],
      answer: "x = 2",
    },
  ],
};

const stub = http.createServer((request, response) => {
  if (request.url?.startsWith("/api/tags")) {
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
    return;
  }
  let raw = "";
  request.on("data", (chunk) => (raw += chunk));
  request.on("end", () => {
    let content = "{}";
    try {
      const body = JSON.parse(raw);
      const system = String(body.messages?.[0]?.content ?? "");
      const user = String(body.messages?.[1]?.content ?? "");
      if (system.includes("study-note transcriber")) {
        content = JSON.stringify({
          title: "Attempt",
          language: "English",
          notes: "x = 3 (from the photo)",
          keyConcepts: ["linear equation"],
        });
      } else if (system.includes("homework checker")) content = JSON.stringify(CHECK_REPLY);
      else if (system.includes("homework writer")) content = JSON.stringify({ title: "Equation", language: "English", answer: "x = 2" });
      else if (system.includes("homework coach")) content = JSON.stringify(GUIDE_REPLY);
      else if (system.includes("homework assistant")) {
        content = user.includes("SHORTER")
          ? "SHORTER: x = 2"
          : user.includes("MORE DETAIL")
            ? "MORE DETAIL: x + 2 = 4, so subtract 2: x = 2"
            : "EXPLAINED DIFFERENTLY: picture a balance scale.";
      }
    } catch {
      /* keep the default */
    }
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ message: { content } }));
  });
});
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));

check("stub app port is free", !(await portInUse(APP_PORT)));
const syncDir = new URL("./.tmp/sync-phase3", import.meta.url).pathname;
mkdirSync(syncDir, { recursive: true });
const app = spawnApp(APP_PORT, {
  OLLAMA_BASE_URL: STUB_BASE,
  OLLAMA_AUTH: "local",
  OLLAMA_VERIFY_KEY: "false",
  SYNC_DIR: syncDir,
});

const browser = await launchBrowser();

try {
  const up = await waitForHttp(`${APP}/api/health`);
  check("stub app instance is up", up);

  if (up) {
    /* 1. server: check mode ---------------------------------------- */
    const apiCheck = await fetch(`${APP}/api/homework`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4", solution: "x = 3" }),
    });
    const events = (await apiCheck.text())
      .trim()
      .split("\n")
      .map((line) => JSON.parse(line));
    const homeworkEvent = events.find((event) => event.type === "homework");
    check("api: check mode returns the feedback payload", homeworkEvent?.homework?.mode === "check");
    check("api: verdict and corrected version", Boolean(homeworkEvent?.homework?.verdict) && homeworkEvent?.homework?.corrected.includes("x = 2"));
    check("api: wrong entry has issue/why/fix", homeworkEvent?.homework?.wrong?.[0]?.issue === "x = 3" && Boolean(homeworkEvent?.homework?.wrong?.[0]?.fix));

    const missingSolution = await fetch(`${APP}/api/homework`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4" }),
    });
    check("api: check without a solution → 400", missingSolution.status === 400, String(missingSolution.status));

    /* Photos of the attempt: transcribed by the vision model first, then the
       check runs on typed text + transcription. */
    const photoCheck = await fetch(`${APP}/api/homework`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({
        mode: "check",
        text: "Solve x + 2 = 4",
        solutionImages: [`data:image/jpeg;base64,${"A".repeat(2000)}`],
      }),
    });
    const photoEvents = (await photoCheck.text())
      .trim()
      .split("\n")
      .map((line) => JSON.parse(line));
    const photoHomework = photoEvents.find((event) => event.type === "homework");
    check(
      "api: solution photos are transcribed into the attempt",
      photoHomework?.meta?.solutionText?.includes("from the photo") && photoHomework?.homework?.mode === "check",
      JSON.stringify(photoHomework?.meta ?? {}).slice(0, 90),
    );
    check(
      "api: vision ran before the check",
      photoEvents.some((event) => event.type === "stage" && event.message.includes("solution")),
      JSON.stringify(photoEvents.filter((event) => event.type === "stage").map((event) => event.stage)),
    );

    /* 2. server: refine -------------------------------------------- */
    const refine = await fetch(`${APP}/api/homework/refine`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ instruction: "shorter", text: "x + 2 = 4, therefore x = 2" }),
    });
    const refineBody = await refine.json();
    check("api: refine returns rewritten text", refine.status === 200 && refineBody.text.includes("SHORTER"), JSON.stringify(refineBody).slice(0, 60));
    const badRefine = await fetch(`${APP}/api/homework/refine`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ instruction: "wrong", text: "x" }),
    });
    check("api: refine validates the instruction", badRefine.status === 400, String(badRefine.status));

    /* 3. phone UI: check my work + follow-ups + to-deck -------------- */
    const phone = await makePage(browser, { width: 390 });
    await seed(phone.page, { decks: [] });
    await phone.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await phone.page.waitForTimeout(900);
    await phone.page.getByRole("button", { name: "Paste text instead" }).click();
    await phone.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await phone.page.getByRole("button", { name: "Next" }).click();
    await phone.page.waitForTimeout(400);
    check("phone: AI disclaimer near the start cards", await phone.page.getByText("Freezmo uses AI and can make mistakes").isVisible());
    await phone.page.getByRole("button", { name: /Check my work/ }).click();
    await phone.page.waitForTimeout(400);
    check("phone: solution field revealed", await phone.page.locator("#homework-solution").isVisible());
    await phone.page.locator("#homework-solution").fill("x = 3");
    await phone.page.getByRole("button", { name: "Check my work", exact: true }).click();
    await phone.page.waitForTimeout(2500);
    check("phone: verdict shown", await phone.page.getByText("One mistake: the sign in the second line.").isVisible());
    check("phone: what's right listed", await phone.page.getByText("The setup is right.").isVisible());
    check("phone: what needs fixing listed", await phone.page.getByText("4 - 2 is 2, not 3.").isVisible());
    check("phone: corrected version shown", await phone.page.getByText("x = 4 - 2").isVisible());
    check("phone: AI disclaimer on the result", await phone.page.getByText("Freezmo uses AI and can make mistakes").isVisible());

    await phone.page.getByRole("button", { name: "Shorter" }).click();
    await phone.page.waitForTimeout(1500);
    check("phone: shorter follow-up replaces the text", await phone.page.getByText("SHORTER: x = 2").isVisible());
    check("phone: original can be restored", await phone.page.getByRole("button", { name: "Original" }).isVisible());

    check(
      "phone: the result carries the report control",
      await phone.page.getByRole("button", { name: "Report exercise" }).isVisible(),
    );
    await phone.page.getByRole("button", { name: "Turn into a study deck" }).click();
    await phone.page.waitForTimeout(1500);
    check("phone: landed on New deck", phone.page.url().includes("/create"));
    const seededText = await phone.page.locator("textarea").first().inputValue().catch(() => "");
    check("phone: deck wizard pre-filled with the content", seededText.includes("x = 4 - 2") || seededText.includes("SHORTER"), seededText.slice(0, 40));
    await phone.context.close();

    /* 4. desktop UI + guide explain ---------------------------------- */
    const desktop = await makePage(browser, { width: 1024, height: 900 });
    await seed(desktop.page, { decks: [] });
    const checkBodies = [];
    desktop.page.on("request", (request) => {
      if (request.method() === "POST" && request.url().includes("/api/homework")) {
        checkBodies.push(request.postData() ?? "");
      }
    });
    await desktop.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await desktop.page.waitForTimeout(900);
    await desktop.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await desktop.page.getByRole("button", { name: /Check my work/ }).click();
    await desktop.page.waitForTimeout(300);
    check("desktop: solution field appears with the mode", await desktop.page.locator("#homework-solution").isVisible());
    await desktop.page.locator("#homework-solution").fill("x = 3");
    /* A photo of the work travels with the attempt (handwriting, scans). */
    await desktop.page.locator('#homework-solution ~ input[type="file"]').first().setInputFiles({
      name: "attempt.png",
      mimeType: "image/png",
      // 1x1 PNG
      buffer: Buffer.from(
        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
        "base64",
      ),
    });
    await desktop.page.waitForTimeout(1200);
    check("desktop: attached photo gets a thumbnail", (await desktop.page.locator('img[src^="data:image"]').count()) >= 1);
    await desktop.page.getByRole("button", { name: "Check my work", exact: true }).click();
    await desktop.page.waitForTimeout(3000);
    check("desktop: check result renders", await desktop.page.getByText("Corrected version").isVisible());
    check("desktop: the attempt travelled with the photo", checkBodies.some((body) => body.includes("solutionImages")));
    check("desktop: AI disclaimer on the result", await desktop.page.getByText("Freezmo uses AI and can make mistakes").isVisible());
    await desktop.context.close();

    /* 4b. a new homework pushes itself when sync is on ---------------- */
    const synced = await makePage(browser, { width: 390 });
    await seed(synced.page, { decks: [] });
    await synced.page.goto(`${APP}/`, { waitUntil: "networkidle" });
    await synced.page.waitForTimeout(800);
    await synced.page.locator("nav.tabbar").getByRole("button").click();
    await synced.page.getByRole("dialog").waitFor({ timeout: 6000 });
    await synced.page.waitForTimeout(500);
    await synced.page.getByRole("button", { name: "Turn on sync" }).click();
    await synced.page.getByPlaceholder("Sync passphrase").fill("correct horse battery");
    await synced.page.getByPlaceholder("Repeat the passphrase").fill("correct horse battery");
    await synced.page.getByRole("button", { name: "Start syncing" }).click();
    await synced.page.waitForTimeout(2500);
    await synced.page.keyboard.press("Escape");
    await synced.page.waitForTimeout(500);

    const blob = () =>
      synced.page.evaluate(async () => {
        const record = JSON.parse(localStorage.getItem("freezmo.sync.v1"));
        const response = await fetch(`/api/sync?id=${record.id}`, { headers: { "x-ollama-key": "test-key" } });
        const body = response.ok ? await response.json() : null;
        return { id: record.id, dirtyAt: record.dirtyAt, updatedAt: body?.updatedAt ?? null };
      });
    const beforeHomework = await blob();
    check("sync: enabled on this device", /^[a-f0-9]{32}$/.test(beforeHomework.id) && beforeHomework.updatedAt !== null, JSON.stringify(beforeHomework));

    // Run a check: saving the homework record must push itself (no Sync now).
    await synced.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await synced.page.waitForTimeout(900);
    await synced.page.getByRole("button", { name: "Paste text instead" }).click();
    await synced.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await synced.page.getByRole("button", { name: "Next" }).click();
    await synced.page.waitForTimeout(400);
    await synced.page.getByRole("button", { name: /Check my work/ }).click();
    await synced.page.waitForTimeout(400);
    await synced.page.locator("#homework-solution").fill("x = 3");
    await synced.page.getByRole("button", { name: "Check my work", exact: true }).click();
    // The check run + the debounced auto-push (see lib/sync-auto, 2.5s).
    await synced.page.waitForTimeout(7500);
    const pushed = await blob();
    check(
      "sync: the new homework pushed itself",
      pushed.dirtyAt === null && pushed.updatedAt !== null && pushed.updatedAt !== beforeHomework.updatedAt,
      JSON.stringify({ before: beforeHomework.updatedAt, after: pushed.updatedAt }),
    );
    await synced.context.close();

    const guide = await makePage(browser, { width: 390 });
    await seed(guide.page, { decks: [] });
    await guide.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await guide.page.waitForTimeout(900);
    await guide.page.getByRole("button", { name: "Paste text instead" }).click();
    await guide.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await guide.page.getByRole("button", { name: "Next" }).click();
    await guide.page.waitForTimeout(400);
    await guide.page.getByRole("button", { name: /Explain how to make this homework/ }).click();
    await guide.page.waitForTimeout(2500);
    const explainButton = guide.page.getByRole("button", { name: /Explain step 1 differently/ });
    check("guide: explain action present on the step", await explainButton.isVisible().catch(() => false));
    if (await explainButton.isVisible().catch(() => false)) {
      await explainButton.click();
      await guide.page.waitForTimeout(1500);
      check("guide: explanation replaced", await guide.page.getByText("EXPLAINED DIFFERENTLY: picture a balance scale.").isVisible());
    }
    await guide.context.close();
  }
} finally {
  app.stop();
  stub.close();
  await browser.close();
}

finish();