freezmo/tests/phase3-test.mjs

/**
 * Phase 3 verification: "Check my work" (server mapping + phone and desktop
 * UI), follow-ups (shorter / more detail / explain a step differently) and
 * "Turn into a study deck" pre-filling the New deck flow.
 * Runs against a stub Ollama so no model is needed.
 */
import http from "node:http";

import { launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";

const { check, finish } = makeChecker();

const STUB_PORT = 11998;
const APP_PORT = 3208;
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
const APP = `http://127.0.0.1:${APP_PORT}`;

const CHECK_REPLY = {
  title: "Equation check",
  language: "English",
  verdict: "One mistake: the sign in the second line.",
  correct: ["The setup is right."],
  wrong: [{ issue: "x = 3", why: "4 - 2 is 2, not 3.", fix: "x = 2" }],
  corrected: "x + 2 = 4\nx = 4 - 2\nx = 2",
};
const GUIDE_REPLY = {
  title: "Equation guide",
  language: "English",
  steps: [
    {
      title: "Move the number",
      explanation: "Subtract 2 from both sides so x stands alone.",
      hints: ["Think about the opposite operation.", "Subtract 2 from both sides.", "x = 4 - 2"],
      answer: "x = 4 - 2",
    },
    {
      title: "Finish it",
      explanation: "Now do the subtraction to get the value of x.",
      hints: ["Just arithmetic now.", "4 minus 2.", "x = 2"],
      answer: "x = 2",
    },
  ],
};

const stub = http.createServer((request, response) => {
  if (request.url?.startsWith("/api/tags")) {
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
    return;
  }
  let raw = "";
  request.on("data", (chunk) => (raw += chunk));
  request.on("end", () => {
    let content = "{}";
    try {
      const body = JSON.parse(raw);
      const system = String(body.messages?.[0]?.content ?? "");
      const user = String(body.messages?.[1]?.content ?? "");
      if (system.includes("homework checker")) content = JSON.stringify(CHECK_REPLY);
      else if (system.includes("homework writer")) content = JSON.stringify({ title: "Equation", language: "English", answer: "x = 2" });
      else if (system.includes("homework coach")) content = JSON.stringify(GUIDE_REPLY);
      else if (system.includes("homework assistant")) {
        content = user.includes("SHORTER")
          ? "SHORTER: x = 2"
          : user.includes("MORE DETAIL")
            ? "MORE DETAIL: x + 2 = 4, so subtract 2: x = 2"
            : "EXPLAINED DIFFERENTLY: picture a balance scale.";
      }
    } catch {
      /* keep the default */
    }
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ message: { content } }));
  });
});
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));

check("stub app port is free", !(await portInUse(APP_PORT)));
const app = spawnApp(APP_PORT, { OLLAMA_BASE_URL: STUB_BASE, OLLAMA_AUTH: "local", OLLAMA_VERIFY_KEY: "false" });

const browser = await launchBrowser();

try {
  const up = await waitForHttp(`${APP}/api/health`);
  check("stub app instance is up", up);

  if (up) {
    /* 1. server: check mode ---------------------------------------- */
    const apiCheck = await fetch(`${APP}/api/homework`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4", solution: "x = 3" }),
    });
    const events = (await apiCheck.text())
      .trim()
      .split("\n")
      .map((line) => JSON.parse(line));
    const homeworkEvent = events.find((event) => event.type === "homework");
    check("api: check mode returns the feedback payload", homeworkEvent?.homework?.mode === "check");
    check("api: verdict and corrected version", Boolean(homeworkEvent?.homework?.verdict) && homeworkEvent?.homework?.corrected.includes("x = 2"));
    check("api: wrong entry has issue/why/fix", homeworkEvent?.homework?.wrong?.[0]?.issue === "x = 3" && Boolean(homeworkEvent?.homework?.wrong?.[0]?.fix));

    const missingSolution = await fetch(`${APP}/api/homework`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4" }),
    });
    check("api: check without a solution → 400", missingSolution.status === 400, String(missingSolution.status));

    /* 2. server: refine -------------------------------------------- */
    const refine = await fetch(`${APP}/api/homework/refine`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ instruction: "shorter", text: "x + 2 = 4, therefore x = 2" }),
    });
    const refineBody = await refine.json();
    check("api: refine returns rewritten text", refine.status === 200 && refineBody.text.includes("SHORTER"), JSON.stringify(refineBody).slice(0, 60));
    const badRefine = await fetch(`${APP}/api/homework/refine`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ instruction: "wrong", text: "x" }),
    });
    check("api: refine validates the instruction", badRefine.status === 400, String(badRefine.status));

    /* 3. phone UI: check my work + follow-ups + to-deck -------------- */
    const phone = await makePage(browser, { width: 390 });
    await seed(phone.page, { decks: [] });
    await phone.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await phone.page.waitForTimeout(900);
    await phone.page.getByRole("button", { name: "Paste text instead" }).click();
    await phone.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await phone.page.getByRole("button", { name: "Next" }).click();
    await phone.page.waitForTimeout(400);
    await phone.page.getByRole("button", { name: /Check my work/ }).click();
    await phone.page.waitForTimeout(400);
    check("phone: solution field revealed", await phone.page.locator("#homework-solution").isVisible());
    await phone.page.locator("#homework-solution").fill("x = 3");
    await phone.page.getByRole("button", { name: "Check my work", exact: true }).click();
    await phone.page.waitForTimeout(2500);
    check("phone: verdict shown", await phone.page.getByText("One mistake: the sign in the second line.").isVisible());
    check("phone: what's right listed", await phone.page.getByText("The setup is right.").isVisible());
    check("phone: what needs fixing listed", await phone.page.getByText("4 - 2 is 2, not 3.").isVisible());
    check("phone: corrected version shown", await phone.page.getByText("x = 4 - 2").isVisible());

    await phone.page.getByRole("button", { name: "Shorter" }).click();
    await phone.page.waitForTimeout(1500);
    check("phone: shorter follow-up replaces the text", await phone.page.getByText("SHORTER: x = 2").isVisible());
    check("phone: original can be restored", await phone.page.getByRole("button", { name: "Original" }).isVisible());

    await phone.page.getByRole("button", { name: "Turn into a study deck" }).click();
    await phone.page.waitForTimeout(1500);
    check("phone: landed on New deck", phone.page.url().includes("/create"));
    const seededText = await phone.page.locator("textarea").first().inputValue().catch(() => "");
    check("phone: deck wizard pre-filled with the content", seededText.includes("x = 4 - 2") || seededText.includes("SHORTER"), seededText.slice(0, 40));
    await phone.context.close();

    /* 4. desktop UI + guide explain ---------------------------------- */
    const desktop = await makePage(browser, { width: 1024, height: 900 });
    await seed(desktop.page, { decks: [] });
    await desktop.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await desktop.page.waitForTimeout(900);
    await desktop.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await desktop.page.getByRole("button", { name: /Check my work/ }).click();
    await desktop.page.waitForTimeout(300);
    check("desktop: solution field appears with the mode", await desktop.page.locator("#homework-solution").isVisible());
    await desktop.page.locator("#homework-solution").fill("x = 3");
    await desktop.page.getByRole("button", { name: "Check my work", exact: true }).click();
    await desktop.page.waitForTimeout(2500);
    check("desktop: check result renders", await desktop.page.getByText("Corrected version").isVisible());
    await desktop.context.close();

    const guide = await makePage(browser, { width: 390 });
    await seed(guide.page, { decks: [] });
    await guide.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
    await guide.page.waitForTimeout(900);
    await guide.page.getByRole("button", { name: "Paste text instead" }).click();
    await guide.page.locator("textarea").first().fill("Solve x + 2 = 4");
    await guide.page.getByRole("button", { name: "Next" }).click();
    await guide.page.waitForTimeout(400);
    await guide.page.getByRole("button", { name: /Explain how to make this homework/ }).click();
    await guide.page.waitForTimeout(2500);
    const explainButton = guide.page.getByRole("button", { name: /Explain step 1 differently/ });
    check("guide: explain action present on the step", await explainButton.isVisible().catch(() => false));
    if (await explainButton.isVisible().catch(() => false)) {
      await explainButton.click();
      await guide.page.waitForTimeout(1500);
      check("guide: explanation replaced", await guide.page.getByText("EXPLAINED DIFFERENTLY: picture a balance scale.").isVisible());
    }
    await guide.context.close();
  }
} finally {
  app.stop();
  stub.close();
  await browser.close();
}

finish();