freezmo/tests/phase2-test.mjs

/**
 * Phase 2 verification: answer history in IndexedDB, SRS review card and
 * session, mistakes that clear themselves, the stats screen, exam mode with a
 * timer, and the push-reminder plumbing (hidden without VAPID, validated with
 * it).
 */
import { mkdirSync, readFileSync } from "node:fs";
import { spawn } from "node:child_process";
import path from "node:path";

import {
  BASE,
  DECK,
  REPO,
  launchBrowser,
  makeChecker,
  makePage,
  portInUse,
  readLearning,
  seed,
  seedLearning,
  spawnApp,
  waitForHttp,
} from "./lib.mjs";

const { check, finish } = makeChecker();

const browser = await launchBrowser();

/* 1. studying records history, mistakes and a schedule --------------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [DECK] });
  await page.goto(`${BASE}/deck/seed_deck`, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);

  /* 0. the study chrome is one pill that its controls fit -------------- */
  const chrome = await page.evaluate(() => {
    const pill = document.querySelector("header .study-chrome");
    if (!pill) return null;
    const box = pill.getBoundingClientRect();
    const radius = Number.parseFloat(getComputedStyle(pill).borderTopLeftRadius) || 0;
    const control = pill.querySelector("a, button").getBoundingClientRect();
    const centreY = box.top + box.height / 2;
    return {
      height: Math.round(box.height),
      fullyRounded: radius >= box.height / 2 - 1,
      /** The control's far corner stays inside the stadium curve. */
      cornerInside: Math.hypot(control.left - (box.left + box.height / 2), control.top - centreY) < box.height / 2,
      counter: /\d+\s*\/\s*\d+/.test(pill.textContent ?? ""),
    };
  });
  check(
    "chrome: one rounded pill, controls inside its ends",
    chrome !== null && chrome.fullyRounded && chrome.cornerInside && chrome.counter && chrome.height <= 70,
    JSON.stringify(chrome),
  );

  await page.getByRole("button", { name: "Green" }).click();
  await page.waitForTimeout(500);
  check("study: wrong answer shows feedback", await page.getByText(/Correct answer/i).first().isVisible());
  await page.getByRole("button", { name: "Next" }).click();
  await page.waitForTimeout(900);
  await page.getByRole("button", { name: "Scattering" }).click();
  await page.waitForTimeout(500);
  await page.getByRole("button", { name: "Next" }).click();
  await page.waitForTimeout(900);

  const stored = await readLearning(page);
  check("history: two answers recorded", stored.answers.length === 2, String(stored.answers.length));
  check("history: correct and wrong kept", stored.answers.some((a) => a.correct) && stored.answers.some((a) => !a.correct));
  check("history: topic kept", stored.answers.every((a) => a.topic === "Sky"));
  check("srs: schedule created for both", stored.srs.length === 2, String(stored.srs.length));
  const wrong = stored.srs.find((entry) => entry.activityId === "a1");
  const right = stored.srs.find((entry) => entry.activityId === "a2");
  check("srs: wrong comes back in ~10 min", wrong && wrong.dueAt - wrong.lastAt === 10 * 60_000);
  check("srs: correct scheduled a day out", right && right.intervalDays === 1 && right.dueAt - right.lastAt === 86_400_000);

  await page.goto(BASE, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  check("library: learning card visible", await page.getByText("Smart review").isVisible());
  check("library: one mistake", await page.getByText("Mistakes · 1 mistake").isVisible());
  check("library: review due is zero", await page.getByText("Review due · 0 due").isVisible());
  check("library: today's goal line", await page.getByText(/Today: 2 \/ 20/).isVisible());

  await page.getByText("Mistakes · 1 mistake").click();
  await page.waitForTimeout(1200);
  check("mistakes: session opens on the wrong item", await page.getByText("What colour is a clear daytime sky?").isVisible());
  await page.getByRole("button", { name: "Blue" }).click();
  await page.waitForTimeout(400);
  await page.getByRole("button", { name: "See results" }).click();
  await page.waitForTimeout(600);
  await page.getByRole("button", { name: "Back to the Library" }).click();
  await page.waitForTimeout(1000);
  check("mistakes: cleared after a correct answer", await page.getByText("Mistakes · 0 mistakes").isVisible());
  await context.close();
}

/* 1b. AI grading feedback carries the disclaimer (stubbed model call) - */
{
  const { context, page } = await makePage(browser);
  const shortDeck = {
    ...DECK,
    id: "seed_short",
    title: "Short answers",
    // Two activities on purpose: with one, grading ends the session and the
    // results screen replaces the graded view before the check runs.
    activities: [
      {
        type: "short_answer",
        id: "s1",
        xp: 10,
        topic: "Sky",
        question: "Why is the sky blue?",
        referenceAnswer: "Rayleigh scattering favours short wavelengths.",
        keyPoints: ["scattering", "short wavelengths"],
        explanation: "Small molecules scatter blue light more than red.",
      },
      {
        type: "multiple_choice",
        id: "s2",
        xp: 10,
        topic: "Sky",
        question: "Which colour scatters most?",
        options: ["Blue", "Red", "Green", "Yellow"],
        correctIndex: 0,
        explanation: "Short wavelengths scatter more.",
      },
    ],
  };
  await page.route("**/api/grade", (route) =>
    route.fulfill({
      status: 200,
      contentType: "application/json",
      body: JSON.stringify({ correct: true, feedback: "Nice — scattering does the work.", missing: [] }),
    }),
  );
  await seed(page, { decks: [shortDeck] });
  await page.goto(`${BASE}/deck/seed_short`, { waitUntil: "networkidle" });
  await page.waitForTimeout(1000);
  await page.locator("textarea").fill("Because small molecules scatter blue light more.");
  await page.getByRole("button", { name: "Submit answer" }).click();
  await page.waitForTimeout(800);
  check("grading: AI feedback rendered", await page.getByText("Nice — scattering does the work.").isVisible());
  // The disclaimer sits below the graded feedback (the feed's row carries it;
  // the activity no longer duplicates it right after the grading box).
  const feedbackDisclaimer = await page.evaluate(() => {
    const feedback = [...document.querySelectorAll("p")].find((el) => el.textContent.includes("Nice — scattering does the work."));
    const note = [...document.querySelectorAll("p")].find((el) => el.textContent.includes("Freezmo uses AI"));
    if (!feedback || !note) return false;
    return note.getBoundingClientRect().top > feedback.getBoundingClientRect().bottom - 4;
  });
  check("grading: AI disclaimer below the feedback", feedbackDisclaimer);
  await context.close();
}

/* 1c. report question: skips, records, hides, undo ------------------- */
{
  const { context, page } = await makePage(browser);
  const posts = [];
  await page.route("**/api/reports", (route) => {
    posts.push(JSON.parse(route.request().postData() ?? "{}"));
    return route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ ok: true }) });
  });
  await seed(page, { decks: [DECK] });
  await page.goto(`${BASE}/deck/seed_deck`, { waitUntil: "networkidle" });
  await page.waitForTimeout(1000);

  const report = page.getByRole("button", { name: "Report question" });
  check("report: button visible", await report.isVisible());
  // It belongs under the card now: bottom-right corner of the exercise, not
  // in the top chrome where it was easy to mis-tap.
  const placement = await page.evaluate(() => {
    const prompt = [...document.querySelectorAll("p")].find(
      (el) => el.textContent === "What colour is a clear daytime sky?",
    );
    const card = prompt?.closest(".glass-solid");
    const button = document.querySelector('[aria-label="Report question"]');
    if (!card || !button) return null;
    const c = card.getBoundingClientRect();
    const b = button.getBoundingClientRect();
    return {
      inChrome: Boolean(button.closest("header")),
      belowCard: b.top >= c.bottom - 6,
      rightAligned: Math.abs(b.right - c.right) <= 10,
      rightSide: b.left + b.width / 2 > c.left + c.width / 2,
      size: [Math.round(b.width), Math.round(b.height)],
    };
  });
  check(
    "report: sits bottom-right under the card, 44px",
    Boolean(placement) &&
      !placement.inChrome &&
      placement.belowCard &&
      placement.rightAligned &&
      placement.rightSide &&
      placement.size[0] >= 43.5 &&
      placement.size[1] >= 43.5,
    JSON.stringify(placement),
  );
  await report.click();
  await page.waitForTimeout(700);
  check("report: toast with Undo", await page.getByText("Question reported").isVisible());
  check(
    "report: sent to the server",
    posts.length === 1 && posts[0].activityId === "a1" && posts[0].question.includes("What colour"),
    JSON.stringify(posts[0] ?? {}).slice(0, 80),
  );
  const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.reports.v1") ?? "[]"));
  check("report: kept locally", stored.length === 1 && stored[0].activityId === "a1");
  check("report: skips to the next activity", await page.getByText("2/2").isVisible());

  // A fresh session leaves the reported question out.
  await page.goto(`${BASE}/deck/seed_deck`, { waitUntil: "networkidle" });
  await page.waitForTimeout(1000);
  check(
    "report: hidden in the next session",
    (await page.getByText("What colour is a clear daytime sky?").count()) === 0 &&
      (await page.getByText("What makes the sky blue?").count()) >= 1,
  );

  // Undo takes a report back (the earlier one stays).
  await page.goto(`${BASE}/deck/seed_deck`, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  await page.getByRole("button", { name: "Report question" }).click();
  await page.waitForTimeout(400);
  const before = await page.evaluate(() =>
    JSON.parse(localStorage.getItem("freezmo.reports.v1") ?? "[]").map((report) => report.activityId),
  );
  await page.getByRole("button", { name: "Undo" }).click();
  await page.waitForTimeout(300);
  const after = await page.evaluate(() =>
    JSON.parse(localStorage.getItem("freezmo.reports.v1") ?? "[]").map((report) => report.activityId),
  );
  check("report: Undo takes the report back", before.includes("a2") && !after.includes("a2") && after.includes("a1"), JSON.stringify({ before, after }));
  await context.close();
}

/* 1d. flashcard flip paints exactly one face ------------------------- */
{
  const { context, page } = await makePage(browser);
  const flashDeck = {
    ...DECK,
    id: "seed_flash",
    title: "Flashcards",
    activities: [
      { type: "flashcard", id: "f1", xp: 10, topic: "Sky", front: "Why is the sky blue?", back: "Rayleigh scattering.", explanation: "e" },
    ],
  };
  await seed(page, { decks: [flashDeck] });
  await page.goto(`${BASE}/deck/seed_flash`, { waitUntil: "networkidle" });
  await page.waitForTimeout(1000);

  const faces = () =>
    page.evaluate(() => {
      const motion = document.querySelector("button.group > div");
      const [front, back] = motion ? [...motion.children] : [];
      const visibility = (element) => (element ? getComputedStyle(element).visibility : null);
      return { front: visibility(front), back: visibility(back) };
    });

  const atRest = await faces();
  check("flashcard: front face painted at rest", atRest.front === "visible" && atRest.back === "hidden", JSON.stringify(atRest));
  check(
    "feed: AI disclaimer under the card",
    (await page.getByText("Freezmo uses AI and can make mistakes").count()) >= 1,
  );
  await page.locator("button.group").first().click();
  await page.waitForTimeout(900);
  const flipped = await faces();
  check(
    "flashcard: flip paints the back face only (never mirrored)",
    flipped.front === "hidden" && flipped.back === "visible",
    JSON.stringify(flipped),
  );
  check("flashcard: the answer reads inside the card", await page.locator("button.group").getByText("Rayleigh scattering.").isVisible());
  await page.locator("button.group").first().click();
  await page.waitForTimeout(1000);
  const back = await faces();
  check("flashcard: flipping back restores the question face", back.front === "visible" && back.back === "hidden", JSON.stringify(back));

  // Answer it and check the results slide carries the disclaimer too.
  await page.locator("button.group").first().click();
  await page.waitForTimeout(800);
  await page.getByRole("button", { name: "I knew it" }).click();
  await page.waitForTimeout(500);
  await page.getByRole("button", { name: "See results" }).click();
  await page.waitForTimeout(800);
  const resultsDisclaimer = await page.evaluate(() => {
    const slides = [...document.querySelectorAll("[data-testid='study-feed'] section")];
    return slides[slides.length - 1]?.textContent?.includes("Freezmo uses AI") ?? false;
  });
  check("results: AI disclaimer on the results slide", resultsDisclaimer);
  await context.close();
}

/* 2. SRS due → review session; stats screen -------------------------- */
{
  const now = Date.now();
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [DECK] });
  await page.goto(BASE, { waitUntil: "networkidle" });
  await page.waitForTimeout(600);
  await seedLearning(page, {
    answers: [
      { id: "h1", deckId: "seed_deck", deckTitle: "Sky deck", activityId: "a1", type: "multiple_choice", topic: "Sky", correct: false, at: now - 3000 },
      { id: "h2", deckId: "seed_deck", deckTitle: "Sky deck", activityId: "a2", type: "multiple_choice", topic: "Sky", correct: true, at: now - 2000 },
      { id: "h3", deckId: "seed_deck", deckTitle: "Sky deck", activityId: "a1", type: "multiple_choice", topic: "Sky", correct: true, at: now - 1000 },
    ],
    srs: [{ key: "seed_deck:a1", deckId: "seed_deck", activityId: "a1", ease: 2.5, reps: 1, intervalDays: 1, lapses: 0, dueAt: now - 60_000, lastAt: now - 1000, lastCorrect: true }],
  });
  await page.reload({ waitUntil: "networkidle" });
  await page.waitForTimeout(1000);
  check("review: due count from the schedule", await page.getByText("Review due · 1 due").isVisible());
  await page.getByText("Review due · 1 due").click();
  await page.waitForTimeout(1200);
  check("review: session shows the due activity", await page.getByText("What colour is a clear daytime sky?").isVisible());
  await page.getByRole("button", { name: "Blue" }).click();
  await page.waitForTimeout(400);
  await page.getByRole("button", { name: "See results" }).click();
  await page.waitForTimeout(600);
  check("review: results slide", await page.getByText("100%").isVisible());
  await page.getByRole("button", { name: "Back to the Library" }).click();
  await page.waitForTimeout(900);

  await page.goto(`${BASE}/stats`, { waitUntil: "networkidle" });
  await page.waitForTimeout(1200);
  check("stats: title", await page.getByRole("heading", { name: "Your progress" }).isVisible());
  const streakValue = await page.locator("section").filter({ hasText: "Day streak" }).first().innerText();
  check("stats: streak counts today", /\b1\b/.test(streakValue), streakValue.replace(/\s+/g, " ").slice(0, 40));
  check("stats: calendar rendered", (await page.locator('[title*="-"]').count()) >= 35);
  check("stats: per deck row", await page.getByText("Sky deck").isVisible());
  check("stats: per subject row", await page.getByText("Physics").isVisible());
  check("stats: weakest topic", await page.getByText("Weakest topics").isVisible());
  await context.close();
}

/* 2b. review and exam leave reported questions out ------------------- */
{
  const { context, page } = await makePage(browser);
  await page.addInitScript(() => {
    localStorage.setItem(
      "freezmo.reports.v1",
      JSON.stringify([
        { deckId: "seed_deck", activityId: "a1", type: "multiple_choice", question: "What colour is a clear daytime sky?", at: Date.now() },
      ]),
    );
  });
  await seed(page, { decks: [DECK] });
  await page.goto(`${BASE}/exam`, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  await page.getByRole("button", { name: /Sky deck/ }).click();
  await page.waitForTimeout(200);
  await page.getByRole("button", { name: /Start exam/ }).click();
  await page.waitForTimeout(1000);
  check(
    "exam: reported question is out of the pool",
    (await page.getByText("What colour is a clear daytime sky?").count()) === 0 &&
      (await page.getByText("What makes the sky blue?").count()) >= 1,
  );
  check("exam: AI disclaimer under the card", (await page.getByText("Freezmo uses AI and can make mistakes").count()) >= 1);
  await context.close();
}

/* 3. exam mode: setup, no hints, timer, breakdown -------------------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [DECK] });
  await page.goto(`${BASE}/exam`, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  check("exam: setup headline", await page.getByRole("heading", { name: "Exam mode" }).isVisible());
  await page.getByRole("button", { name: /Sky deck/ }).click();
  await page.waitForTimeout(200);
  await page.getByRole("button", { name: "5 min", exact: true }).click();
  await page.waitForTimeout(200);
  await page.getByRole("button", { name: /Start exam/ }).click();
  await page.waitForTimeout(1000);
  check("exam: timer visible", await page.getByLabel(/left$/).isVisible());
  /* The exam runs inside the app layout: the fixed feed (z-[45]) must sit
     over the sticky app header (z-40), not under it. */
  const examChrome = await page.evaluate(() => {
    const pill = document.querySelector("header .study-chrome");
    if (!pill) return null;
    const box = pill.getBoundingClientRect();
    const atTop = document.elementFromPoint(Math.round(box.left + box.width / 2), Math.round(box.top + 4));
    return Boolean(atTop?.closest(".study-chrome"));
  });
  check("exam: chrome paints over the app header", examChrome === true, String(examChrome));
  check("exam: no hint rows", (await page.getByText(/Tap a term|Shortcuts|Use the arrow/i).count()) === 0);
  // Order-independent: MC listens for number keys on the active slide and the
  // feed for ArrowDown. Both activities have the correct answer at index 0.
  await page.keyboard.press("1");
  await page.waitForTimeout(500);
  /* The feedback shows, but the "from your notes" passage stays hidden in
     exam mode with the other reveals (the deck's notes do contain the answer). */
  check(
    "exam: no notes source under the feedback",
    (await page.getByText("Why:").count()) >= 1 && (await page.locator("[data-notes-source]").count()) === 0,
  );
  await page.keyboard.press("ArrowDown");
  await page.waitForTimeout(800);
  await page.keyboard.press("2");
  await page.waitForTimeout(500);
  await page.keyboard.press("ArrowDown");
  await page.waitForTimeout(900);
  check("exam: results show the score", await page.getByText("Exam results").isVisible());
  check("exam: per-deck breakdown", await page.getByText("By deck").isVisible() && (await page.getByText("Sky deck").count()) >= 1);
  check("exam: per-topic breakdown", await page.getByText("By topic").isVisible());
  check("exam: wrong answers offered as mistakes", await page.getByRole("button", { name: "Study mistakes" }).isVisible());
  const stored = await readLearning(page);
  check("exam: answers recorded", stored.answers.length >= 2, String(stored.answers.length));
  await context.close();
}

/* 4. reminders hidden without VAPID ---------------------------------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [] });
  await page.goto(BASE, { waitUntil: "networkidle" });
  await page.waitForTimeout(800);
  const status = await page.evaluate(async () => {
    const response = await fetch("/api/push", { headers: { "x-ollama-key": "test-key" } });
    return { status: response.status, body: await response.json() };
  });
  check("push: status says disabled without VAPID", status.body.enabled === false, JSON.stringify(status.body));
  await page.locator("nav.tabbar").getByRole("button").click();
  await page.getByRole("dialog").waitFor({ timeout: 6000 });
  await page.waitForTimeout(500);
  check("push: daily goal setting visible", await page.getByText("Daily goal").isVisible());
  check("push: reminder UI hidden", (await page.getByText("Daily reminder").count()) === 0);
  await context.close();
}

/* 5. reminders with VAPID: status, subscribe, delete, test send ------ */
{
  check("push: port 3209 is free", !(await portInUse(3209)));

  mkdirSync(path.join(REPO, "tests", ".tmp", "push"), { recursive: true });
  const vapid = await new Promise((resolve) => {
    const child = spawn("npx", ["web-push", "generate-vapid-keys", "--json"], { cwd: REPO });
    let output = "";
    child.stdout.on("data", (chunk) => (output += chunk));
    child.on("close", () => {
      try {
        resolve(JSON.parse(output.match(/\{[\s\S]*\}/)[0]));
      } catch {
        resolve(null);
      }
    });
  });
  check("push: vapid keys generated", Boolean(vapid?.publicKey && vapid?.privateKey));

  const app = spawnApp(3209, {
    VAPID_PUBLIC_KEY: vapid.publicKey,
    VAPID_PRIVATE_KEY: vapid.privateKey,
    VAPID_SUBJECT: "mailto:test@example.com",
    PUSH_DIR: path.join(REPO, "tests", ".tmp", "push"),
    // The endpoint in this suite is a fake host: allow it explicitly.
    PUSH_ALLOWED_HOSTS: "push.example.invalid",
  });

  try {
    const up = await waitForHttp("http://127.0.0.1:3209/api/health");
    check("push: second instance is up", up);
    if (up) {
      const context = await browser.newContext({ viewport: { width: 390, height: 844 } });
      const page = await context.newPage();
      await seed(page, { decks: [] });
      await page.goto("http://127.0.0.1:3209/", { waitUntil: "networkidle" });
      await page.waitForTimeout(900);
      await page.locator("nav.tabbar").getByRole("button").click();
      await page.getByRole("dialog").waitFor({ timeout: 6000 });
      await page.waitForTimeout(700);
      check("push: reminder UI appears with VAPID", await page.getByText("Daily reminder").isVisible());

      const subscription = {
        endpoint: "https://push.example.invalid/abc",
        keys: { p256dh: "B".repeat(64), auth: "A".repeat(22) },
      };
      const subscribe = await page.evaluate(async (sub) => {
        const response = await fetch("/api/push", {
          method: "POST",
          headers: { "Content-Type": "application/json", "x-ollama-key": "test-key" },
          body: JSON.stringify(sub),
        });
        return { status: response.status, body: await response.json() };
      }, subscription);
      check("push: subscription stored", subscribe.status === 200 && subscribe.body.count === 1, JSON.stringify(subscribe.body));
      const file = JSON.parse(readFileSync(path.join(REPO, "tests", ".tmp", "push", "subscriptions.json"), "utf8"));
      check("push: stored without personal data", file.length === 1 && Object.keys(file[0]).sort().join(",") === "createdAt,endpoint,keys");

      const badBody = await page.evaluate(async () => {
        const response = await fetch("/api/push", {
          method: "POST",
          headers: { "Content-Type": "application/json", "x-ollama-key": "test-key" },
          body: JSON.stringify({ endpoint: "not-a-url" }),
        });
        return response.status;
      });
      check("push: invalid subscription → 400", badBody === 400, String(badBody));

      const testSend = await page.evaluate(async (sub) => {
        const response = await fetch("/api/push/test", {
          method: "POST",
          headers: { "Content-Type": "application/json", "x-ollama-key": "test-key" },
          body: JSON.stringify({ endpoint: sub.endpoint }),
        });
        return { status: response.status, body: await response.json() };
      }, subscription);
      check("push: test send reaches the sender (unreachable endpoint → 502)", testSend.status === 502, JSON.stringify(testSend.body));

      const removed = await page.evaluate(async (sub) => {
        const response = await fetch(`/api/push?endpoint=${encodeURIComponent(sub.endpoint)}`, {
          method: "DELETE",
          headers: { "x-ollama-key": "test-key" },
        });
        return { status: response.status, body: await response.json() };
      }, subscription);
      check("push: delete path works", removed.status === 200 && removed.body.count === 0, JSON.stringify(removed.body));
      await context.close();
    }
  } finally {
    app.stop();
  }
}

await browser.close();
finish();