freezmo/tests/teach-test.mjs

/**
 * Teach it back: the student prompt on the chat route, the session flow (start
 * → explain → curious questions → recap), the recap's two follow-ups (put the
 * missed cards on the review schedule, turn the gaps into new flashcards) and
 * the session history.
 *
 * Runs against a stub Ollama, so no real model is needed: the stub counts the
 * turns and answers each one with a question, which lets the test drive the
 * exact conversation it wants to see.
 */
import http from "node:http";
import { mkdirSync } from "node:fs";

import { DECK, launchBrowser, makeChecker, makePage, portInUse, readLearning, seed, spawnApp, waitForHttp } from "./lib.mjs";

const { check, finish } = makeChecker();
const browser = await launchBrowser();

const STUB_PORT = 11999;
const APP_PORT = 3214;
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
const APP = `http://127.0.0.1:${APP_PORT}`;

/** What the recap stub returns for the finished session. */
const SUMMARY_REPLY = {
  well: ["Why the sky is blue", "Molecules scatter short wavelengths"],
  gaps: [
    { what: "Which colours scatter the most, and why violet is not the sky's colour", card: 1 },
    { what: "Something only the explanation covered", card: null },
  ],
  next: ["Re-explain the scattering with one concrete wavelength", "Add a card about the missing colours"],
};

/** The gap-card stub: four cards, one of them not a flashcard (mapped when save). */
const GAP_REPLY = {
  title: "Gap cards",
  subject: "Physics",
  language: "English",
  summary: "Cards for the missed ideas.",
  activities: [
    { type: "flashcard", topic: "Scattering", front: "GAP-FRONT-1 which colours scatter most?", back: "GAP-BACK-1 the short ones, like blue", explanation: "Rayleigh scattering is stronger for short wavelengths." },
    { type: "flashcard", topic: "Scattering", front: "GAP-FRONT-2 why does the sky look blue?", back: "GAP-BACK-2 scattered blue reaches your eyes", explanation: "Short wavelengths scatter far more." },
    { type: "flashcard", topic: "Scattering", front: "GAP-FRONT-3 what scatters the light?", back: "GAP-BACK-3 air molecules", explanation: "The molecules, not dust, do most of it." },
    { type: "multiple_choice", topic: "Scattering", question: "GAP-FRONT-4 what happens to long wavelengths?", options: ["GAP-BACK-4 they pass straight through", "they vanish", "they turn blue", "they stop"], correctIndex: 0, explanation: "Long wavelengths scatter much less." },
  ],
};

/* ------------------------------------------------------------------ */
/* Stub Ollama                                                         */
/* ------------------------------------------------------------------ */

let turns = 0;
let lastChat = null;

const stub = http.createServer((request, response) => {
  if (request.url?.startsWith("/api/tags")) {
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ models: [{ name: "stub-text" }, { name: "stub-vision" }] }));
    return;
  }
  let raw = "";
  request.on("data", (chunk) => (raw += chunk));
  request.on("end", () => {
    let body = {};
    try {
      body = JSON.parse(raw || "{}");
    } catch {
      /* ignore */
    }
    const system = String(body.messages?.[0]?.content ?? "");

    if (request.url?.startsWith("/api/chat")) {
      lastChat = body;
      /* Gap cards: the activity designer wants the deck JSON (non-streamed). */
      if (system.includes("activity designer")) {
        response.writeHead(200, { "Content-Type": "application/json" });
        response.end(JSON.stringify({ message: { content: JSON.stringify(GAP_REPLY) } }));
        return;
      }
      /* The recap: JSON, streamed like any other answer. */
      if (system.includes("write the app's recap")) {
        const content = JSON.stringify(SUMMARY_REPLY);
        response.writeHead(200, { "Content-Type": "application/x-ndjson" });
        response.write(JSON.stringify({ message: { content: content.slice(0, 40) }, done: false }) + "\n");
        response.write(JSON.stringify({ message: { content: content.slice(40) }, done: false }) + "\n");
        response.write(JSON.stringify({ message: { content: "" }, done: true }) + "\n");
        response.end();
        return;
      }
      /* Student turns: one short question per answer. */
      turns += 1;
      const content = `STUDENT-Q${turns}: wait, why does that step work?`;
      response.writeHead(200, { "Content-Type": "application/x-ndjson" });
      response.write(JSON.stringify({ message: { content }, done: false }) + "\n");
      response.write(JSON.stringify({ message: { content: "" }, done: true }) + "\n");
      response.end();
      return;
    }
    response.writeHead(404);
    response.end();
  });
});
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));

check("teach app port is free", !(await portInUse(APP_PORT)));
const syncDir = new URL("./.tmp/sync-teach", import.meta.url).pathname;
mkdirSync(syncDir, { recursive: true });
const app = spawnApp(APP_PORT, {
  OLLAMA_BASE_URL: STUB_BASE,
  OLLAMA_AUTH: "local",
  OLLAMA_VERIFY_KEY: "false",
  CHAT_MODELS: "stub-text,stub-vision",
  CHAT_MODEL_DEFAULT: "stub-text",
  SYNC_DIR: syncDir,
});

async function chatPost(payload) {
  const response = await fetch(`${APP}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(payload),
  });
  const contentType = response.headers.get("content-type") ?? "";
  if (contentType.includes("ndjson")) {
    const events = (await response.text())
      .trim()
      .split("\n")
      .filter(Boolean)
      .map((line) => JSON.parse(line));
    return { status: response.status, events, body: null };
  }
  return { status: response.status, events: null, body: await response.json().catch(() => null) };
}

const text = (events) => (events ?? []).filter((event) => event.type === "token").map((event) => event.text).join("");

try {
  const up = await waitForHttp(`${APP}/api/health`);
  check("stub app instance is up", up);

  if (up) {
    /* 1. API: one student turn -------------------------------------- */
    const turn = await chatPost({
      messages: [{ role: "user", content: "The sky is blue because molecules scatter short wavelengths." }],
      context: [{ deckId: "seed_deck", deckTitle: "Sky deck", section: "Notes", text: "RAYLEIGH-SCATTERING-NOTES." }],
      mode: "teach",
      teach: {
        topic: "Why the sky is blue",
        difficulty: "skeptical",
        exchange: 0,
        limit: 10,
        cards: ["[1] What colour is a clear daytime sky? → Blue"],
      },
    });
    const turnSystem = String(lastChat?.messages?.[0]?.content ?? "");
    check("api: a teach turn streams the student's question", turn.status === 200 && text(turn.events).includes("STUDENT-Q"), JSON.stringify(turn.body));
    check(
      "api: the student prompt carries the role, the cards and the tone",
      turnSystem.includes("You are the STUDENT") &&
        turnSystem.includes("[1] What colour is a clear daytime sky?") &&
        turnSystem.includes("skeptical student"),
    );
    check("api: the student prompt carries the notes context", turnSystem.includes("RAYLEIGH-SCATTERING-NOTES"));
    check("api: the student prompt forbids grading", turnSystem.includes("NEVER judge the explanation"));

    /* 2. API: the recap shape --------------------------------------- */
    const recap = await chatPost({
      messages: [
        { role: "user", content: "The sky is blue because molecules scatter short wavelengths." },
        { role: "assistant", content: "Why do the short ones scatter more?" },
        { role: "user", content: "Because the molecules are comparable in size." },
      ],
      mode: "teach_summary",
      teach: { topic: "Why the sky is blue", difficulty: "friendly", exchange: 2, limit: 10, cards: ["[1] What colour is a clear daytime sky? → Blue"] },
    });
    const recapSystem = String(lastChat?.messages?.[0]?.content ?? "");
    check(
      "api: the recap prompt asks for the JSON shape",
      recap.status === 200 && recapSystem.includes('"well"') && recapSystem.includes('"gaps"') && recapSystem.includes('"next"'),
    );
    check("api: the recap streams plain JSON", text(recap.events).startsWith('{"well"'));

    /* 3. API: mode guards ------------------------------------------- */
    const missing = await chatPost({
      messages: [{ role: "user", content: "hi" }],
      mode: "teach",
    });
    check("api: a teach mode without its payload → 400", missing.status === 400, JSON.stringify(missing.body));
    const badMode = await chatPost({
      messages: [{ role: "user", content: "hi" }],
      mode: "roleplay",
    });
    check("api: an unknown mode → 400", badMode.status === 400, JSON.stringify(badMode.body));
    const plain = await chatPost({ messages: [{ role: "user", content: "Why is the sky blue?" }] });
    check(
      "api: plain chat is untouched",
      plain.status === 200 && String(lastChat?.messages?.[0]?.content ?? "").includes("chat tutor"),
    );

    /* 4. UI: entry points -------------------------------------------- */
    const { context, page } = await makePage(browser, { width: 1280, height: 900 });
    await seed(page, { decks: [DECK] });

    await page.goto(`${APP}/`, { waitUntil: "networkidle" });
    const sidebarLink = page.locator('a[href="/teach"]').first();
    check("ui: the sidebar has a Teach it back entry", (await sidebarLink.count()) > 0 && (await sidebarLink.getAttribute("title")) === "Teach it back");

    await page.locator('button[aria-label="Deck actions"]').first().click();
    const deckLink = page.locator('[data-testid="deck-actions"] a[href="/teach?deck=seed_deck"]');
    check("ui: the deck's actions offer Teach it back", (await deckLink.count()) > 0);

    /* 5. UI: the start screen ---------------------------------------- */
    await page.goto(`${APP}/teach?deck=seed_deck`, { waitUntil: "networkidle" });
    await page.waitForTimeout(600);
    const topicInput = page.locator("[data-teach-topic]");
    check("ui: a deck brings its topic along", (await topicInput.inputValue()) === "Sky deck", await topicInput.inputValue());
    check(
      "ui: the mode explains itself and shows its empty state later",
      (await page.getByText("You explain. The AI plays your student.").count()) > 0,
    );
    await page.getByRole("button", { name: "Skeptical student" }).click();
    check(
      "ui: the difficulty is pickable",
      (await page.getByRole("button", { name: "Skeptical student" }).getAttribute("aria-pressed")) === "true",
    );

    /* 6. UI: a session ----------------------------------------------- */
    turns = 0;
    await page.getByTestId("teach-start").click();
    await page.waitForTimeout(400);
    check("ui: starting opens the session with its own empty state", (await page.getByText("You are the teacher now").count()) > 0);

    const composer = page.locator("textarea").last();
    await composer.fill("The sky is blue because molecules scatter short wavelengths.");
    await composer.press("Enter");
    await page.getByText("STUDENT-Q1", { exact: false }).first().waitFor({ timeout: 15_000 });
    check("ui: the student answers with a question", true);
    const promptSystem = String(lastChat?.messages?.[0]?.content ?? "");
    check(
      "ui: the student saw the deck's cards and the session's topic",
      promptSystem.includes("[1] What colour is a clear daytime sky?") && promptSystem.includes("Sky deck"),
    );
    check("ui: the header counts the exchanges", (await page.getByText("1/10").count()) > 0);

    await composer.fill("Rayleigh scattering makes the shorter wavelengths scatter far more.");
    await composer.press("Enter");
    await page.getByText("STUDENT-Q2", { exact: false }).first().waitFor({ timeout: 15_000 });

    /* 7. UI: finish → recap ------------------------------------------ */
    await page.getByRole("button", { name: "Finish" }).click();
    await page.getByTestId("teach-recap").waitFor({ timeout: 15_000 });
    await page.getByText("Explained well").waitFor({ timeout: 15_000 });
    check("ui: the recap lists what was explained well", (await page.getByText("Why the sky is blue").count()) > 0);
    const gapLink = page.locator('[data-testid="teach-recap"] a[href="/deck/seed_deck"]');
    check("ui: a gap links to the card's deck", (await gapLink.count()) > 0);
    check("ui: the recap suggests next steps", (await page.getByText("Re-explain the scattering with one concrete wavelength").count()) > 0);

    /* 8. UI: put the missed cards on the review schedule -------------- */
    await page.getByTestId("teach-review").click();
    await page.waitForTimeout(700);
    const learning = await readLearning(page);
    const scheduled = learning.srs.filter((record) => record.deckId === "seed_deck");
    check(
      "ui: the missed cards are scheduled for review now",
      scheduled.length >= 1 && scheduled.every((record) => record.dueAt <= Date.now()),
      JSON.stringify(scheduled.map((record) => record.key)),
    );

    /* 9. UI: turn the gaps into cards ---------------------------------- */
    await page.getByTestId("teach-gap-cards").click();
    await page.getByTestId("teach-gap-panel").waitFor();
    check(
      "ui: the gap cards default to the session's deck",
      (await page.locator("[data-testid=teach-gap-panel] select").inputValue()) === "seed_deck",
    );
    await page.getByRole("button", { name: "Create the cards" }).click();
    await page.getByText("Review the cards").waitFor({ timeout: 20_000 });
    const frontInputs = page.locator("[data-testid=teach-gap-panel] input:not([type=checkbox])");
    const backs = page.locator("[data-testid=teach-gap-panel] textarea");
    check("ui: every proposed card can be edited before saving", (await frontInputs.count()) === 4 && (await backs.count()) === 4);
    check(
      "ui: a non-flashcard card is offered as a card too",
      (await backs.nth(3).inputValue()).includes("GAP-BACK-4"),
      await backs.nth(3).inputValue(),
    );
    await frontInputs.first().fill("EDITED-GAP-FRONT");
    await page.locator("[data-testid=teach-gap-panel] input[type=checkbox]").last().uncheck();
    await page.getByTestId("teach-save-cards").click();
    await page.waitForTimeout(700);
    const savedDecks = await page.evaluate(() => JSON.parse(window.localStorage.getItem("freezmo.decks.v1") ?? "[]"));
    const savedDeck = savedDecks.find((entry) => entry.id === "seed_deck");
    const added = (savedDeck?.activities ?? []).filter((activity) => activity.type === "flashcard" && String(activity.front).includes("GAP-FRONT"));
    check(
      "ui: saving adds the reviewed cards to the deck",
      added.length === 3 && added.some((activity) => activity.front === "EDITED-GAP-FRONT"),
      JSON.stringify(added.map((activity) => activity.front)),
    );
    check(
      "ui: the unchecked card was left out",
      !(savedDeck?.activities ?? []).some((activity) => String(activity.front).includes("GAP-FRONT-4")),
    );

    /* 10. UI: the session is in the history --------------------------- */
    await page.goto(`${APP}/teach`, { waitUntil: "networkidle" });
    await page.waitForTimeout(600);
    const recent = page.getByText("Sky deck", { exact: true });
    check("ui: the finished session appears in the recent list", (await recent.count()) > 0);
    await recent.first().click();
    await page.getByTestId("teach-recap").waitFor({ timeout: 10_000 });
    check("ui: reopening the session brings the recap back", (await page.getByText("Explained well").count()) > 0);
    await page.getByRole("button", { name: "New session" }).click();
    check("ui: New session returns to the start screen", (await page.getByTestId("teach-start").count()) > 0);

    /* 11. UI: an unfinished session continues after a reload ----------- */
    await page.locator("[data-teach-topic]").fill("Continuing reload");
    await page.getByTestId("teach-start").click();
    const reloadComposer = page.locator("textarea").last();
    await reloadComposer.fill("First part of the explanation.");
    await reloadComposer.press("Enter");
    await page.getByText("STUDENT-Q", { exact: false }).first().waitFor({ timeout: 15_000 });
    const sessionUrl = page.url();
    await page.goto(sessionUrl, { waitUntil: "networkidle" });
    await page.waitForTimeout(700);
    check("ui: an unfinished session reopens where it was", (await page.getByText("First part of the explanation.").count()) > 0);
    const beforeReload = (await page.evaluate(() => document.body.innerText.match(/STUDENT-Q/g)?.length ?? 0));
    const continueComposer = page.locator("textarea").last();
    await continueComposer.fill("Second part after the reload.");
    await continueComposer.press("Enter");
    await page.waitForFunction(
      (expected) => (document.body.innerText.match(/STUDENT-Q/g) ?? []).length > expected,
      beforeReload,
      { timeout: 15_000 },
    );
    check("ui: the conversation carries on after the reload", true);

    /* 11b. UI: sessions can be removed from the list ------------------- */
    await page.goto(`${APP}/teach`, { waitUntil: "networkidle" });
    await page.waitForTimeout(500);
    const doomed = page.locator("li", { hasText: "Continuing reload" }).first();
    await doomed.getByRole("button", { name: "Delete session" }).click();
    await page.waitForTimeout(400);
    check("ui: a session can be removed from the recent list", (await page.getByText("Continuing reload").count()) === 0);

    /* 12. UI: teach sessions stay out of the chat history -------------- */
    await page.goto(`${APP}/chat`, { waitUntil: "networkidle" });
    await page.waitForTimeout(500);
    const chatSidebar = await page.locator("aside").first().innerText();
    check("ui: teach sessions do not leak into the chat list", !chatSidebar.includes("Sky deck"), chatSidebar.slice(0, 120));
    await context.close();

    /* 13. UI: the exchange limit ends the session on its own ----------- */
    const { context: autoContext, page: autoPage } = await makePage(browser, { width: 390, height: 844, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
    await seed(autoPage, { decks: [DECK] });
    await autoPage.goto(`${APP}/teach`, { waitUntil: "networkidle" });
    await autoPage.waitForTimeout(500);
    await autoPage.locator("[data-teach-topic]").fill("Scattering");
    await autoPage.getByTestId("teach-start").click();
    await autoPage.waitForTimeout(300);
    const mobileComposer = autoPage.locator("textarea").last();
    check("mobile: the composer is usable at 390px", await mobileComposer.isVisible());
    const turnsBefore = turns;
    for (let index = 0; index < 10; index += 1) {
      await mobileComposer.fill(`Explanation part ${index + 1}.`);
      /* Phones send with the button: Enter is a newline there. */
      await autoPage.getByRole("button", { name: "Send" }).click();
      /* Either the next question lands, or — on the last exchange — the
         session wraps itself up and the recap replaces the conversation. */
      await autoPage.waitForFunction(
        (expected) =>
          Boolean(document.querySelector("[data-testid=teach-recap]")) ||
          (document.body.innerText.match(/STUDENT-Q/g) ?? []).length >= expected,
        index + 1,
        { timeout: 20_000 },
      );
    }
    await autoPage.getByTestId("teach-recap").waitFor({ timeout: 20_000 });
    check(
      "ui: the student ends the session after the exchange limit",
      (await autoPage.getByText("Explained well").count()) > 0 && turns === turnsBefore + 10,
      `${turnsBefore} -> ${turns}`,
    );
    await autoContext.close();

    /* 14. UI: 360px, the narrow phone --------------------------------- */
    const { context: narrowContext, page: narrowPage } = await makePage(browser, { width: 360, height: 740, deviceScaleFactor: 1, isMobile: true, hasTouch: true });
    await seed(narrowPage, { decks: [DECK] });
    await narrowPage.goto(`${APP}/teach?deck=seed_deck`, { waitUntil: "networkidle" });
    await narrowPage.waitForTimeout(500);
    const narrowOverflow = await narrowPage.evaluate(() => document.documentElement.scrollWidth - window.innerWidth);
    const startButton = narrowPage.getByTestId("teach-start");
    const startBox = await startButton.boundingBox();
    check(
      "mobile: the start screen fits 360px and its action is on screen",
      narrowOverflow <= 1 && Boolean(startBox) && startBox.y + startBox.height <= 740 + 1,
      `overflow ${narrowOverflow}px`,
    );
    await narrowPage.getByTestId("teach-start").click();
    await narrowPage.waitForTimeout(300);
    const narrowComposer = await narrowPage.locator("textarea").last().boundingBox();
    check(
      "mobile: the composer stays on screen at 360px",
      Boolean(narrowComposer) && narrowComposer.y + narrowComposer.height <= 740 + 1,
      JSON.stringify(narrowComposer),
    );
    await narrowContext.close();

    /* 15. UI: the mic appears only where dictation exists -------------- */
    const { context: plainContext, page: plainPage } = await makePage(browser, { width: 390, height: 844, isMobile: true, hasTouch: true });
    await seed(plainPage, { decks: [DECK] });
    /* Chromium ships a real recogniser; a browser without one is the case the
       mic must disappear in, so take both constructors off the window. */
    await plainPage.addInitScript(() => {
      Object.defineProperty(window, "SpeechRecognition", { configurable: true, value: undefined });
      Object.defineProperty(window, "webkitSpeechRecognition", { configurable: true, value: undefined });
    });
    await plainPage.goto(`${APP}/teach?deck=seed_deck`, { waitUntil: "networkidle" });
    await plainPage.getByTestId("teach-start").click();
    await plainPage.waitForTimeout(300);
    check("mic: hidden when the browser cannot dictate", (await plainPage.locator("[data-voice-button]").count()) === 0);
    await plainContext.close();

    const { context: voiceContext, page: voicePage } = await makePage(browser, { width: 390, height: 844, isMobile: true, hasTouch: true });
    await seed(voicePage, { decks: [DECK] });
    await voicePage.addInitScript(() => {
      class FakeRecognition {
        constructor() {
          this.lang = "";
          this.continuous = false;
          this.interimResults = false;
          this.maxAlternatives = 1;
          this.onresult = null;
          this.onerror = null;
          this.onend = null;
          window.__recognition = this;
        }
        start() {}
        stop() {
          if (this.onend) this.onend();
        }
        abort() {}
      }
      /* Chrome ships both names; shadow them so the app uses the stub. */
      Object.defineProperty(window, "SpeechRecognition", { configurable: true, value: FakeRecognition });
      Object.defineProperty(window, "webkitSpeechRecognition", { configurable: true, value: FakeRecognition });
    });
    await voicePage.goto(`${APP}/teach?deck=seed_deck`, { waitUntil: "networkidle" });
    await voicePage.getByTestId("teach-start").click();
    await voicePage.waitForTimeout(300);
    const mic = voicePage.locator("[data-voice-button]");
    check("mic: shown when the browser can dictate", (await mic.count()) === 1);
    await mic.click();
    await voicePage.waitForTimeout(200);
    check("mic: the button reads as listening", (await mic.getAttribute("aria-pressed")) === "true");
    await voicePage.evaluate(() => {
      window.__recognition.onresult({
        resultIndex: 0,
        results: [{ isFinal: true, 0: { transcript: "the sky is blue" } }],
      });
    });
    check(
      "mic: dictated words land in the draft",
      (await voicePage.locator("textarea").last().inputValue()) === "the sky is blue",
      await voicePage.locator("textarea").last().inputValue(),
    );
    await voiceContext.close();
  }
} finally {
  app.stop();
  stub.close();
  await browser.close();
}

finish();