Move the e2e suites into the repo (tests/), out of /tmp
The browser harness lived in /tmp/opencode and an overnight reboot wiped it. It now lives in tests/ and survives reboots, is versioned and runs with one command. - tests/lib.mjs: shared helpers (Chromium auto-detect, seeding, IndexedDB fixtures, stub/extra instance spawn with process-group cleanup, check/finish). - tests/run-all.mjs + npm run test:e2e [filter]: runs the ten suites in order with a summary; suites that need a stub Ollama or VAPID keys start their own next start instance on 3208/3209, verify the port is free and kill the group on exit. Test scratch in tests/.tmp/ (gitignored). - Suites rebuilt and green against the current build: regression 38, mobile 94 (tab matrix, ... sheet, settings sheet, axe), wizard 83, native 31 (PWA/offline/share/back/undo/haptics/draft), i18n 13, freeze-loading 6, review 15, phase1 36 (backup/sync/usage limits), phase2 42 (learning engine/reminders), phase3 21 (homework check). 379 checks, 10/10 suites in one full run. - playwright-core + axe-core added as devDependencies; CHROME env or the newest ~/.cache/ms-playwright chromium is used. - README: new decision 45 documenting the harness; decision 44's verification note updated. The older suites' exact sources were lost with the wipe, so the rebuilt versions reconstruct the same feature areas rather than being byte-identical.
This commit is contained in:
parent
2103f9fc69
commit
dbd5dfbc58
17 changed files with 2280 additions and 8 deletions
197
tests/phase3-test.mjs
Normal file
197
tests/phase3-test.mjs
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
/**
|
||||
* Phase 3 verification: "Check my work" (server mapping + phone and desktop
|
||||
* UI), follow-ups (shorter / more detail / explain a step differently) and
|
||||
* "Turn into a study deck" pre-filling the New deck flow.
|
||||
* Runs against a stub Ollama so no model is needed.
|
||||
*/
|
||||
import http from "node:http";
|
||||
|
||||
import { launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";
|
||||
|
||||
const { check, finish } = makeChecker();
|
||||
|
||||
const STUB_PORT = 11998;
|
||||
const APP_PORT = 3208;
|
||||
const STUB_BASE = `http://127.0.0.1:${STUB_PORT}`;
|
||||
const APP = `http://127.0.0.1:${APP_PORT}`;
|
||||
|
||||
const CHECK_REPLY = {
|
||||
title: "Equation check",
|
||||
language: "English",
|
||||
verdict: "One mistake: the sign in the second line.",
|
||||
correct: ["The setup is right."],
|
||||
wrong: [{ issue: "x = 3", why: "4 - 2 is 2, not 3.", fix: "x = 2" }],
|
||||
corrected: "x + 2 = 4\nx = 4 - 2\nx = 2",
|
||||
};
|
||||
const GUIDE_REPLY = {
|
||||
title: "Equation guide",
|
||||
language: "English",
|
||||
steps: [
|
||||
{
|
||||
title: "Move the number",
|
||||
explanation: "Subtract 2 from both sides so x stands alone.",
|
||||
hints: ["Think about the opposite operation.", "Subtract 2 from both sides.", "x = 4 - 2"],
|
||||
answer: "x = 4 - 2",
|
||||
},
|
||||
{
|
||||
title: "Finish it",
|
||||
explanation: "Now do the subtraction to get the value of x.",
|
||||
hints: ["Just arithmetic now.", "4 minus 2.", "x = 2"],
|
||||
answer: "x = 2",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const stub = http.createServer((request, response) => {
|
||||
if (request.url?.startsWith("/api/tags")) {
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
|
||||
return;
|
||||
}
|
||||
let raw = "";
|
||||
request.on("data", (chunk) => (raw += chunk));
|
||||
request.on("end", () => {
|
||||
let content = "{}";
|
||||
try {
|
||||
const body = JSON.parse(raw);
|
||||
const system = String(body.messages?.[0]?.content ?? "");
|
||||
const user = String(body.messages?.[1]?.content ?? "");
|
||||
if (system.includes("homework checker")) content = JSON.stringify(CHECK_REPLY);
|
||||
else if (system.includes("homework writer")) content = JSON.stringify({ title: "Equation", language: "English", answer: "x = 2" });
|
||||
else if (system.includes("homework coach")) content = JSON.stringify(GUIDE_REPLY);
|
||||
else if (system.includes("homework assistant")) {
|
||||
content = user.includes("SHORTER")
|
||||
? "SHORTER: x = 2"
|
||||
: user.includes("MORE DETAIL")
|
||||
? "MORE DETAIL: x + 2 = 4, so subtract 2: x = 2"
|
||||
: "EXPLAINED DIFFERENTLY: picture a balance scale.";
|
||||
}
|
||||
} catch {
|
||||
/* keep the default */
|
||||
}
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ message: { content } }));
|
||||
});
|
||||
});
|
||||
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));
|
||||
|
||||
check("stub app port is free", !(await portInUse(APP_PORT)));
|
||||
const app = spawnApp(APP_PORT, { OLLAMA_BASE_URL: STUB_BASE, OLLAMA_AUTH: "local", OLLAMA_VERIFY_KEY: "false" });
|
||||
|
||||
const browser = await launchBrowser();
|
||||
|
||||
try {
|
||||
const up = await waitForHttp(`${APP}/api/health`);
|
||||
check("stub app instance is up", up);
|
||||
|
||||
if (up) {
|
||||
/* 1. server: check mode ---------------------------------------- */
|
||||
const apiCheck = await fetch(`${APP}/api/homework`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4", solution: "x = 3" }),
|
||||
});
|
||||
const events = (await apiCheck.text())
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
const homeworkEvent = events.find((event) => event.type === "homework");
|
||||
check("api: check mode returns the feedback payload", homeworkEvent?.homework?.mode === "check");
|
||||
check("api: verdict and corrected version", Boolean(homeworkEvent?.homework?.verdict) && homeworkEvent?.homework?.corrected.includes("x = 2"));
|
||||
check("api: wrong entry has issue/why/fix", homeworkEvent?.homework?.wrong?.[0]?.issue === "x = 3" && Boolean(homeworkEvent?.homework?.wrong?.[0]?.fix));
|
||||
|
||||
const missingSolution = await fetch(`${APP}/api/homework`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ mode: "check", text: "Solve x + 2 = 4" }),
|
||||
});
|
||||
check("api: check without a solution → 400", missingSolution.status === 400, String(missingSolution.status));
|
||||
|
||||
/* 2. server: refine -------------------------------------------- */
|
||||
const refine = await fetch(`${APP}/api/homework/refine`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ instruction: "shorter", text: "x + 2 = 4, therefore x = 2" }),
|
||||
});
|
||||
const refineBody = await refine.json();
|
||||
check("api: refine returns rewritten text", refine.status === 200 && refineBody.text.includes("SHORTER"), JSON.stringify(refineBody).slice(0, 60));
|
||||
const badRefine = await fetch(`${APP}/api/homework/refine`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ instruction: "wrong", text: "x" }),
|
||||
});
|
||||
check("api: refine validates the instruction", badRefine.status === 400, String(badRefine.status));
|
||||
|
||||
/* 3. phone UI: check my work + follow-ups + to-deck -------------- */
|
||||
const phone = await makePage(browser, { width: 390 });
|
||||
await seed(phone.page, { decks: [] });
|
||||
await phone.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
||||
await phone.page.waitForTimeout(900);
|
||||
await phone.page.getByRole("button", { name: "Paste text instead" }).click();
|
||||
await phone.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
||||
await phone.page.getByRole("button", { name: "Next" }).click();
|
||||
await phone.page.waitForTimeout(400);
|
||||
await phone.page.getByRole("button", { name: /Check my work/ }).click();
|
||||
await phone.page.waitForTimeout(400);
|
||||
check("phone: solution field revealed", await phone.page.locator("#homework-solution").isVisible());
|
||||
await phone.page.locator("#homework-solution").fill("x = 3");
|
||||
await phone.page.getByRole("button", { name: "Check my work", exact: true }).click();
|
||||
await phone.page.waitForTimeout(2500);
|
||||
check("phone: verdict shown", await phone.page.getByText("One mistake: the sign in the second line.").isVisible());
|
||||
check("phone: what's right listed", await phone.page.getByText("The setup is right.").isVisible());
|
||||
check("phone: what needs fixing listed", await phone.page.getByText("4 - 2 is 2, not 3.").isVisible());
|
||||
check("phone: corrected version shown", await phone.page.getByText("x = 4 - 2").isVisible());
|
||||
|
||||
await phone.page.getByRole("button", { name: "Shorter" }).click();
|
||||
await phone.page.waitForTimeout(1500);
|
||||
check("phone: shorter follow-up replaces the text", await phone.page.getByText("SHORTER: x = 2").isVisible());
|
||||
check("phone: original can be restored", await phone.page.getByRole("button", { name: "Original" }).isVisible());
|
||||
|
||||
await phone.page.getByRole("button", { name: "Turn into a study deck" }).click();
|
||||
await phone.page.waitForTimeout(1500);
|
||||
check("phone: landed on New deck", phone.page.url().includes("/create"));
|
||||
const seededText = await phone.page.locator("textarea").first().inputValue().catch(() => "");
|
||||
check("phone: deck wizard pre-filled with the content", seededText.includes("x = 4 - 2") || seededText.includes("SHORTER"), seededText.slice(0, 40));
|
||||
await phone.context.close();
|
||||
|
||||
/* 4. desktop UI + guide explain ---------------------------------- */
|
||||
const desktop = await makePage(browser, { width: 1024, height: 900 });
|
||||
await seed(desktop.page, { decks: [] });
|
||||
await desktop.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
||||
await desktop.page.waitForTimeout(900);
|
||||
await desktop.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
||||
await desktop.page.getByRole("button", { name: /Check my work/ }).click();
|
||||
await desktop.page.waitForTimeout(300);
|
||||
check("desktop: solution field appears with the mode", await desktop.page.locator("#homework-solution").isVisible());
|
||||
await desktop.page.locator("#homework-solution").fill("x = 3");
|
||||
await desktop.page.getByRole("button", { name: "Check my work", exact: true }).click();
|
||||
await desktop.page.waitForTimeout(2500);
|
||||
check("desktop: check result renders", await desktop.page.getByText("Corrected version").isVisible());
|
||||
await desktop.context.close();
|
||||
|
||||
const guide = await makePage(browser, { width: 390 });
|
||||
await seed(guide.page, { decks: [] });
|
||||
await guide.page.goto(`${APP}/homework`, { waitUntil: "networkidle" });
|
||||
await guide.page.waitForTimeout(900);
|
||||
await guide.page.getByRole("button", { name: "Paste text instead" }).click();
|
||||
await guide.page.locator("textarea").first().fill("Solve x + 2 = 4");
|
||||
await guide.page.getByRole("button", { name: "Next" }).click();
|
||||
await guide.page.waitForTimeout(400);
|
||||
await guide.page.getByRole("button", { name: /Explain how to make this homework/ }).click();
|
||||
await guide.page.waitForTimeout(2500);
|
||||
const explainButton = guide.page.getByRole("button", { name: /Explain step 1 differently/ });
|
||||
check("guide: explain action present on the step", await explainButton.isVisible().catch(() => false));
|
||||
if (await explainButton.isVisible().catch(() => false)) {
|
||||
await explainButton.click();
|
||||
await guide.page.waitForTimeout(1500);
|
||||
check("guide: explanation replaced", await guide.page.getByText("EXPLAINED DIFFERENTLY: picture a balance scale.").isVisible());
|
||||
}
|
||||
await guide.context.close();
|
||||
}
|
||||
} finally {
|
||||
app.stop();
|
||||
stub.close();
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
finish();
|
||||
Loading…
Add table
Add a link
Reference in a new issue