freezmo/tests/phase1-test.mjs

/**
 * Phase 1 verification: backup export/import, storage persistence row, backup
 * nudge, opt-in encrypted sync (enable/join/push/delete + server validation),
 * and usage-limit handling (client wiring + real server mapping against a stub
 * Ollama).
 */
import { mkdirSync, readFileSync, rmSync } from "node:fs";
import http from "node:http";

import {
  BASE,
  DECK,
  launchBrowser,
  makeChecker,
  makePage,
  portInUse,
  seed,
  spawnApp,
  waitForHttp,
} from "./lib.mjs";

const { check, finish } = makeChecker();

const DECK_A = { ...DECK, id: "deck_a", title: "Photosynthesis basics", folderId: "fold_bio", updatedAt: 1 };
const DECK_B = { ...DECK, id: "deck_b", title: "Trigonometry", folderId: undefined, updatedAt: 1 };
const FOLDER = { id: "fold_bio", name: "Biology", createdAt: 1 };
const HOMEWORK = {
  id: "hw_1",
  mode: "make",
  title: "Quadratic equation",
  language: "English",
  createdAt: 1,
  updatedAt: 1,
  sourceText: "Solve x^2 - 4 = 0",
  answer: "x = 2 or x = -2",
};

const browser = await launchBrowser();

async function openSettings(page) {
  await page.goto(BASE, { waitUntil: "networkidle" });
  await page.waitForTimeout(800);
  await page.locator("nav.tabbar").getByRole("button").click();
  await page.getByRole("dialog").waitFor({ timeout: 6000 });
  await page.waitForTimeout(500);
}

/* 1. export contains everything except the key --------------------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [DECK_A, DECK_B], folders: [FOLDER], homework: [HOMEWORK] });
  await openSettings(page);
  const downloadPromise = page.waitForEvent("download");
  await page.getByRole("button", { name: "Export all data" }).click();
  const download = await downloadPromise;
  const raw = readFileSync(await download.path(), "utf8");
  const backup = JSON.parse(raw);
  check("backup: kind and version", backup.app === "freezmo" && backup.kind === "backup" && backup.version === 1);
  check(
    "backup: decks, folders and homework included",
    backup.data.decks.length === 2 && backup.data.folders.length === 1 && backup.data.homework.length === 1,
  );
  check("backup: API key never in the file", !raw.includes("test-key") && !raw.includes("apiKey") && !raw.includes("x-ollama-key"));
  check("backup: safe settings included", typeof backup.data.settings?.haptics === "boolean");
  await context.close();
}

/* 2. import preview + merge + replace ------------------------------ */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [DECK_A, DECK_B], folders: [FOLDER], homework: [HOMEWORK] });
  await openSettings(page);
  const incoming = {
    app: "freezmo",
    kind: "backup",
    version: 1,
    exportedAt: new Date().toISOString(),
    data: { decks: [{ ...DECK_A, id: "deck_new", title: "Imported deck", folderId: undefined }], folders: [], homework: [], settings: { haptics: true } },
  };
  await page.getByRole("dialog").locator('input[type="file"][accept*="json"]').setInputFiles({
    name: "backup.json",
    mimeType: "application/json",
    buffer: Buffer.from(JSON.stringify(incoming)),
  });
  await page.waitForTimeout(400);
  check("import: preview shown", await page.getByText(/This backup has 1 decks/).isVisible());
  await page.getByRole("button", { name: "Merge" }).click();
  await page.waitForTimeout(400);
  check("import: merge adds the deck", await page.getByText(/Imported 3 decks/).isVisible());
  await page.keyboard.press("Escape");
  await page.waitForTimeout(500);
  check("import: deck visible in the library", await page.getByText("Imported deck").first().isVisible());

  await openSettings(page);
  const replace = {
    app: "freezmo",
    kind: "backup",
    version: 1,
    exportedAt: new Date().toISOString(),
    data: { decks: [{ ...DECK_A, id: "only_deck", title: "Only deck", folderId: undefined }], folders: [], homework: [] },
  };
  await page.getByRole("dialog").locator('input[type="file"][accept*="json"]').setInputFiles({
    name: "backup.json",
    mimeType: "application/json",
    buffer: Buffer.from(JSON.stringify(replace)),
  });
  await page.waitForTimeout(400);
  await page.getByRole("button", { name: "Replace" }).click();
  await page.waitForTimeout(400);
  const stored = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.decks.v1")).map((deck) => deck.title));
  check("import: replace wipes and uses the backup", stored.length === 1 && stored[0] === "Only deck", JSON.stringify(stored));
  await context.close();
}

/* 3. invalid backup + persistence row ------------------------------ */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [DECK_A, DECK_B] });
  await openSettings(page);
  await page.getByRole("dialog").locator('input[type="file"][accept*="json"]').setInputFiles({
    name: "junk.json",
    mimeType: "application/json",
    buffer: Buffer.from("{not json"),
  });
  await page.waitForTimeout(300);
  check("import: junk file gets a friendly error", await page.getByText("That file is not valid JSON.").isVisible());
  check("storage: persistence row present", await page.getByText("Storage", { exact: true }).isVisible());
  const storageText = await page.getByText(/Persistent —|Best-effort —/).first().innerText();
  check("storage: state explained", /Persistent|Best-effort/.test(storageText), storageText.slice(0, 40));
  await context.close();
}

/* 4. backup nudge --------------------------------------------------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, {
    decks: [DECK_A, DECK_B],
    onlyIfMissing: true,
    extra: { "freezmo.backup.v1": JSON.stringify({ lastBackupAt: Date.now() - 15 * 86_400_000, decksAtLastBackup: 0, snoozedUntil: null }) },
  });
  await page.goto(BASE, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  check("nudge: appears after 14 days", await page.getByText("Back up your decks?").isVisible());
  await page.getByRole("button", { name: "Not now" }).click();
  await page.waitForTimeout(300);
  check("nudge: dismissed", (await page.getByText("Back up your decks?").count()) === 0);
  await page.reload({ waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  check("nudge: stays dismissed after reload", (await page.getByText("Back up your decks?").count()) === 0);
  await context.close();
}

/* 5. encrypted sync: enable, upload, push, join, delete ------------- */
{
  const { context, page } = await makePage(browser);
  // No folder references here: a deck pointing at a folder that is not in
  // this browser's library is not rendered, which would hide the deck the
  // suite then expects to find again on the joined device.
  const syncA = { ...DECK, id: "deck_a", title: "Photosynthesis basics", updatedAt: 1 };
  const syncB = { ...DECK, id: "deck_b", title: "Trigonometry", updatedAt: 1 };
  await seed(page, { decks: [syncA, syncB] });
  await openSettings(page);
  check("sync: off by default", (await page.getByRole("button", { name: "Turn on sync" }).count()) === 1);
  await page.getByRole("button", { name: "Turn on sync" }).click();
  await page.waitForTimeout(200);
  await page.getByPlaceholder("Sync passphrase").fill("correct horse battery");
  await page.getByPlaceholder("Repeat the passphrase").fill("correct horse battery");
  await page.getByRole("button", { name: "Start syncing" }).click();
  await page.waitForTimeout(2500);
  check("sync: enabled and uploaded", await page.getByText("Uploaded from this device.").isVisible());
  const syncRecord = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.sync.v1")));
  check("sync: random id (>=32 hex)", /^[a-f0-9]{32}$/.test(syncRecord.id), syncRecord.id);

  const envelope = await page.evaluate(async (id) => {
    const response = await fetch(`/api/sync?id=${id}`, { headers: { "x-ollama-key": "test-key" } });
    return { status: response.status, body: await response.text() };
  }, syncRecord.id);
  check("sync: server has the blob", envelope.status === 200);
  check(
    "sync: blob is opaque",
    !envelope.body.includes("Photosynthesis") && !envelope.body.includes("test-key") && !envelope.body.includes("apiKey"),
  );

  // a real UI write marks dirty, then push
  await page.keyboard.press("Escape");
  await page.waitForTimeout(500);
  await page.getByRole("button", { name: "Deck actions" }).first().click();
  await page.waitForTimeout(300);
  await page.getByRole("button", { name: "Delete deck" }).click();
  await page.waitForTimeout(400);
  const dirty = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.sync.v1")).dirtyAt);
  check("sync: local change marks dirty", typeof dirty === "number");
  const remainingTitle = await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.decks.v1"))[0].title);
  await openSettings(page);
  await page.getByRole("button", { name: "Sync now" }).click();
  await page.waitForTimeout(2500);
  check("sync: push reports uploaded", await page.getByText("Uploaded from this device.").isVisible());
  const id = syncRecord.id;
  await context.close();

  // second device joins
  const second = await browser.newContext({ viewport: { width: 390, height: 844 } });
  const page2 = await second.newPage();
  await seed(page2, { decks: [], key: "other-key" });
  await page2.goto(BASE, { waitUntil: "networkidle" });
  await page2.waitForTimeout(800);
  await page2.locator("nav.tabbar").getByRole("button").click();
  await page2.getByRole("dialog").waitFor({ timeout: 6000 });
  await page2.waitForTimeout(400);
  await page2.getByRole("button", { name: "Join another sync" }).click();
  await page2.waitForTimeout(200);
  await page2.getByPlaceholder("Sync ID").fill(id);
  await page2.getByPlaceholder("Sync passphrase").fill("wrong passphrase");
  await page2.getByRole("button", { name: "Join" }).click();
  await page2.waitForTimeout(2000);
  check("sync: wrong passphrase rejected", await page2.getByText(/can't unlock the synced copy/).isVisible());
  await page2.getByPlaceholder("Sync passphrase").fill("correct horse battery");
  await page2.getByRole("button", { name: "Join" }).click();
  await page2.waitForTimeout(2500);
  await page2.keyboard.press("Escape");
  await page2.waitForTimeout(600);
  check("sync: joined device pulled the decks", await page2.getByText(remainingTitle).first().isVisible(), remainingTitle);
  await second.close();

  // delete synced data (a device without the stored key can still revoke)
  const third = await makePage(browser);
  await seed(third.page, {
    decks: [],
    extra: { "freezmo.sync.v1": JSON.stringify({ v: 1, enabled: true, id, salt: "", lastSyncedAt: null, dirtyAt: null, conflict: false }) },
  });
  await openSettings(third.page);
  await third.page.getByRole("button", { name: "Delete synced data" }).click();
  await third.page.waitForTimeout(200);
  await third.page.getByRole("button", { name: "Delete", exact: true }).click();
  await third.page.waitForTimeout(1500);
  const after = await third.page.evaluate(async (syncId) => {
    const response = await fetch(`/api/sync?id=${syncId}`, { headers: { "x-ollama-key": "test-key" } });
    return response.status;
  }, id);
  check("sync: delete removes the server copy", after === 404, String(after));
  await third.context.close();
}

/* 6. sync server validation ----------------------------------------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [] });
  await page.goto(BASE, { waitUntil: "networkidle" });
  const badId = await page.evaluate(async () => {
    const response = await fetch("/api/sync?id=short", { headers: { "x-ollama-key": "test-key" } });
    return response.status;
  });
  check("sync api: bad id → 400", badId === 400, String(badId));
  const badBody = await page.evaluate(async () => {
    const response = await fetch("/api/sync", {
      method: "POST",
      headers: { "Content-Type": "application/json", "x-ollama-key": "test-key" },
      body: JSON.stringify({ id: "a".repeat(32), envelope: { v: 2 } }),
    });
    return response.status;
  });
  check("sync api: invalid envelope → 400", badBody === 400, String(badBody));
  const noAuth = await page.evaluate(async () => {
    const response = await fetch(`/api/sync?id=${"a".repeat(32)}`);
    return response.status;
  });
  check("sync api: no key → 401", noAuth === 401, String(noAuth));
  await context.close();
}

/* 7. usage limit: client wiring (deck flow + settings banner) ------- */
{
  const { context, page } = await makePage(browser);
  await seed(page, { decks: [] });
  await page.route("**/api/generate", (route) =>
    route.fulfill({
      status: 200,
      contentType: "application/x-ndjson",
      body: JSON.stringify({ type: "error", error: "monthly usage", kind: "usage_limit" }) + "\n",
    }),
  );
  await page.route("**/api/health", (route) =>
    route.fulfill({
      json: {
        ok: false,
        reachable: true,
        mode: "direct-cloud",
        authMode: "user-key",
        needsKey: false,
        keyAccepted: true,
        baseUrl: "https://ollama.com",
        hasApiKey: true,
        vision: { configured: "v", resolved: "v", available: true },
        text: { configured: "t", resolved: "t", available: true },
        latencyMs: 10,
        modelCount: 5,
        error: "monthly usage",
        kind: "usage_limit",
        checkedAt: new Date().toISOString(),
      },
    }),
  );
  await page.goto(`${BASE}/create`, { waitUntil: "networkidle" });
  await page.waitForTimeout(900);
  await page.getByRole("button", { name: "Paste text instead" }).click();
  await page.locator("textarea").first().fill("Notes for the usage limit test.");
  await page.getByRole("button", { name: "Next" }).click();
  await page.waitForTimeout(300);
  await page.getByRole("button", { name: "Next" }).click();
  await page.waitForTimeout(300);
  await page.getByRole("button", { name: "Freeze my deck" }).click();
  await page.waitForTimeout(1200);
  check("usage: clear title in the deck flow", await page.getByText("Free monthly usage used up").isVisible());
  check("usage: upgrade/new-key message shown", await page.getByText(/Upgrade your Ollama account, or create a new one/).isVisible());
  check("usage: Open Ollama button", await page.getByRole("button", { name: "Open Ollama" }).isVisible());
  await page.getByRole("button", { name: "Open Settings" }).click();
  await page.getByRole("dialog").waitFor({ timeout: 6000 });
  await page.waitForTimeout(600);
  check("usage: settings banner shows the limit", await page.getByRole("dialog").getByText("Free monthly usage used up").isVisible());
  const focused = await page.evaluate(() => document.activeElement?.id ?? "");
  check("usage: key field focused", focused === "ollama-api-key-phone" || focused === "ollama-api-key", focused);
  await context.close();
}

/* 8. usage limit: real server mapping against a stub Ollama --------- */
{
  check("stub port is free", !(await portInUse(3208)));
  let mode = "monthly";
  const stub = http.createServer((request, response) => {
    if (request.url?.startsWith("/api/tags")) {
      response.writeHead(200, { "Content-Type": "application/json" });
      response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
      return;
    }
    response.writeHead(429, { "Content-Type": "application/json" });
    response.end(
      JSON.stringify({
        error: mode === "monthly" ? "You have reached your monthly usage limit. Upgrade your plan." : "Too many requests, slow down.",
      }),
    );
  });
  await new Promise((resolve) => stub.listen(11999, "127.0.0.1", resolve));

  const app = spawnApp(3208, {
    OLLAMA_BASE_URL: "http://127.0.0.1:11999",
    OLLAMA_AUTH: "local",
    OLLAMA_VERIFY_KEY: "false",
  });

  try {
    const up = await waitForHttp("http://127.0.0.1:3208/api/health");
    check("stub app instance is up", up);
    if (up) {
      const monthly = await fetch("http://127.0.0.1:3208/api/grade", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ question: "q", referenceAnswer: "a", keyPoints: ["k"], userAnswer: "x" }),
      });
      const monthlyBody = await monthly.json();
      check("stub server: monthly 429 → usage_limit", monthlyBody.kind === "usage_limit", JSON.stringify(monthlyBody).slice(0, 80));

      mode = "burst";
      const burst = await fetch("http://127.0.0.1:3208/api/grade", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ question: "q", referenceAnswer: "a", keyPoints: ["k"], userAnswer: "x" }),
      });
      const burstBody = await burst.json();
      check("stub server: burst 429 → throttled", burstBody.kind === "throttled", JSON.stringify(burstBody).slice(0, 80));
    }
  } finally {
    app.stop();
    stub.close();
  }
}

/* 9. generation guards: cards with repeated wording never ship ------- */
{
  check("guard: stub port free", !(await portInUse(11998)));
  // Mirrors the deck that reached a phone: three terms, one shared definition.
  const repeated = "Jedna z hlavních postav v knížkách lidového čtení";
  const deckReply = {
    title: "Lidové čtení",
    subject: "Literatura",
    language: "Czech",
    summary: "Deck for the repeated-wording guards.",
    activities: [
      { type: "flashcard", topic: "Postavy", front: "Kdo je Bruncvík?", back: "Rytíř se lvem ve znaku.", explanation: "x" },
      { type: "flashcard", topic: "Postavy", front: "Kdo je Štilfrid?", back: "Bruncvíkův otec.", explanation: "x" },
      { type: "multiple_choice", topic: "Postavy", question: "Kdo je hlavní postava?", options: ["Bruncvík", "Paleček", "Štilfrid", "Karel"], correctIndex: 0, explanation: "x" },
      { type: "multiple_choice", topic: "Postavy", question: "Kdo patří k lidovému čtení?", options: ["Paleček", "Paleček", "Bruncvík", "Štilfrid"], correctIndex: 2, explanation: "x" },
      {
        type: "matching",
        topic: "Postavy",
        instruction: "Přiřaď postavy k jejich popisu.",
        pairs: [
          { term: "Bruncvík", definition: "Rytíř se lvem ve znaku" },
          { term: "Štilfrid", definition: "Bruncvíkův otec" },
          { term: "Paleček", definition: "Chytrý sluha" },
        ],
        explanation: "x",
      },
      {
        type: "matching",
        topic: "Postavy",
        instruction: "Přiřaď postavy k jejich zařazení.",
        pairs: [
          { term: "Bruncvík", definition: repeated },
          { term: "Štilfrid", definition: repeated },
          { term: "Paleček", definition: repeated },
        ],
        explanation: "x",
      },
      {
        type: "matching",
        topic: "Postavy",
        instruction: "Přiřaď postavy k vlastnostem.",
        pairs: [
          { term: "A", definition: "statečný" },
          { term: "B", definition: "statečný" },
          { term: "C", definition: "moudrý" },
          { term: "D", definition: "věrný" },
        ],
        explanation: "x",
      },
    ],
  };
  const stub = http.createServer((request, response) => {
    if (request.url?.startsWith("/api/tags")) {
      response.writeHead(200, { "Content-Type": "application/json" });
      response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] }));
      return;
    }
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ message: { content: JSON.stringify(deckReply) } }));
  });
  await new Promise((resolve) => stub.listen(11998, "127.0.0.1", resolve));

  const app = spawnApp(3210, {
    OLLAMA_BASE_URL: "http://127.0.0.1:11998",
    OLLAMA_AUTH: "local",
    OLLAMA_VERIFY_KEY: "false",
  });

  try {
    const up = await waitForHttp("http://127.0.0.1:3210/api/health");
    check("guard: stub app is up", up);
    if (up) {
      const response = await fetch("http://127.0.0.1:3210/api/generate", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ text: "Lidové čtení: Bruncvík, Štilfrid a Paleček.", difficulty: "easy", count: 6 }),
      });
      const events = (await response.text())
        .trim()
        .split("\n")
        .filter(Boolean)
        .map((line) => JSON.parse(line));
      const deck = events.find((event) => event.type === "deck")?.deck;
      const activities = deck?.activities ?? [];
      const matchings = activities.filter((activity) => activity.type === "matching");
      const choices = activities.filter((activity) => activity.type === "multiple_choice");
      check("guard: the good cards still ship", activities.length === 5, `got ${activities.length}: ${activities.map((a) => a.type).join(",")}`);
      check("guard: repeated-option card dropped", choices.length === 1 && new Set(choices[0].options).size === 4);
      check("guard: repeated-definition card dropped", matchings.length === 2, `matchings: ${matchings.length}`);
      check(
        "guard: every matching card keeps its definitions distinct",
        matchings.every((activity) => new Set(activity.pairs.map((pair) => pair.definition)).size === activity.pairs.length),
      );
      const salvaged = matchings.find((activity) => activity.pairs.some((pair) => pair.term === "A"));
      check("guard: a salvageable card is trimmed, not dropped", salvaged?.pairs.length === 3, `pairs: ${salvaged?.pairs?.length}`);
    }
  } finally {
    app.stop();
    stub.close();
  }
}

/* 10. reports: appended as JSON lines, write-only over HTTP --------- */
{
  check("reports: app port free", !(await portInUse(3211)));
  const reportsDir = new URL("./.tmp/reports-api", import.meta.url).pathname;
  rmSync(reportsDir, { recursive: true, force: true });
  mkdirSync(reportsDir, { recursive: true });
  const app = spawnApp(3211, {
    OLLAMA_BASE_URL: "http://127.0.0.1:11999",
    OLLAMA_AUTH: "local",
    OLLAMA_VERIFY_KEY: "false",
    REPORTS_DIR: reportsDir,
  });

  try {
    const up = await waitForHttp("http://127.0.0.1:3211/api/health");
    check("reports: stub app is up", up);
    if (up) {
      const report = {
        deckId: "seed_deck",
        deckTitle: "Lidové čtení",
        activityId: "a1",
        type: "matching",
        question: "Přiřaď postavy k jejich zařazení.",
        at: Date.now(),
      };
      const accepted = await fetch("http://127.0.0.1:3211/api/reports", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify(report),
      });
      check("reports: accepted", accepted.status === 200 && (await accepted.json()).ok === true, String(accepted.status));

      const lines = readFileSync(`${reportsDir}/reports.jsonl`, "utf8").trim().split("\n");
      const stored = JSON.parse(lines[lines.length - 1]);
      check(
        "reports: one JSON line with a server timestamp",
        stored.activityId === "a1" && stored.question === report.question && typeof stored.receivedAt === "string",
      );

      const invalid = await fetch("http://127.0.0.1:3211/api/reports", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ deckId: "x" }),
      });
      check("reports: invalid body rejected", invalid.status === 400, String(invalid.status));

      const readBack = await fetch("http://127.0.0.1:3211/api/reports");
      check("reports: write-only over HTTP (no GET)", readBack.status === 405, String(readBack.status));
    }
  } finally {
    app.stop();
  }
}

await browser.close();
finish();