freezmo/tests/model-compare.mjs

/**
 * Model A/B benchmark (diagnostic, not part of `npm run test:e2e`).
 *
 * Answers "is model X a better fit than model Y for Freezmo's strict-JSON
 * pipeline?" with numbers: it runs the same notes through two throwaway
 * production instances (one per model), times them and reports how often the
 * model needed the JSON repair round.
 *
 *   OLLAMA_API_KEY=... node tests/model-compare.mjs
 *   OLLAMA_API_KEY=... node tests/model-compare.mjs gemma4:31b-cloud gpt-oss:120b-cloud
 *
 * The key travels only to the spawned local instances (server auth mode) and
 * is never printed, stored or committed. Uses your Ollama quota: 2 generate
 * calls + 1 grade call per model.
 */
import { spawnApp, waitForHttp, portInUse } from "./lib.mjs";

const MODELS = process.argv.slice(2);
const [MODEL_A, MODEL_B] = MODELS.length >= 2 ? MODELS : ["gpt-oss:120b-cloud", "gemma4:31b-cloud"];
const API_KEY = process.env.OLLAMA_API_KEY?.trim();
const BASE_URL = process.env.OLLAMA_BASE_URL?.trim() || "https://ollama.com";

if (!API_KEY) {
  console.error("Set OLLAMA_API_KEY (your Ollama key) for this command — it is never printed.");
  process.exit(1);
}

const NOTES = `Rayleigh scattering
Light from the sun hits molecules in the atmosphere. Short wavelengths (blue and violet) scatter much more strongly than long ones (red). This is why a clear daytime sky is blue.
Sunsets
At sunset the light travels through much more atmosphere, so almost all the blue light is scattered away and only the red end of the spectrum reaches our eyes.
Why not violet?
Violet scatters even more than blue, but our eyes are far less sensitive to violet, and some of it is absorbed in the upper atmosphere.
Interesting: the sky on the Moon is black because there is no atmosphere to scatter light.`;

const GRADE_INPUT = {
  question: "Why is a clear daytime sky blue?",
  referenceAnswer: "Short wavelengths scatter more strongly than long ones.",
  keyPoints: ["Rayleigh scattering", "short wavelengths scatter more", "molecules in the atmosphere"],
  userAnswer: "Because blue light scatters more than red in the atmosphere.",
};

async function timed(label, run) {
  const started = Date.now();
  try {
    const value = await run();
    return { label, ms: Date.now() - started, value };
  } catch (error) {
    return { label, ms: Date.now() - started, error: error instanceof Error ? error.message : String(error) };
  }
}

async function generateOnce(port, index) {
  const response = await fetch(`http://127.0.0.1:${port}/api/generate`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ text: NOTES, difficulty: "medium", count: 8 }),
  });
  if (!response.ok) throw new Error(`HTTP ${response.status}: ${(await response.text()).slice(0, 120)}`);
  const events = (await response.text())
    .trim()
    .split("\n")
    .map((line) => JSON.parse(line));
  const deckEvent = events.find((event) => event.type === "deck");
  const errorEvent = events.find((event) => event.type === "error");
  if (errorEvent) throw new Error(errorEvent.error);
  if (!deckEvent) throw new Error("no deck event");
  return {
    run: index,
    activities: deckEvent.deck.activities.length,
    repaired: Boolean(deckEvent.meta?.repaired),
    requested: deckEvent.meta?.requested ?? null,
    types: deckEvent.deck.activities.map((activity) => activity.type),
    sample: deckEvent.deck.activities[0]?.question ?? deckEvent.deck.activities[0]?.front ?? "",
  };
}

async function gradeOnce(port) {
  const response = await fetch(`http://127.0.0.1:${port}/api/grade`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(GRADE_INPUT),
  });
  const body = await response.json().catch(() => null);
  if (!response.ok) throw new Error(body?.error ?? `HTTP ${response.status}`);
  return { correct: body.correct, score: body.score, feedback: String(body.feedback ?? "").slice(0, 70) };
}

async function bench(model, port) {
  if (await portInUse(port)) throw new Error(`port ${port} is busy`);
  const app = spawnApp(port, {
    OLLAMA_AUTH: "server",
    OLLAMA_API_KEY: API_KEY,
    OLLAMA_BASE_URL: BASE_URL,
    OLLAMA_VERIFY_KEY: "false",
    TEXT_MODEL: model,
    VISION_MODEL: model,
  });
  try {
    const up = await waitForHttp(`http://127.0.0.1:${port}/api/health`);
    if (!up) throw new Error("instance did not start");
    const rows = [];
    for (const index of [1, 2]) rows.push(await timed(`generate #${index}`, () => generateOnce(port, index)));
    rows.push(await timed("grade", () => gradeOnce(port)));
    return { model, rows };
  } finally {
    app.stop();
  }
}

console.log(`Comparing ${MODEL_A} vs ${MODEL_B} against ${BASE_URL}\n`);
const results = [];
for (const [model, port] of [
  [MODEL_A, 3210],
  [MODEL_B, 3211],
]) {
  console.log(`--- ${model} ---`);
  const result = await bench(model, port);
  for (const row of result.rows) {
    if (row.error) console.log(`  ${row.label}: ERROR after ${row.ms} ms — ${row.error}`);
    else if (row.label.startsWith("generate")) {
      console.log(
        `  ${row.label}: ${row.ms} ms | ${row.value.activities} activities | repaired: ${row.value.repaired ? "YES" : "no"} | types: ${row.value.types.join(", ")}`,
      );
      console.log(`      first: ${row.value.sample.slice(0, 90)}`);
    } else {
      console.log(`  ${row.label}: ${row.ms} ms | correct: ${row.value.correct} | score: ${row.value.score} | ${row.value.feedback}`);
    }
  }
  results.push(result);
}

console.log("\nSummary");
for (const result of results) {
  const generates = result.rows.filter((row) => row.label.startsWith("generate") && !row.error);
  const repaired = generates.filter((row) => row.value.repaired).length;
  const times = generates.map((row) => row.ms);
  const average = times.length ? Math.round(times.reduce((sum, value) => sum + value, 0) / times.length) : 0;
  console.log(`  ${result.model}: avg generate ${average} ms | repairs ${repaired}/${generates.length} | errors ${result.rows.filter((row) => row.error).length}`);
}
console.log("\nRead it as: fewer repairs + sensible activity types + acceptable latency = the better pipeline fit.");