freezmo/tests/providers-test.mjs

/**
 * Cloud providers: bring-your-own keys (OpenAI-compatible endpoints).
 *
 * One stub server speaks both protocols: the Ollama endpoints the app's shell
 * needs (/api/tags) and the OpenAI-compatible endpoints a custom provider uses
 * (/v1/models, /v1/chat/completions, streaming and not). The app runs with
 * OLLAMA_AUTH=local against the stub, so the server side needs no key while
 * the provider side exercises the browser-held key end to end.
 */
import http from "node:http";

import { DECK, launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";

const { check, finish } = makeChecker();
const browser = await launchBrowser();

const STUB_PORT = 11998;
const APP_PORT = 3213;
const APP = `http://127.0.0.1:${APP_PORT}`;
const PROVIDER_BASE = `http://127.0.0.1:${STUB_PORT}/v1`;
const PROVIDER_KEY = "test-provider-key";

const DECK_REPLY = {
  title: "Provider deck",
  subject: "Physics",
  summary: "Generated on a provider model.",
  language: "English",
  activities: [
    { type: "multiple_choice", topic: "Sky", explanation: "Rayleigh scattering.", question: "Why is the sky blue?", options: ["Scattering", "Paint", "Wind", "Salt"], correctIndex: 0 },
    { type: "true_false", topic: "Sky", explanation: "It scatters.", statement: "Short wavelengths scatter more.", answer: true },
    { type: "flashcard", topic: "Sky", explanation: "Basics.", front: "Rayleigh", back: "Scattering by small molecules" },
    { type: "fill_blank", topic: "Sky", explanation: "The blank is 'blue'.", sentence: "The clear sky looks ____.", answer: "blue" },
    { type: "matching", topic: "Sky", explanation: "Terms and meanings.", instruction: "Match the terms.", pairs: [{ term: "Rayleigh", definition: "Short wavelengths" }, { term: "Molecule", definition: "Small particle" }, { term: "Sky", definition: "Above us" }] },
  ],
};

let lastProviderRequest = null;
const seen = { models: 0, completions: 0 };

const stub = http.createServer((request, response) => {
  const url = request.url ?? "";

  // ---- Ollama side (the app's shell needs a healthy daemon) ----
  if (url.startsWith("/api/tags")) {
    response.writeHead(200, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ models: [{ name: "stub-text" }, { name: "stub-vision" }] }));
    return;
  }
  if (url.startsWith("/api/chat")) {
    response.writeHead(200, { "Content-Type": "application/x-ndjson" });
    response.end(JSON.stringify({ message: { content: "ollama" }, model: "stub-text", done: true }) + "\n");
    return;
  }

  // ---- OpenAI-compatible side (the provider under test) ----
  let raw = "";
  request.on("data", (chunk) => (raw += chunk));
  request.on("end", () => {
    lastProviderRequest = { url, headers: request.headers, body: raw ? JSON.parse(raw) : {} };

    if (!request.headers.authorization?.includes(PROVIDER_KEY)) {
      response.writeHead(401, { "Content-Type": "application/json" });
      response.end(JSON.stringify({ error: { message: "missing key" } }));
      return;
    }

    if (url.startsWith("/v1/models")) {
      seen.models += 1;
      response.writeHead(200, { "Content-Type": "application/json" });
      response.end(JSON.stringify({ data: [{ id: "free-model-a" }, { id: "free-model-b" }] }));
      return;
    }

    if (url.startsWith("/v1/chat/completions")) {
      seen.completions += 1;
      const stream = Boolean(lastProviderRequest.body?.stream);
      if (stream) {
        response.writeHead(200, { "Content-Type": "text/event-stream" });
        response.write(`data: ${JSON.stringify({ model: "free-model-a", choices: [{ delta: { content: "PROVIDER-" } }] })}\n\n`);
        response.write(`data: ${JSON.stringify({ choices: [{ delta: { content: "REPLY-MARKER" } }] })}\n\n`);
        response.end("data: [DONE]\n\n");
        return;
      }
      response.writeHead(200, { "Content-Type": "application/json" });
      response.end(
        JSON.stringify({
          model: "free-model-a",
          choices: [{ message: { content: JSON.stringify(DECK_REPLY) } }],
        }),
      );
      return;
    }

    response.writeHead(404, { "Content-Type": "application/json" });
    response.end(JSON.stringify({ error: { message: "not found" } }));
  });
});
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));

check("stub app port is free", !(await portInUse(APP_PORT)));
const app = spawnApp(APP_PORT, {
  OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`,
  OLLAMA_AUTH: "local",
  OLLAMA_VERIFY_KEY: "false",
  CHAT_MODELS: "stub-text,stub-vision",
});
await waitForHttp(`${APP}/api/healthz`);

const providerSeed = {
  keys: { custom: PROVIDER_KEY },
  models: { custom: ["free-model-a"] },
  customBaseUrl: PROVIDER_BASE,
  generation: { provider: "custom", textModel: "free-model-a", visionModel: "free-model-a" },
};

let gateApp = null;
try {
  /* 1. API: the listing route forwards the key and returns the models ---- */
  {
    const missing = await fetch(`${APP}/api/providers`, { headers: { "x-freezmo-provider": "custom", "x-freezmo-provider-base": PROVIDER_BASE } });
    const missingBody = await missing.json();
    check(
      "api: listing without a key explains where to add it",
      missing.status === 401 && /Settings/.test(String(missingBody.error)),
      `${missing.status} ${missingBody.error}`,
    );

    const listed = await fetch(`${APP}/api/providers`, {
      headers: {
        "x-freezmo-provider": "custom",
        "x-freezmo-provider-key": PROVIDER_KEY,
        "x-freezmo-provider-base": PROVIDER_BASE,
      },
    });
    const body = await listed.json();
    check(
      "api: listing returns the provider's models",
      listed.status === 200 && body.models?.includes("free-model-a") && body.models?.includes("free-model-b"),
      JSON.stringify(body).slice(0, 160),
    );
  }

  /* 2. Chat on a provider model ----------------------------------------- */
  {
    const { context, page } = await makePage(browser, { width: 1280, height: 800 });
    await seed(page, { decks: [DECK], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } });
    await page.goto(`${APP}/chat`, { waitUntil: "networkidle" });
    await page.waitForTimeout(1400);

    await page.locator("[data-picker-trigger]").nth(1).click();
    await page.waitForTimeout(500);
    check("chat: the picker lists the provider model", (await page.getByText("free-model-a").count()) >= 1);
    await page.getByRole("button", { name: /free-model-a/ }).first().click();
    await page.waitForTimeout(300);
    check(
      "chat: the chip labels the provider",
      /Custom/.test(await page.locator("[data-picker-trigger]").nth(1).innerText()),
      await page.locator("[data-picker-trigger]").nth(1).innerText(),
    );

    await page.getByRole("textbox").fill("Why is the sky blue?");
    await page.getByRole("button", { name: "Send", exact: true }).click();
    await page.waitForTimeout(2000);
    check("chat: the provider reply streams into the bubble", await page.getByText("PROVIDER-REPLY-MARKER").isVisible());
    // The provider only ever sees the bearer key (the app keeps the routing
    // header to itself), so this asserts the key, the model and the endpoint.
    check(
      "chat: the provider got the browser's key and the picked model",
      String(lastProviderRequest?.headers.authorization).includes(PROVIDER_KEY) &&
        lastProviderRequest?.body?.model === "free-model-a" &&
        lastProviderRequest?.url === "/v1/chat/completions",
      JSON.stringify({ auth: String(lastProviderRequest?.headers.authorization).slice(0, 24), model: lastProviderRequest?.body?.model }),
    );
    await context.close();
  }

  /* 3. Study generation on the provider ---------------------------------- */
  {
    const { context, page } = await makePage(browser, { width: 390, height: 844 });
    await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } });
    await page.goto(`${APP}/create`, { waitUntil: "networkidle" });
    await page.waitForTimeout(800);
    await page.getByRole("button", { name: "Paste text instead" }).click();
    await page.locator("textarea").first().fill("Rayleigh scattering makes the sky blue.");
    await page.getByRole("button", { name: "Next" }).click();
    await page.waitForTimeout(300);
    await page.getByRole("button", { name: "Next" }).click();
    await page.waitForTimeout(300);
    await page.getByRole("button", { name: "Freeze my deck" }).click();
    await page.waitForTimeout(2500);
    if (!page.url().includes("/deck/")) {
      const text = await page.evaluate(() => document.body.innerText.slice(0, 600).replace(/\n+/g, " | "));
      console.log("DEBUG generation page:", text);
      console.log("DEBUG last provider request:", JSON.stringify(lastProviderRequest).slice(0, 300));
    }
    check("generation: the deck lands on the study page", page.url().includes("/deck/"), page.url());
    check("generation: the deck came from the provider", await page.getByText("Provider deck").first().isVisible());
    check(
      "generation: the request carried the provider model",
      lastProviderRequest?.body?.model === "free-model-a" && lastProviderRequest?.body?.stream === false,
      JSON.stringify({ model: lastProviderRequest?.body?.model, stream: lastProviderRequest?.body?.stream }),
    );
    await context.close();
  }

  /* 4. Settings: saving a key lists the models --------------------------- */
  {
    const { context, page } = await makePage(browser, { width: 390, height: 844 });
    await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify({ ...providerSeed, keys: {}, models: {} }) } });
    await page.goto(`${APP}/`, { waitUntil: "networkidle" });
    await page.waitForTimeout(1000);
    await page.locator("nav.tabbar").getByRole("button").click();
    await page.getByRole("dialog").waitFor({ timeout: 6000 });
    await page.waitForTimeout(600);
    check("settings: Google Gemini is offered", (await page.locator('input[aria-label="Google Gemini API key"]').count()) === 1);
    const row = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]');
    await row.scrollIntoViewIfNeeded();
    await row.fill(PROVIDER_KEY);
    await row.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click();
    await page.waitForTimeout(1600);
    check("settings: saving the key lists the models", (await page.getByText("2 models").count()) >= 1);
    check(
      "settings: the key is stored in this browser only",
      (await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}").keys?.custom)) === PROVIDER_KEY,
    );
    await context.close();
  }

  /* 5. First run: the gate offers the free Ollama path and paid providers - */
  {
    const GATE_PORT = 3214;
    check("gate app port is free", !(await portInUse(GATE_PORT)));
    // No OLLAMA_AUTH: the default user-key mode, so the first-run gate shows.
    gateApp = spawnApp(GATE_PORT, {
      OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`,
      OLLAMA_VERIFY_KEY: "false",
    });
    await waitForHttp(`http://127.0.0.1:${GATE_PORT}/api/healthz`);

    const { context, page } = await makePage(browser, { width: 390, height: 844 });
    await seed(page, {
      decks: [],
      key: "",
      extra: {
        "freezmo.providers.v1": JSON.stringify({ keys: {}, models: {}, customBaseUrl: PROVIDER_BASE, generation: null }),
      },
    });
    await page.goto(`http://127.0.0.1:${GATE_PORT}/`, { waitUntil: "networkidle" });
    await page.waitForTimeout(1400);

    check("gate: the free Ollama path is the headline", (await page.getByText("Free").count()) >= 1);
    const paid = page.getByRole("button", { name: "Use a paid provider instead" });
    check("gate: the paid option is visible on first run", await paid.isVisible());
    await paid.click();
    await page.waitForTimeout(500);
    check("gate: the provider rows open", (await page.locator('input[aria-label="OpenAI API key"]').count()) === 1);

    const gateKey = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]');
    await gateKey.scrollIntoViewIfNeeded();
    await gateKey.fill(PROVIDER_KEY);
    await gateKey.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click();
    await page.waitForTimeout(2200);
    check(
      "gate: a saved provider key lets the app through",
      await page.getByRole("heading", { name: "Your icebox" }).isVisible(),
    );
    check(
      "gate: the provider became the study-generation model",
      (await page.evaluate(() => {
        const state = JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}");
        return state.generation?.provider === "custom" && state.generation?.textModel === "free-model-a";
      })),
    );
    await context.close();
  }

  check("stub saw both endpoints", seen.models >= 1 && seen.completions >= 1, JSON.stringify(seen));
} finally {
  gateApp?.stop();
  app.stop();
  stub.close();
  await browser.close();
}

finish();