/** * Cloud providers: bring-your-own keys (OpenAI-compatible endpoints). * * One stub server speaks both protocols: the Ollama endpoints the app's shell * needs (/api/tags) and the OpenAI-compatible endpoints a custom provider uses * (/v1/models, /v1/chat/completions, streaming and not). The app runs with * OLLAMA_AUTH=local against the stub, so the server side needs no key while * the provider side exercises the browser-held key end to end. */ import http from "node:http"; import { DECK, launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs"; const { check, finish } = makeChecker(); const browser = await launchBrowser(); const STUB_PORT = 11998; const APP_PORT = 3213; const APP = `http://127.0.0.1:${APP_PORT}`; const PROVIDER_BASE = `http://127.0.0.1:${STUB_PORT}/v1`; const PROVIDER_KEY = "test-provider-key"; const DECK_REPLY = { title: "Provider deck", subject: "Physics", summary: "Generated on a provider model.", language: "English", activities: [ { type: "multiple_choice", topic: "Sky", explanation: "Rayleigh scattering.", question: "Why is the sky blue?", options: ["Scattering", "Paint", "Wind", "Salt"], correctIndex: 0 }, { type: "true_false", topic: "Sky", explanation: "It scatters.", statement: "Short wavelengths scatter more.", answer: true }, { type: "flashcard", topic: "Sky", explanation: "Basics.", front: "Rayleigh", back: "Scattering by small molecules" }, { type: "fill_blank", topic: "Sky", explanation: "The blank is 'blue'.", sentence: "The clear sky looks ____.", answer: "blue" }, { type: "matching", topic: "Sky", explanation: "Terms and meanings.", instruction: "Match the terms.", pairs: [{ term: "Rayleigh", definition: "Short wavelengths" }, { term: "Molecule", definition: "Small particle" }, { term: "Sky", definition: "Above us" }] }, ], }; let lastProviderRequest = null; const seen = { models: 0, completions: 0 }; const stub = http.createServer((request, response) => { const url = request.url ?? ""; // ---- Ollama side (the app's shell needs a healthy daemon) ---- if (url.startsWith("/api/tags")) { response.writeHead(200, { "Content-Type": "application/json" }); response.end(JSON.stringify({ models: [{ name: "stub-text" }, { name: "stub-vision" }] })); return; } if (url.startsWith("/api/chat")) { response.writeHead(200, { "Content-Type": "application/x-ndjson" }); response.end(JSON.stringify({ message: { content: "ollama" }, model: "stub-text", done: true }) + "\n"); return; } // ---- OpenAI-compatible side (the provider under test) ---- let raw = ""; request.on("data", (chunk) => (raw += chunk)); request.on("end", () => { lastProviderRequest = { url, headers: request.headers, body: raw ? JSON.parse(raw) : {} }; if (!request.headers.authorization?.includes(PROVIDER_KEY)) { response.writeHead(401, { "Content-Type": "application/json" }); response.end(JSON.stringify({ error: { message: "missing key" } })); return; } if (url.startsWith("/v1/models")) { seen.models += 1; response.writeHead(200, { "Content-Type": "application/json" }); response.end(JSON.stringify({ data: [{ id: "free-model-a" }, { id: "free-model-b" }] })); return; } if (url.startsWith("/v1/chat/completions")) { seen.completions += 1; const stream = Boolean(lastProviderRequest.body?.stream); if (stream) { response.writeHead(200, { "Content-Type": "text/event-stream" }); response.write(`data: ${JSON.stringify({ model: "free-model-a", choices: [{ delta: { content: "PROVIDER-" } }] })}\n\n`); response.write(`data: ${JSON.stringify({ choices: [{ delta: { content: "REPLY-MARKER" } }] })}\n\n`); response.end("data: [DONE]\n\n"); return; } response.writeHead(200, { "Content-Type": "application/json" }); response.end( JSON.stringify({ model: "free-model-a", choices: [{ message: { content: JSON.stringify(DECK_REPLY) } }], }), ); return; } response.writeHead(404, { "Content-Type": "application/json" }); response.end(JSON.stringify({ error: { message: "not found" } })); }); }); await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve)); check("stub app port is free", !(await portInUse(APP_PORT))); const app = spawnApp(APP_PORT, { OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`, OLLAMA_AUTH: "local", OLLAMA_VERIFY_KEY: "false", CHAT_MODELS: "stub-text,stub-vision", }); await waitForHttp(`${APP}/api/healthz`); const providerSeed = { keys: { custom: PROVIDER_KEY }, models: { custom: ["free-model-a"] }, customBaseUrl: PROVIDER_BASE, generation: { provider: "custom", textModel: "free-model-a", visionModel: "free-model-a" }, }; let gateApp = null; try { /* 1. API: the listing route forwards the key and returns the models ---- */ { const missing = await fetch(`${APP}/api/providers`, { headers: { "x-freezmo-provider": "custom", "x-freezmo-provider-base": PROVIDER_BASE } }); const missingBody = await missing.json(); check( "api: listing without a key explains where to add it", missing.status === 401 && /Settings/.test(String(missingBody.error)), `${missing.status} ${missingBody.error}`, ); const listed = await fetch(`${APP}/api/providers`, { headers: { "x-freezmo-provider": "custom", "x-freezmo-provider-key": PROVIDER_KEY, "x-freezmo-provider-base": PROVIDER_BASE, }, }); const body = await listed.json(); check( "api: listing returns the provider's models", listed.status === 200 && body.models?.includes("free-model-a") && body.models?.includes("free-model-b"), JSON.stringify(body).slice(0, 160), ); } /* 2. Chat on a provider model ----------------------------------------- */ { const { context, page } = await makePage(browser, { width: 1280, height: 800 }); await seed(page, { decks: [DECK], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } }); await page.goto(`${APP}/chat`, { waitUntil: "networkidle" }); await page.waitForTimeout(1400); await page.locator("[data-picker-trigger]").nth(1).click(); await page.waitForTimeout(500); check("chat: the picker lists the provider model", (await page.getByText("free-model-a").count()) >= 1); await page.getByRole("button", { name: /free-model-a/ }).first().click(); await page.waitForTimeout(300); check( "chat: the chip labels the provider", /Custom/.test(await page.locator("[data-picker-trigger]").nth(1).innerText()), await page.locator("[data-picker-trigger]").nth(1).innerText(), ); await page.getByRole("textbox").fill("Why is the sky blue?"); await page.getByRole("button", { name: "Send", exact: true }).click(); await page.waitForTimeout(2000); check("chat: the provider reply streams into the bubble", await page.getByText("PROVIDER-REPLY-MARKER").isVisible()); // The provider only ever sees the bearer key (the app keeps the routing // header to itself), so this asserts the key, the model and the endpoint. check( "chat: the provider got the browser's key and the picked model", String(lastProviderRequest?.headers.authorization).includes(PROVIDER_KEY) && lastProviderRequest?.body?.model === "free-model-a" && lastProviderRequest?.url === "/v1/chat/completions", JSON.stringify({ auth: String(lastProviderRequest?.headers.authorization).slice(0, 24), model: lastProviderRequest?.body?.model }), ); await context.close(); } /* 3. Study generation on the provider ---------------------------------- */ { const { context, page } = await makePage(browser, { width: 390, height: 844 }); await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } }); await page.goto(`${APP}/create`, { waitUntil: "networkidle" }); await page.waitForTimeout(800); await page.getByRole("button", { name: "Paste text instead" }).click(); await page.locator("textarea").first().fill("Rayleigh scattering makes the sky blue."); await page.getByRole("button", { name: "Next" }).click(); await page.waitForTimeout(300); await page.getByRole("button", { name: "Next" }).click(); await page.waitForTimeout(300); await page.getByRole("button", { name: "Freeze my deck" }).click(); await page.waitForTimeout(2500); if (!page.url().includes("/deck/")) { const text = await page.evaluate(() => document.body.innerText.slice(0, 600).replace(/\n+/g, " | ")); console.log("DEBUG generation page:", text); console.log("DEBUG last provider request:", JSON.stringify(lastProviderRequest).slice(0, 300)); } check("generation: the deck lands on the study page", page.url().includes("/deck/"), page.url()); check("generation: the deck came from the provider", await page.getByText("Provider deck").first().isVisible()); check( "generation: the request carried the provider model", lastProviderRequest?.body?.model === "free-model-a" && lastProviderRequest?.body?.stream === false, JSON.stringify({ model: lastProviderRequest?.body?.model, stream: lastProviderRequest?.body?.stream }), ); await context.close(); } /* 4. Settings: saving a key lists the models --------------------------- */ { const { context, page } = await makePage(browser, { width: 390, height: 844 }); await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify({ ...providerSeed, keys: {}, models: {} }) } }); await page.goto(`${APP}/`, { waitUntil: "networkidle" }); await page.waitForTimeout(1000); await page.locator("nav.tabbar").getByRole("button").click(); await page.getByRole("dialog").waitFor({ timeout: 6000 }); await page.waitForTimeout(600); check("settings: Google Gemini is offered", (await page.locator('input[aria-label="Google Gemini API key"]').count()) === 1); const row = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]'); await row.scrollIntoViewIfNeeded(); await row.fill(PROVIDER_KEY); await row.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click(); await page.waitForTimeout(1600); check("settings: saving the key lists the models", (await page.getByText("2 models").count()) >= 1); check( "settings: the key is stored in this browser only", (await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}").keys?.custom)) === PROVIDER_KEY, ); await context.close(); } /* 5. First run: the gate offers the free Ollama path and paid providers - */ { const GATE_PORT = 3214; check("gate app port is free", !(await portInUse(GATE_PORT))); // No OLLAMA_AUTH: the default user-key mode, so the first-run gate shows. gateApp = spawnApp(GATE_PORT, { OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`, OLLAMA_VERIFY_KEY: "false", }); await waitForHttp(`http://127.0.0.1:${GATE_PORT}/api/healthz`); const { context, page } = await makePage(browser, { width: 390, height: 844 }); await seed(page, { decks: [], key: "", extra: { "freezmo.providers.v1": JSON.stringify({ keys: {}, models: {}, customBaseUrl: PROVIDER_BASE, generation: null }), }, }); /* A driveable visual viewport, installed before the app mounts, so the on-screen-keyboard layout can be tested without a real keyboard. */ await page.addInitScript(() => { window.__vv = { resize: [] }; Object.defineProperty(window, "visualViewport", { configurable: true, get: () => ({ get height() { return window.__vvHeight ?? window.innerHeight; }, get offsetTop() { return window.__vvOffset ?? 0; }, addEventListener: (type, handler) => { (window.__vv[type] ??= []).push(handler); }, removeEventListener: (type, handler) => { window.__vv[type] = (window.__vv[type] ?? []).filter((entry) => entry !== handler); }, }), }); }); await page.goto(`http://127.0.0.1:${GATE_PORT}/`, { waitUntil: "networkidle" }); await page.waitForTimeout(1400); check("gate: the free Ollama path is the headline", (await page.getByText("Free").count()) >= 1); const paid = page.getByRole("button", { name: "Use a paid provider instead" }); check("gate: the paid option is visible on first run", await paid.isVisible()); await paid.click(); await page.waitForTimeout(500); check("gate: the provider rows open", (await page.locator('input[aria-label="OpenAI API key"]').count()) === 1); const gateKey = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]'); await gateKey.scrollIntoViewIfNeeded(); /* An API key is not a login: the field must keep iOS/Safari's saved password sheet away ("new-password" + the password-manager hints), or the AutoFill overlay lands on top of the gate (as it did on the user's phone). */ const hardening = await gateKey.evaluate((element) => ({ autocomplete: element.getAttribute("autocomplete"), formType: element.getAttribute("data-form-type"), lpignore: element.getAttribute("data-lpignore"), inputMode: element.getAttribute("inputmode"), })); check( "gate: the key field keeps the password UI away", hardening.autocomplete === "new-password" && hardening.formType === "other" && hardening.lpignore === "true" && hardening.inputMode === "text", JSON.stringify(hardening), ); /* The on-screen keyboard: fake the visual viewport shrinking the way a phone keyboard does, then check the layout still holds — the focused field stays in view, the wrapper stays scrollable and the corner controls step aside instead of landing on the card. */ await gateKey.focus(); await page.evaluate(() => { window.__vvHeight = 340; (window.__vv.resize ?? []).forEach((handler) => handler()); }); await page.waitForTimeout(500); const keyboardLayout = await page.evaluate(() => { const input = document.querySelector('input[aria-label="Custom (OpenAI-compatible) API key"]'); const root = document.querySelector("div[style*='height']"); const wrap = input?.closest("div[class*='overflow-y-auto']"); const corner = document.querySelector("button[aria-label='How do I get a key?']"); return { inputBottom: input ? Math.round(input.getBoundingClientRect().bottom) : null, rootHeight: root ? Math.round(root.getBoundingClientRect().height) : null, wrapped: Boolean(wrap), scrollable: wrap ? getComputedStyle(wrap).overflowY : null, cornerVisible: corner ? corner.offsetParent !== null : false, }; }); check( "gate: with the keyboard up the field stays in view", keyboardLayout.inputBottom !== null && keyboardLayout.rootHeight !== null && keyboardLayout.inputBottom <= keyboardLayout.rootHeight, JSON.stringify(keyboardLayout), ); check( "gate: with the keyboard up the card stays scrollable", keyboardLayout.wrapped && keyboardLayout.scrollable === "auto", JSON.stringify(keyboardLayout), ); check("gate: the corner controls step aside for the keyboard", keyboardLayout.cornerVisible === false, JSON.stringify(keyboardLayout)); await page.evaluate(() => { window.__vvHeight = undefined; (window.__vv.resize ?? []).forEach((handler) => handler()); }); await page.waitForTimeout(300); await gateKey.fill(PROVIDER_KEY); await gateKey.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click(); await page.waitForTimeout(2200); check( "gate: a saved provider key lets the app through", await page.getByRole("heading", { name: "Your freezer" }).isVisible(), ); check( "gate: the provider became the study-generation model", (await page.evaluate(() => { const state = JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}"); return state.generation?.provider === "custom" && state.generation?.textModel === "free-model-a"; })), ); await context.close(); } check("stub saw both endpoints", seen.models >= 1 && seen.completions >= 1, JSON.stringify(seen)); } finally { gateApp?.stop(); app.stop(); stub.close(); await browser.close(); } finish();