Martin Dvorak
84a94581dc
"Trying to use a different provider on the web app glitches out completely" -
an iPhone photo of the first-run gate with iOS's saved-password sheet laid
over it ("password for proxmox...") and the provider field stuck under the
keyboard.
- The key fields (gate, Settings, provider rows) were type=password with
autocomplete=off, which iOS/Safari and 1Password/LastPass read as a login.
They now carry autoComplete="new-password", data-form-type="other",
data-lpignore="true", autoCorrect/autoCapitalize=off and inputMode=text:
the mask stays, the password overlay goes. The custom base URL gets
inputMode=url and the same hints.
- The gate's keyboard handling scrolled to the Continue button instead of the
field being edited, so with the provider rows open the keyboard covered the
input and the locked wrapper could not be scrolled to it. The wrapper is
user-scrollable again, alignment anchors on the focused element (Continue
is the fallback), and the corner controls hide while the keyboard is up
rather than landing on the card.
providers-test drives a fake visualViewport (340px, installed before mount)
and pins the three layout contracts plus the hardening attributes. All 16
suites green, contrast clean. Version 0.7.2, SW v94.
367 lines
16 KiB
JavaScript
367 lines
16 KiB
JavaScript
/**
|
|
* Cloud providers: bring-your-own keys (OpenAI-compatible endpoints).
|
|
*
|
|
* One stub server speaks both protocols: the Ollama endpoints the app's shell
|
|
* needs (/api/tags) and the OpenAI-compatible endpoints a custom provider uses
|
|
* (/v1/models, /v1/chat/completions, streaming and not). The app runs with
|
|
* OLLAMA_AUTH=local against the stub, so the server side needs no key while
|
|
* the provider side exercises the browser-held key end to end.
|
|
*/
|
|
import http from "node:http";
|
|
|
|
import { DECK, launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";
|
|
|
|
const { check, finish } = makeChecker();
|
|
const browser = await launchBrowser();
|
|
|
|
const STUB_PORT = 11998;
|
|
const APP_PORT = 3213;
|
|
const APP = `http://127.0.0.1:${APP_PORT}`;
|
|
const PROVIDER_BASE = `http://127.0.0.1:${STUB_PORT}/v1`;
|
|
const PROVIDER_KEY = "test-provider-key";
|
|
|
|
const DECK_REPLY = {
|
|
title: "Provider deck",
|
|
subject: "Physics",
|
|
summary: "Generated on a provider model.",
|
|
language: "English",
|
|
activities: [
|
|
{ type: "multiple_choice", topic: "Sky", explanation: "Rayleigh scattering.", question: "Why is the sky blue?", options: ["Scattering", "Paint", "Wind", "Salt"], correctIndex: 0 },
|
|
{ type: "true_false", topic: "Sky", explanation: "It scatters.", statement: "Short wavelengths scatter more.", answer: true },
|
|
{ type: "flashcard", topic: "Sky", explanation: "Basics.", front: "Rayleigh", back: "Scattering by small molecules" },
|
|
{ type: "fill_blank", topic: "Sky", explanation: "The blank is 'blue'.", sentence: "The clear sky looks ____.", answer: "blue" },
|
|
{ type: "matching", topic: "Sky", explanation: "Terms and meanings.", instruction: "Match the terms.", pairs: [{ term: "Rayleigh", definition: "Short wavelengths" }, { term: "Molecule", definition: "Small particle" }, { term: "Sky", definition: "Above us" }] },
|
|
],
|
|
};
|
|
|
|
let lastProviderRequest = null;
|
|
const seen = { models: 0, completions: 0 };
|
|
|
|
const stub = http.createServer((request, response) => {
|
|
const url = request.url ?? "";
|
|
|
|
// ---- Ollama side (the app's shell needs a healthy daemon) ----
|
|
if (url.startsWith("/api/tags")) {
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ models: [{ name: "stub-text" }, { name: "stub-vision" }] }));
|
|
return;
|
|
}
|
|
if (url.startsWith("/api/chat")) {
|
|
response.writeHead(200, { "Content-Type": "application/x-ndjson" });
|
|
response.end(JSON.stringify({ message: { content: "ollama" }, model: "stub-text", done: true }) + "\n");
|
|
return;
|
|
}
|
|
|
|
// ---- OpenAI-compatible side (the provider under test) ----
|
|
let raw = "";
|
|
request.on("data", (chunk) => (raw += chunk));
|
|
request.on("end", () => {
|
|
lastProviderRequest = { url, headers: request.headers, body: raw ? JSON.parse(raw) : {} };
|
|
|
|
if (!request.headers.authorization?.includes(PROVIDER_KEY)) {
|
|
response.writeHead(401, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ error: { message: "missing key" } }));
|
|
return;
|
|
}
|
|
|
|
if (url.startsWith("/v1/models")) {
|
|
seen.models += 1;
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ data: [{ id: "free-model-a" }, { id: "free-model-b" }] }));
|
|
return;
|
|
}
|
|
|
|
if (url.startsWith("/v1/chat/completions")) {
|
|
seen.completions += 1;
|
|
const stream = Boolean(lastProviderRequest.body?.stream);
|
|
if (stream) {
|
|
response.writeHead(200, { "Content-Type": "text/event-stream" });
|
|
response.write(`data: ${JSON.stringify({ model: "free-model-a", choices: [{ delta: { content: "PROVIDER-" } }] })}\n\n`);
|
|
response.write(`data: ${JSON.stringify({ choices: [{ delta: { content: "REPLY-MARKER" } }] })}\n\n`);
|
|
response.end("data: [DONE]\n\n");
|
|
return;
|
|
}
|
|
response.writeHead(200, { "Content-Type": "application/json" });
|
|
response.end(
|
|
JSON.stringify({
|
|
model: "free-model-a",
|
|
choices: [{ message: { content: JSON.stringify(DECK_REPLY) } }],
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
response.writeHead(404, { "Content-Type": "application/json" });
|
|
response.end(JSON.stringify({ error: { message: "not found" } }));
|
|
});
|
|
});
|
|
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));
|
|
|
|
check("stub app port is free", !(await portInUse(APP_PORT)));
|
|
const app = spawnApp(APP_PORT, {
|
|
OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`,
|
|
OLLAMA_AUTH: "local",
|
|
OLLAMA_VERIFY_KEY: "false",
|
|
CHAT_MODELS: "stub-text,stub-vision",
|
|
});
|
|
await waitForHttp(`${APP}/api/healthz`);
|
|
|
|
const providerSeed = {
|
|
keys: { custom: PROVIDER_KEY },
|
|
models: { custom: ["free-model-a"] },
|
|
customBaseUrl: PROVIDER_BASE,
|
|
generation: { provider: "custom", textModel: "free-model-a", visionModel: "free-model-a" },
|
|
};
|
|
|
|
let gateApp = null;
|
|
try {
|
|
/* 1. API: the listing route forwards the key and returns the models ---- */
|
|
{
|
|
const missing = await fetch(`${APP}/api/providers`, { headers: { "x-freezmo-provider": "custom", "x-freezmo-provider-base": PROVIDER_BASE } });
|
|
const missingBody = await missing.json();
|
|
check(
|
|
"api: listing without a key explains where to add it",
|
|
missing.status === 401 && /Settings/.test(String(missingBody.error)),
|
|
`${missing.status} ${missingBody.error}`,
|
|
);
|
|
|
|
const listed = await fetch(`${APP}/api/providers`, {
|
|
headers: {
|
|
"x-freezmo-provider": "custom",
|
|
"x-freezmo-provider-key": PROVIDER_KEY,
|
|
"x-freezmo-provider-base": PROVIDER_BASE,
|
|
},
|
|
});
|
|
const body = await listed.json();
|
|
check(
|
|
"api: listing returns the provider's models",
|
|
listed.status === 200 && body.models?.includes("free-model-a") && body.models?.includes("free-model-b"),
|
|
JSON.stringify(body).slice(0, 160),
|
|
);
|
|
}
|
|
|
|
/* 2. Chat on a provider model ----------------------------------------- */
|
|
{
|
|
const { context, page } = await makePage(browser, { width: 1280, height: 800 });
|
|
await seed(page, { decks: [DECK], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } });
|
|
await page.goto(`${APP}/chat`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1400);
|
|
|
|
await page.locator("[data-picker-trigger]").nth(1).click();
|
|
await page.waitForTimeout(500);
|
|
check("chat: the picker lists the provider model", (await page.getByText("free-model-a").count()) >= 1);
|
|
await page.getByRole("button", { name: /free-model-a/ }).first().click();
|
|
await page.waitForTimeout(300);
|
|
check(
|
|
"chat: the chip labels the provider",
|
|
/Custom/.test(await page.locator("[data-picker-trigger]").nth(1).innerText()),
|
|
await page.locator("[data-picker-trigger]").nth(1).innerText(),
|
|
);
|
|
|
|
await page.getByRole("textbox").fill("Why is the sky blue?");
|
|
await page.getByRole("button", { name: "Send", exact: true }).click();
|
|
await page.waitForTimeout(2000);
|
|
check("chat: the provider reply streams into the bubble", await page.getByText("PROVIDER-REPLY-MARKER").isVisible());
|
|
// The provider only ever sees the bearer key (the app keeps the routing
|
|
// header to itself), so this asserts the key, the model and the endpoint.
|
|
check(
|
|
"chat: the provider got the browser's key and the picked model",
|
|
String(lastProviderRequest?.headers.authorization).includes(PROVIDER_KEY) &&
|
|
lastProviderRequest?.body?.model === "free-model-a" &&
|
|
lastProviderRequest?.url === "/v1/chat/completions",
|
|
JSON.stringify({ auth: String(lastProviderRequest?.headers.authorization).slice(0, 24), model: lastProviderRequest?.body?.model }),
|
|
);
|
|
await context.close();
|
|
}
|
|
|
|
/* 3. Study generation on the provider ---------------------------------- */
|
|
{
|
|
const { context, page } = await makePage(browser, { width: 390, height: 844 });
|
|
await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } });
|
|
await page.goto(`${APP}/create`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(800);
|
|
await page.getByRole("button", { name: "Paste text instead" }).click();
|
|
await page.locator("textarea").first().fill("Rayleigh scattering makes the sky blue.");
|
|
await page.getByRole("button", { name: "Next" }).click();
|
|
await page.waitForTimeout(300);
|
|
await page.getByRole("button", { name: "Next" }).click();
|
|
await page.waitForTimeout(300);
|
|
await page.getByRole("button", { name: "Freeze my deck" }).click();
|
|
await page.waitForTimeout(2500);
|
|
if (!page.url().includes("/deck/")) {
|
|
const text = await page.evaluate(() => document.body.innerText.slice(0, 600).replace(/\n+/g, " | "));
|
|
console.log("DEBUG generation page:", text);
|
|
console.log("DEBUG last provider request:", JSON.stringify(lastProviderRequest).slice(0, 300));
|
|
}
|
|
check("generation: the deck lands on the study page", page.url().includes("/deck/"), page.url());
|
|
check("generation: the deck came from the provider", await page.getByText("Provider deck").first().isVisible());
|
|
check(
|
|
"generation: the request carried the provider model",
|
|
lastProviderRequest?.body?.model === "free-model-a" && lastProviderRequest?.body?.stream === false,
|
|
JSON.stringify({ model: lastProviderRequest?.body?.model, stream: lastProviderRequest?.body?.stream }),
|
|
);
|
|
await context.close();
|
|
}
|
|
|
|
/* 4. Settings: saving a key lists the models --------------------------- */
|
|
{
|
|
const { context, page } = await makePage(browser, { width: 390, height: 844 });
|
|
await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify({ ...providerSeed, keys: {}, models: {} }) } });
|
|
await page.goto(`${APP}/`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1000);
|
|
await page.locator("nav.tabbar").getByRole("button").click();
|
|
await page.getByRole("dialog").waitFor({ timeout: 6000 });
|
|
await page.waitForTimeout(600);
|
|
check("settings: Google Gemini is offered", (await page.locator('input[aria-label="Google Gemini API key"]').count()) === 1);
|
|
const row = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]');
|
|
await row.scrollIntoViewIfNeeded();
|
|
await row.fill(PROVIDER_KEY);
|
|
await row.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click();
|
|
await page.waitForTimeout(1600);
|
|
check("settings: saving the key lists the models", (await page.getByText("2 models").count()) >= 1);
|
|
check(
|
|
"settings: the key is stored in this browser only",
|
|
(await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}").keys?.custom)) === PROVIDER_KEY,
|
|
);
|
|
await context.close();
|
|
}
|
|
|
|
/* 5. First run: the gate offers the free Ollama path and paid providers - */
|
|
{
|
|
const GATE_PORT = 3214;
|
|
check("gate app port is free", !(await portInUse(GATE_PORT)));
|
|
// No OLLAMA_AUTH: the default user-key mode, so the first-run gate shows.
|
|
gateApp = spawnApp(GATE_PORT, {
|
|
OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`,
|
|
OLLAMA_VERIFY_KEY: "false",
|
|
});
|
|
await waitForHttp(`http://127.0.0.1:${GATE_PORT}/api/healthz`);
|
|
|
|
const { context, page } = await makePage(browser, { width: 390, height: 844 });
|
|
await seed(page, {
|
|
decks: [],
|
|
key: "",
|
|
extra: {
|
|
"freezmo.providers.v1": JSON.stringify({ keys: {}, models: {}, customBaseUrl: PROVIDER_BASE, generation: null }),
|
|
},
|
|
});
|
|
/* A driveable visual viewport, installed before the app mounts, so the
|
|
on-screen-keyboard layout can be tested without a real keyboard. */
|
|
await page.addInitScript(() => {
|
|
window.__vv = { resize: [] };
|
|
Object.defineProperty(window, "visualViewport", {
|
|
configurable: true,
|
|
get: () => ({
|
|
get height() {
|
|
return window.__vvHeight ?? window.innerHeight;
|
|
},
|
|
get offsetTop() {
|
|
return window.__vvOffset ?? 0;
|
|
},
|
|
addEventListener: (type, handler) => {
|
|
(window.__vv[type] ??= []).push(handler);
|
|
},
|
|
removeEventListener: (type, handler) => {
|
|
window.__vv[type] = (window.__vv[type] ?? []).filter((entry) => entry !== handler);
|
|
},
|
|
}),
|
|
});
|
|
});
|
|
await page.goto(`http://127.0.0.1:${GATE_PORT}/`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1400);
|
|
|
|
check("gate: the free Ollama path is the headline", (await page.getByText("Free").count()) >= 1);
|
|
const paid = page.getByRole("button", { name: "Use a paid provider instead" });
|
|
check("gate: the paid option is visible on first run", await paid.isVisible());
|
|
await paid.click();
|
|
await page.waitForTimeout(500);
|
|
check("gate: the provider rows open", (await page.locator('input[aria-label="OpenAI API key"]').count()) === 1);
|
|
|
|
const gateKey = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]');
|
|
await gateKey.scrollIntoViewIfNeeded();
|
|
|
|
/* An API key is not a login: the field must keep iOS/Safari's saved
|
|
password sheet away ("new-password" + the password-manager hints), or
|
|
the AutoFill overlay lands on top of the gate (as it did on the user's
|
|
phone). */
|
|
const hardening = await gateKey.evaluate((element) => ({
|
|
autocomplete: element.getAttribute("autocomplete"),
|
|
formType: element.getAttribute("data-form-type"),
|
|
lpignore: element.getAttribute("data-lpignore"),
|
|
inputMode: element.getAttribute("inputmode"),
|
|
}));
|
|
check(
|
|
"gate: the key field keeps the password UI away",
|
|
hardening.autocomplete === "new-password" &&
|
|
hardening.formType === "other" &&
|
|
hardening.lpignore === "true" &&
|
|
hardening.inputMode === "text",
|
|
JSON.stringify(hardening),
|
|
);
|
|
|
|
/* The on-screen keyboard: fake the visual viewport shrinking the way a
|
|
phone keyboard does, then check the layout still holds — the focused
|
|
field stays in view, the wrapper stays scrollable and the corner
|
|
controls step aside instead of landing on the card. */
|
|
await gateKey.focus();
|
|
await page.evaluate(() => {
|
|
window.__vvHeight = 340;
|
|
(window.__vv.resize ?? []).forEach((handler) => handler());
|
|
});
|
|
await page.waitForTimeout(500);
|
|
const keyboardLayout = await page.evaluate(() => {
|
|
const input = document.querySelector('input[aria-label="Custom (OpenAI-compatible) API key"]');
|
|
const root = document.querySelector("div[style*='height']");
|
|
const wrap = input?.closest("div[class*='overflow-y-auto']");
|
|
const corner = document.querySelector("button[aria-label='How do I get a key?']");
|
|
return {
|
|
inputBottom: input ? Math.round(input.getBoundingClientRect().bottom) : null,
|
|
rootHeight: root ? Math.round(root.getBoundingClientRect().height) : null,
|
|
wrapped: Boolean(wrap),
|
|
scrollable: wrap ? getComputedStyle(wrap).overflowY : null,
|
|
cornerVisible: corner ? corner.offsetParent !== null : false,
|
|
};
|
|
});
|
|
check(
|
|
"gate: with the keyboard up the field stays in view",
|
|
keyboardLayout.inputBottom !== null && keyboardLayout.rootHeight !== null && keyboardLayout.inputBottom <= keyboardLayout.rootHeight,
|
|
JSON.stringify(keyboardLayout),
|
|
);
|
|
check(
|
|
"gate: with the keyboard up the card stays scrollable",
|
|
keyboardLayout.wrapped && keyboardLayout.scrollable === "auto",
|
|
JSON.stringify(keyboardLayout),
|
|
);
|
|
check("gate: the corner controls step aside for the keyboard", keyboardLayout.cornerVisible === false, JSON.stringify(keyboardLayout));
|
|
await page.evaluate(() => {
|
|
window.__vvHeight = undefined;
|
|
(window.__vv.resize ?? []).forEach((handler) => handler());
|
|
});
|
|
await page.waitForTimeout(300);
|
|
|
|
await gateKey.fill(PROVIDER_KEY);
|
|
await gateKey.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click();
|
|
await page.waitForTimeout(2200);
|
|
check(
|
|
"gate: a saved provider key lets the app through",
|
|
await page.getByRole("heading", { name: "Your freezer" }).isVisible(),
|
|
);
|
|
check(
|
|
"gate: the provider became the study-generation model",
|
|
(await page.evaluate(() => {
|
|
const state = JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}");
|
|
return state.generation?.provider === "custom" && state.generation?.textModel === "free-model-a";
|
|
})),
|
|
);
|
|
await context.close();
|
|
}
|
|
|
|
check("stub saw both endpoints", seen.models >= 1 && seen.completions >= 1, JSON.stringify(seen));
|
|
} finally {
|
|
gateApp?.stop();
|
|
app.stop();
|
|
stub.close();
|
|
await browser.close();
|
|
}
|
|
|
|
finish();
|