Bring-your-own keys for cloud providers (OpenAI-compatible)
Settings -> Providers takes keys for OpenAI, Anthropic (their official OpenAI-compatibility layer), DeepSeek, Groq, OpenRouter and any custom OpenAI-compatible endpoint. One adapter (lib/openai-compat) covers listing, streaming chat, JSON generation and vision, and every model flow can run on a provider: the chat picker lists each enabled provider's models (labelled, remembered), and a "Study generation" choice moves decks, homework, grading and attachment transcription onto a provider model. Keys stay in this browser exactly like the Ollama one (freezmo.providers.v1): never stored server-side, never in a backup or the sync payload, forwarded only with the request that needs them. The provider test asserts the key reaches the provider while the routing header stays on the app side. New suite tests/providers-test.mjs (13 checks) stubs both protocols: model listing, chat streaming, study generation and the Settings flow. Privacy policy updated (en+cs). Contrast 0 findings, axe clean, mobile 126/126, chat 102/102, wizard 95/95, bubbles 34/34, regression 44/44, i18n 13/13, native 33/33. Version 0.4.7, SW -> freezmo-v72.
This commit is contained in:
parent
37f7d22041
commit
4372423167
28 changed files with 1538 additions and 150 deletions
|
|
@ -147,15 +147,15 @@ for (const locale of ["en", "cs"]) {
|
|||
check(`${name} >= 44px`, bounds.width >= 43.5 && bounds.height >= 43.5, `${Math.round(bounds.width)}x${Math.round(bounds.height)}`);
|
||||
}
|
||||
|
||||
check("Save hidden until the key is edited", (await dialog.getByRole("button", { name: "Uložit" }).count()) === 0);
|
||||
check("Save hidden until the key is edited", (await dialog.getByRole("button", { name: "Uložit", exact: true }).count()) === 0);
|
||||
const keyInput = dialog.locator("#ollama-api-key-phone");
|
||||
await keyInput.fill("test-key2");
|
||||
await page.waitForTimeout(200);
|
||||
const save = dialog.getByRole("button", { name: "Uložit" });
|
||||
const save = dialog.getByRole("button", { name: "Uložit", exact: true });
|
||||
check("Save appears after editing, >= 44px", (await save.isVisible()) && (await box(save)).height >= 43.5);
|
||||
await keyInput.fill("test-key");
|
||||
await page.waitForTimeout(200);
|
||||
check("Save hides again when unchanged", (await dialog.getByRole("button", { name: "Uložit" }).count()) === 0);
|
||||
check("Save hides again when unchanged", (await dialog.getByRole("button", { name: "Uložit", exact: true }).count()) === 0);
|
||||
|
||||
const tip = dialog.locator("details").first();
|
||||
check("privacy note starts collapsed", !(await tip.evaluate((el) => el.open)));
|
||||
|
|
|
|||
234
tests/providers-test.mjs
Normal file
234
tests/providers-test.mjs
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
/**
|
||||
* Cloud providers: bring-your-own keys (OpenAI-compatible endpoints).
|
||||
*
|
||||
* One stub server speaks both protocols: the Ollama endpoints the app's shell
|
||||
* needs (/api/tags) and the OpenAI-compatible endpoints a custom provider uses
|
||||
* (/v1/models, /v1/chat/completions, streaming and not). The app runs with
|
||||
* OLLAMA_AUTH=local against the stub, so the server side needs no key while
|
||||
* the provider side exercises the browser-held key end to end.
|
||||
*/
|
||||
import http from "node:http";
|
||||
|
||||
import { DECK, launchBrowser, makeChecker, makePage, portInUse, seed, spawnApp, waitForHttp } from "./lib.mjs";
|
||||
|
||||
const { check, finish } = makeChecker();
|
||||
const browser = await launchBrowser();
|
||||
|
||||
const STUB_PORT = 11998;
|
||||
const APP_PORT = 3213;
|
||||
const APP = `http://127.0.0.1:${APP_PORT}`;
|
||||
const PROVIDER_BASE = `http://127.0.0.1:${STUB_PORT}/v1`;
|
||||
const PROVIDER_KEY = "test-provider-key";
|
||||
|
||||
const DECK_REPLY = {
|
||||
title: "Provider deck",
|
||||
subject: "Physics",
|
||||
summary: "Generated on a provider model.",
|
||||
language: "English",
|
||||
activities: [
|
||||
{ type: "multiple_choice", topic: "Sky", explanation: "Rayleigh scattering.", question: "Why is the sky blue?", options: ["Scattering", "Paint", "Wind", "Salt"], correctIndex: 0 },
|
||||
{ type: "true_false", topic: "Sky", explanation: "It scatters.", statement: "Short wavelengths scatter more.", answer: true },
|
||||
{ type: "flashcard", topic: "Sky", explanation: "Basics.", front: "Rayleigh", back: "Scattering by small molecules" },
|
||||
{ type: "fill_blank", topic: "Sky", explanation: "The blank is 'blue'.", sentence: "The clear sky looks ____.", answer: "blue" },
|
||||
{ type: "matching", topic: "Sky", explanation: "Terms and meanings.", instruction: "Match the terms.", pairs: [{ term: "Rayleigh", definition: "Short wavelengths" }, { term: "Molecule", definition: "Small particle" }, { term: "Sky", definition: "Above us" }] },
|
||||
],
|
||||
};
|
||||
|
||||
let lastProviderRequest = null;
|
||||
const seen = { models: 0, completions: 0 };
|
||||
|
||||
const stub = http.createServer((request, response) => {
|
||||
const url = request.url ?? "";
|
||||
|
||||
// ---- Ollama side (the app's shell needs a healthy daemon) ----
|
||||
if (url.startsWith("/api/tags")) {
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ models: [{ name: "stub-text" }, { name: "stub-vision" }] }));
|
||||
return;
|
||||
}
|
||||
if (url.startsWith("/api/chat")) {
|
||||
response.writeHead(200, { "Content-Type": "application/x-ndjson" });
|
||||
response.end(JSON.stringify({ message: { content: "ollama" }, model: "stub-text", done: true }) + "\n");
|
||||
return;
|
||||
}
|
||||
|
||||
// ---- OpenAI-compatible side (the provider under test) ----
|
||||
let raw = "";
|
||||
request.on("data", (chunk) => (raw += chunk));
|
||||
request.on("end", () => {
|
||||
lastProviderRequest = { url, headers: request.headers, body: raw ? JSON.parse(raw) : {} };
|
||||
|
||||
if (!request.headers.authorization?.includes(PROVIDER_KEY)) {
|
||||
response.writeHead(401, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ error: { message: "missing key" } }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.startsWith("/v1/models")) {
|
||||
seen.models += 1;
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "free-model-a" }, { id: "free-model-b" }] }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.startsWith("/v1/chat/completions")) {
|
||||
seen.completions += 1;
|
||||
const stream = Boolean(lastProviderRequest.body?.stream);
|
||||
if (stream) {
|
||||
response.writeHead(200, { "Content-Type": "text/event-stream" });
|
||||
response.write(`data: ${JSON.stringify({ model: "free-model-a", choices: [{ delta: { content: "PROVIDER-" } }] })}\n\n`);
|
||||
response.write(`data: ${JSON.stringify({ choices: [{ delta: { content: "REPLY-MARKER" } }] })}\n\n`);
|
||||
response.end("data: [DONE]\n\n");
|
||||
return;
|
||||
}
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
model: "free-model-a",
|
||||
choices: [{ message: { content: JSON.stringify(DECK_REPLY) } }],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(404, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ error: { message: "not found" } }));
|
||||
});
|
||||
});
|
||||
await new Promise((resolve) => stub.listen(STUB_PORT, "127.0.0.1", resolve));
|
||||
|
||||
check("stub app port is free", !(await portInUse(APP_PORT)));
|
||||
const app = spawnApp(APP_PORT, {
|
||||
OLLAMA_BASE_URL: `http://127.0.0.1:${STUB_PORT}`,
|
||||
OLLAMA_AUTH: "local",
|
||||
OLLAMA_VERIFY_KEY: "false",
|
||||
CHAT_MODELS: "stub-text,stub-vision",
|
||||
});
|
||||
await waitForHttp(`${APP}/api/healthz`);
|
||||
|
||||
const providerSeed = {
|
||||
keys: { custom: PROVIDER_KEY },
|
||||
models: { custom: ["free-model-a"] },
|
||||
customBaseUrl: PROVIDER_BASE,
|
||||
generation: { provider: "custom", textModel: "free-model-a", visionModel: "free-model-a" },
|
||||
};
|
||||
|
||||
try {
|
||||
/* 1. API: the listing route forwards the key and returns the models ---- */
|
||||
{
|
||||
const missing = await fetch(`${APP}/api/providers`, { headers: { "x-freezmo-provider": "custom", "x-freezmo-provider-base": PROVIDER_BASE } });
|
||||
const missingBody = await missing.json();
|
||||
check(
|
||||
"api: listing without a key explains where to add it",
|
||||
missing.status === 401 && /Settings/.test(String(missingBody.error)),
|
||||
`${missing.status} ${missingBody.error}`,
|
||||
);
|
||||
|
||||
const listed = await fetch(`${APP}/api/providers`, {
|
||||
headers: {
|
||||
"x-freezmo-provider": "custom",
|
||||
"x-freezmo-provider-key": PROVIDER_KEY,
|
||||
"x-freezmo-provider-base": PROVIDER_BASE,
|
||||
},
|
||||
});
|
||||
const body = await listed.json();
|
||||
check(
|
||||
"api: listing returns the provider's models",
|
||||
listed.status === 200 && body.models?.includes("free-model-a") && body.models?.includes("free-model-b"),
|
||||
JSON.stringify(body).slice(0, 160),
|
||||
);
|
||||
}
|
||||
|
||||
/* 2. Chat on a provider model ----------------------------------------- */
|
||||
{
|
||||
const { context, page } = await makePage(browser, { width: 1280, height: 800 });
|
||||
await seed(page, { decks: [DECK], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } });
|
||||
await page.goto(`${APP}/chat`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(1400);
|
||||
|
||||
await page.locator("[data-picker-trigger]").nth(1).click();
|
||||
await page.waitForTimeout(500);
|
||||
check("chat: the picker lists the provider model", (await page.getByText("free-model-a").count()) >= 1);
|
||||
await page.getByRole("button", { name: /free-model-a/ }).first().click();
|
||||
await page.waitForTimeout(300);
|
||||
check(
|
||||
"chat: the chip labels the provider",
|
||||
/Custom/.test(await page.locator("[data-picker-trigger]").nth(1).innerText()),
|
||||
await page.locator("[data-picker-trigger]").nth(1).innerText(),
|
||||
);
|
||||
|
||||
await page.getByRole("textbox").fill("Why is the sky blue?");
|
||||
await page.getByRole("button", { name: "Send", exact: true }).click();
|
||||
await page.waitForTimeout(2000);
|
||||
check("chat: the provider reply streams into the bubble", await page.getByText("PROVIDER-REPLY-MARKER").isVisible());
|
||||
// The provider only ever sees the bearer key (the app keeps the routing
|
||||
// header to itself), so this asserts the key, the model and the endpoint.
|
||||
check(
|
||||
"chat: the provider got the browser's key and the picked model",
|
||||
String(lastProviderRequest?.headers.authorization).includes(PROVIDER_KEY) &&
|
||||
lastProviderRequest?.body?.model === "free-model-a" &&
|
||||
lastProviderRequest?.url === "/v1/chat/completions",
|
||||
JSON.stringify({ auth: String(lastProviderRequest?.headers.authorization).slice(0, 24), model: lastProviderRequest?.body?.model }),
|
||||
);
|
||||
await context.close();
|
||||
}
|
||||
|
||||
/* 3. Study generation on the provider ---------------------------------- */
|
||||
{
|
||||
const { context, page } = await makePage(browser, { width: 390, height: 844 });
|
||||
await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify(providerSeed) } });
|
||||
await page.goto(`${APP}/create`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(800);
|
||||
await page.getByRole("button", { name: "Paste text instead" }).click();
|
||||
await page.locator("textarea").first().fill("Rayleigh scattering makes the sky blue.");
|
||||
await page.getByRole("button", { name: "Next" }).click();
|
||||
await page.waitForTimeout(300);
|
||||
await page.getByRole("button", { name: "Next" }).click();
|
||||
await page.waitForTimeout(300);
|
||||
await page.getByRole("button", { name: "Freeze my deck" }).click();
|
||||
await page.waitForTimeout(2500);
|
||||
if (!page.url().includes("/deck/")) {
|
||||
const text = await page.evaluate(() => document.body.innerText.slice(0, 600).replace(/\n+/g, " | "));
|
||||
console.log("DEBUG generation page:", text);
|
||||
console.log("DEBUG last provider request:", JSON.stringify(lastProviderRequest).slice(0, 300));
|
||||
}
|
||||
check("generation: the deck lands on the study page", page.url().includes("/deck/"), page.url());
|
||||
check("generation: the deck came from the provider", await page.getByText("Provider deck").first().isVisible());
|
||||
check(
|
||||
"generation: the request carried the provider model",
|
||||
lastProviderRequest?.body?.model === "free-model-a" && lastProviderRequest?.body?.stream === false,
|
||||
JSON.stringify({ model: lastProviderRequest?.body?.model, stream: lastProviderRequest?.body?.stream }),
|
||||
);
|
||||
await context.close();
|
||||
}
|
||||
|
||||
/* 4. Settings: saving a key lists the models --------------------------- */
|
||||
{
|
||||
const { context, page } = await makePage(browser, { width: 390, height: 844 });
|
||||
await seed(page, { decks: [], extra: { "freezmo.providers.v1": JSON.stringify({ ...providerSeed, keys: {}, models: {} }) } });
|
||||
await page.goto(`${APP}/`, { waitUntil: "networkidle" });
|
||||
await page.waitForTimeout(1000);
|
||||
await page.locator("nav.tabbar").getByRole("button").click();
|
||||
await page.getByRole("dialog").waitFor({ timeout: 6000 });
|
||||
await page.waitForTimeout(600);
|
||||
const row = page.locator('input[aria-label="Custom (OpenAI-compatible) API key"]');
|
||||
await row.scrollIntoViewIfNeeded();
|
||||
await row.fill(PROVIDER_KEY);
|
||||
await row.locator("xpath=..").getByRole("button", { name: "Save key", exact: true }).click();
|
||||
await page.waitForTimeout(1600);
|
||||
check("settings: saving the key lists the models", (await page.getByText("2 models").count()) >= 1);
|
||||
check(
|
||||
"settings: the key is stored in this browser only",
|
||||
(await page.evaluate(() => JSON.parse(localStorage.getItem("freezmo.providers.v1") ?? "{}").keys?.custom)) === PROVIDER_KEY,
|
||||
);
|
||||
await context.close();
|
||||
}
|
||||
|
||||
check("stub saw both endpoints", seen.models >= 1 && seen.completions >= 1, JSON.stringify(seen));
|
||||
} finally {
|
||||
app.stop();
|
||||
stub.close();
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
finish();
|
||||
|
|
@ -24,6 +24,7 @@ const SUITES = [
|
|||
"phase3-test",
|
||||
"security-test",
|
||||
"chat-test",
|
||||
"providers-test",
|
||||
"review-test",
|
||||
];
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue