Chat mode, phase 1: grounded streaming chat over your notes
Entry points (no sixth tab): Ask in the Library header (desktop button, phone icon), Ask about this deck in the deck menu, Ask a question on study/homework results (compact result travels as context), and a Chat row in the desktop sidebar. Phones get a full-screen route with back + New chat; the tab bar steps aside. - Client-side retrieval: chunk scoped decks (~800 tokens), BM25 against the question + recent turns, top chunks under ~7k tokens, each labelled with deck + section. Notes come from the deck's stored sourceText (already kept for Regenerate); decks without it fall back to generated content and the UI says so. - Stateless server: /api/chat validates against the CHAT_MODELS allowlist, trims the history to ~6k tokens, caps context at ~9k and streams token/thinking/done/error NDJSON. /api/chat/extract reduces attachments to text through the existing vision step. - UI: user bubbles + full-width assistant text, streaming with stop, auto-scroll that pauses with a jump-to-bottom button, safe markdown (React nodes, no HTML injection; KaTeX dynamically imported only when a message has math), Thinking row for reasoning, copy/regenerate/ edit-and-resend, model + context pickers (last model remembered), composer with visualViewport keyboard handling, safe-area, 16px textarea, Enter-to-send on desktop. - Usage-limit failures keep the draft and show the shared message with Settings/Ollama actions; stopping maps the reader abort to a clean stopped state; offline disables sending with a reason. - Settings: Keep notes text for chat toggle; deck menu: Delete notes text. RATE_LIMIT_CHAT + CHAT_MODELS in .env.example and README. - Tests: chat suite (63 checks, stub Ollama) wired into run-all. Full set: 12/12 suites, 487 checks. SW -> freezmo-v32. Also: tests/model-compare.mjs (diagnostic) benchmarks two models on the same notes (latency, repair rounds, activity types) so the gpt-oss vs gemma default can be decided on numbers; defaults unchanged.
This commit is contained in:
parent
6b6a6ea158
commit
756a6257a2
46 changed files with 3840 additions and 56 deletions
141
tests/model-compare.mjs
Normal file
141
tests/model-compare.mjs
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Model A/B benchmark (diagnostic, not part of `npm run test:e2e`).
|
||||
*
|
||||
* Answers "is model X a better fit than model Y for Freezmo's strict-JSON
|
||||
* pipeline?" with numbers: it runs the same notes through two throwaway
|
||||
* production instances (one per model), times them and reports how often the
|
||||
* model needed the JSON repair round.
|
||||
*
|
||||
* OLLAMA_API_KEY=... node tests/model-compare.mjs
|
||||
* OLLAMA_API_KEY=... node tests/model-compare.mjs gemma4:31b-cloud gpt-oss:120b-cloud
|
||||
*
|
||||
* The key travels only to the spawned local instances (server auth mode) and
|
||||
* is never printed, stored or committed. Uses your Ollama quota: 2 generate
|
||||
* calls + 1 grade call per model.
|
||||
*/
|
||||
import { spawnApp, waitForHttp, portInUse } from "./lib.mjs";
|
||||
|
||||
const MODELS = process.argv.slice(2);
|
||||
const [MODEL_A, MODEL_B] = MODELS.length >= 2 ? MODELS : ["gpt-oss:120b-cloud", "gemma4:31b-cloud"];
|
||||
const API_KEY = process.env.OLLAMA_API_KEY?.trim();
|
||||
const BASE_URL = process.env.OLLAMA_BASE_URL?.trim() || "https://ollama.com";
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error("Set OLLAMA_API_KEY (your Ollama key) for this command — it is never printed.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const NOTES = `Rayleigh scattering
|
||||
Light from the sun hits molecules in the atmosphere. Short wavelengths (blue and violet) scatter much more strongly than long ones (red). This is why a clear daytime sky is blue.
|
||||
Sunsets
|
||||
At sunset the light travels through much more atmosphere, so almost all the blue light is scattered away and only the red end of the spectrum reaches our eyes.
|
||||
Why not violet?
|
||||
Violet scatters even more than blue, but our eyes are far less sensitive to violet, and some of it is absorbed in the upper atmosphere.
|
||||
Interesting: the sky on the Moon is black because there is no atmosphere to scatter light.`;
|
||||
|
||||
const GRADE_INPUT = {
|
||||
question: "Why is a clear daytime sky blue?",
|
||||
referenceAnswer: "Short wavelengths scatter more strongly than long ones.",
|
||||
keyPoints: ["Rayleigh scattering", "short wavelengths scatter more", "molecules in the atmosphere"],
|
||||
userAnswer: "Because blue light scatters more than red in the atmosphere.",
|
||||
};
|
||||
|
||||
async function timed(label, run) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const value = await run();
|
||||
return { label, ms: Date.now() - started, value };
|
||||
} catch (error) {
|
||||
return { label, ms: Date.now() - started, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
async function generateOnce(port, index) {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/api/generate`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: NOTES, difficulty: "medium", count: 8 }),
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${(await response.text()).slice(0, 120)}`);
|
||||
const events = (await response.text())
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
const deckEvent = events.find((event) => event.type === "deck");
|
||||
const errorEvent = events.find((event) => event.type === "error");
|
||||
if (errorEvent) throw new Error(errorEvent.error);
|
||||
if (!deckEvent) throw new Error("no deck event");
|
||||
return {
|
||||
run: index,
|
||||
activities: deckEvent.deck.activities.length,
|
||||
repaired: Boolean(deckEvent.meta?.repaired),
|
||||
requested: deckEvent.meta?.requested ?? null,
|
||||
types: deckEvent.deck.activities.map((activity) => activity.type),
|
||||
sample: deckEvent.deck.activities[0]?.question ?? deckEvent.deck.activities[0]?.front ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
async function gradeOnce(port) {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/api/grade`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(GRADE_INPUT),
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
if (!response.ok) throw new Error(body?.error ?? `HTTP ${response.status}`);
|
||||
return { correct: body.correct, score: body.score, feedback: String(body.feedback ?? "").slice(0, 70) };
|
||||
}
|
||||
|
||||
async function bench(model, port) {
|
||||
if (await portInUse(port)) throw new Error(`port ${port} is busy`);
|
||||
const app = spawnApp(port, {
|
||||
OLLAMA_AUTH: "server",
|
||||
OLLAMA_API_KEY: API_KEY,
|
||||
OLLAMA_BASE_URL: BASE_URL,
|
||||
OLLAMA_VERIFY_KEY: "false",
|
||||
TEXT_MODEL: model,
|
||||
VISION_MODEL: model,
|
||||
});
|
||||
try {
|
||||
const up = await waitForHttp(`http://127.0.0.1:${port}/api/health`);
|
||||
if (!up) throw new Error("instance did not start");
|
||||
const rows = [];
|
||||
for (const index of [1, 2]) rows.push(await timed(`generate #${index}`, () => generateOnce(port, index)));
|
||||
rows.push(await timed("grade", () => gradeOnce(port)));
|
||||
return { model, rows };
|
||||
} finally {
|
||||
app.stop();
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Comparing ${MODEL_A} vs ${MODEL_B} against ${BASE_URL}\n`);
|
||||
const results = [];
|
||||
for (const [model, port] of [
|
||||
[MODEL_A, 3210],
|
||||
[MODEL_B, 3211],
|
||||
]) {
|
||||
console.log(`--- ${model} ---`);
|
||||
const result = await bench(model, port);
|
||||
for (const row of result.rows) {
|
||||
if (row.error) console.log(` ${row.label}: ERROR after ${row.ms} ms — ${row.error}`);
|
||||
else if (row.label.startsWith("generate")) {
|
||||
console.log(
|
||||
` ${row.label}: ${row.ms} ms | ${row.value.activities} activities | repaired: ${row.value.repaired ? "YES" : "no"} | types: ${row.value.types.join(", ")}`,
|
||||
);
|
||||
console.log(` first: ${row.value.sample.slice(0, 90)}`);
|
||||
} else {
|
||||
console.log(` ${row.label}: ${row.ms} ms | correct: ${row.value.correct} | score: ${row.value.score} | ${row.value.feedback}`);
|
||||
}
|
||||
}
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
console.log("\nSummary");
|
||||
for (const result of results) {
|
||||
const generates = result.rows.filter((row) => row.label.startsWith("generate") && !row.error);
|
||||
const repaired = generates.filter((row) => row.value.repaired).length;
|
||||
const times = generates.map((row) => row.ms);
|
||||
const average = times.length ? Math.round(times.reduce((sum, value) => sum + value, 0) / times.length) : 0;
|
||||
console.log(` ${result.model}: avg generate ${average} ms | repairs ${repaired}/${generates.length} | errors ${result.rows.filter((row) => row.error).length}`);
|
||||
}
|
||||
console.log("\nRead it as: fewer repairs + sensible activity types + acceptable latency = the better pipeline fit.");
|
||||
Loading…
Add table
Add a link
Reference in a new issue