From f202653017a7fb6ff7defb56d850a5affb08a13b Mon Sep 17 00:00:00 2001 From: Martin Dvorak Date: Mon, 12 Oct 2026 05:37:03 +0000 Subject: [PATCH] Generation guards: cards with repeated wording never ship MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A generated Czech literature deck reached a phone with three identical matching definitions — the card could not be solved. The pipeline only deduped matching terms (and ordering items), so repeated definitions and options sailed through. - parseDeck now rejects any card whose options, items, terms or definitions repeat (case-insensitive), through the same drop-or-repair path as the ordering giveaway guard: the card is dropped, or a deck left with too few cards gets one repair round with a clear reason. - The matching dedupe covers definitions too, and both it and the ordering one only trim when the card still meets the schema minimum afterwards — a shorter card would fail validation on the next read and take the whole deck down with it (latent data-loss bug). - The generation prompt states the uniqueness rule. - phase-1 suite: a stub Ollama returns a deck with a repeated-definition matching card (the reported shape), a repeated-option card and a salvageable one; asserts the first two are dropped, the third is trimmed to three pairs and the good cards ship (43 checks). - README: decision 51. Server-side only (lib/ai.ts, lib/prompts.ts) — no client bundle change, no SW bump. --- README.md | 17 ++++++- lib/ai.ts | 77 ++++++++++++++++++++++--------- lib/prompts.ts | 3 ++ tests/phase1-test.mjs | 102 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e52d2e4..a0ed736 100644 --- a/README.md +++ b/README.md @@ -1346,8 +1346,21 @@ Things I chose while building, with the reasoning: (`classifyOllamaFailure`) and mapped to `usage_limit` / `throttled` kinds — the client renders the same localized message in every model flow, with a jump to the key field and a link to Ollama, and keeps the user's input so Retry works right after swapping the key. Verified by the - phase-1 suite (38 checks, including the real route mapping against a stub Ollama returning a - monthly-limit body, and a two-device sync round-trip). + phase-1 suite (43 checks, including the real route mapping against a stub Ollama returning a + monthly-limit body, a two-device sync round-trip, and the generation guards below). + +51. **Cards with repeated wording never ship.** The model sometimes pads a matching card with one + definition copied across several terms (a Czech literature deck arrived on a phone with three + identical definitions, so the card could not be solved by knowing the material), or repeats an + option. The generation pipeline now rejects any card whose options, items, terms or definitions + repeat (case-insensitive) — the same drop-or-repair path the ordering giveaway guard already + used, so a deck with too few cards left over gets one repair round instead. The matching and + ordering dedupe only trims a card when it still meets the schema's minimum afterwards: a + shorter card would fail validation on the next read and take the whole deck down with it, so + below the minimum the original is kept and the card is rejected instead. The generation prompt + states the uniqueness rule too. Verified by the phase-1 suite's stub-Ollama generation: the + repeated-definition card is dropped, a salvageable card is trimmed to three pairs, the + repeated-option card is dropped and the good cards still ship. 41. **The learning log is an event log, so it lives in IndexedDB.** Decks and homework stay in `localStorage` (small, always loaded), but answers + SRS schedules go to a versioned diff --git a/lib/ai.ts b/lib/ai.ts index 91d0817..1bf6087 100644 --- a/lib/ai.ts +++ b/lib/ai.ts @@ -310,6 +310,14 @@ function validateDeck(parsed: unknown, minimum: number, requested: number) { ); return; } + // Repeated wording makes a card impossible to answer: two identical + // options, items, terms or definitions cannot be told apart. + if (repeatedEntries(activity)) { + rejected.push( + `#${index + 1} (${activity.type}): repeated entries — every option, item, term and definition must be unique.`, + ); + return; + } activities.push(activity); } else { const type = (item as { type?: unknown } | null)?.type; @@ -581,42 +589,71 @@ export async function refineHomeworkText(input: { /* ------------------------------------------------------------------ */ /** - * Small fixes so the UI can trust the data: ordering items must be unique - * (they are used as React keys and are ambiguous otherwise), fill_blank - * sentences must actually contain a blank, and word banks must contain - * the answer. + * Small fixes so the UI can trust the data: ordering items and matching pairs + * must be unique (they are used as React keys, and a repeat makes the card + * ambiguous), fill_blank sentences must actually contain a blank, and word + * banks must contain the answer. + * + * A salvage only happens when the card still meets the schema's minimum after + * deduping — a shorter list would fail validation on the next read and take + * the whole deck down with it. */ function sanitiseActivity(activity: GeneratedActivity): GeneratedActivity { switch (activity.type) { case "ordering": { - const seen = new Set(); - const items = activity.items.filter((item) => { - const key = item.trim().toLowerCase(); - if (seen.has(key)) return false; - seen.add(key); - return true; - }); - return items.length === activity.items.length ? activity : { ...activity, items }; + const items = dedupeBy(activity.items, (item) => item); + return items.length === activity.items.length || items.length < 3 ? activity : { ...activity, items }; } case "fill_blank": { if (/_{2,}/.test(activity.sentence)) return activity; return { ...activity, sentence: `${activity.sentence.trim()} ____` }; } case "matching": { - const seen = new Set(); - const pairs = activity.pairs.filter((pair) => { - const key = pair.term.trim().toLowerCase(); - if (seen.has(key)) return false; - seen.add(key); - return true; - }); - return pairs.length === activity.pairs.length ? activity : { ...activity, pairs }; + // Terms first, then definitions: one definition per term, or the learner + // cannot tell the pairs apart. + const pairs = dedupeBy(dedupeBy(activity.pairs, (pair) => pair.term), (pair) => pair.definition); + return pairs.length === activity.pairs.length || pairs.length < 3 ? activity : { ...activity, pairs }; } default: return activity; } } +/** Case-insensitive, trimmed dedupe that keeps the first occurrence. */ +function dedupeBy(items: T[], key: (item: T) => string): T[] { + const seen = new Set(); + return items.filter((item) => { + const value = key(item).trim().toLowerCase(); + if (seen.has(value)) return false; + seen.add(value); + return true; + }); +} + +/** True when a list repeats an entry (case-insensitive, trimmed). */ +function hasDuplicates(values: string[]): boolean { + return dedupeBy(values, (value) => value).length !== values.length; +} + +/** The values a card needs to keep distinct to stay answerable. */ +function repeatedEntries(activity: GeneratedActivity): boolean { + switch (activity.type) { + case "multiple_choice": + return hasDuplicates(activity.options); + case "fill_blank": + return activity.options ? hasDuplicates(activity.options) : false; + case "ordering": + return hasDuplicates(activity.items); + case "matching": + return ( + hasDuplicates(activity.pairs.map((pair) => pair.term)) || + hasDuplicates(activity.pairs.map((pair) => pair.definition)) + ); + default: + return false; + } +} + /** Compact, human-readable zod error for the repair prompt. */ function summariseZodError(error: { issues: Array<{ path: PropertyKey[]; message: string }> }): string { return error.issues diff --git a/lib/prompts.ts b/lib/prompts.ts index d9dda68..992f0b4 100644 --- a/lib/prompts.ts +++ b/lib/prompts.ts @@ -82,6 +82,7 @@ export const ACTIVITY_JSON_SPEC = `Each activity is one JSON object with a "type "options" is optional but preferred: 3-6 plausible choices including the answer. 5. matching - 3 to 6 term/definition pairs + Every term and definition must be distinct: a definition describes exactly one term, so never reuse a definition for two pairs. { "type": "matching", "topic": "short label", "instruction": "Match each term to its definition.", "pairs": [{ "term": "Osmosis", "definition": "diffusion of water across a membrane" }], "explanation": "..." } 6. ordering - 3 to 7 items given in the CORRECT order (the app shuffles them) @@ -105,6 +106,8 @@ Hard rules: - Vary the activity types. Use a healthy mix of all seven types below, and never put more than two activities of the same type in a row. - Cover the whole notes, not just the first paragraph. Spread activities across the different topics, definitions, formulas and facts. - Make the wrong options plausible (common mix-ups from the same notes), never silly filler. +- Every option, item, term and definition on a card must be unique: never repeat the same wording + twice on one card — a repeated option or definition makes the card impossible to solve. - Ordering items must never reveal their position: no dates, years, times, step numbers or ordinals ("first", "next", "finally", "step 2"). Describe the event or stage itself. - Keep questions short enough to read on a phone screen. No walls of text. diff --git a/tests/phase1-test.mjs b/tests/phase1-test.mjs index d893f7d..2c77fbf 100644 --- a/tests/phase1-test.mjs +++ b/tests/phase1-test.mjs @@ -374,5 +374,107 @@ async function openSettings(page) { } } +/* 9. generation guards: cards with repeated wording never ship ------- */ +{ + check("guard: stub port free", !(await portInUse(11998))); + // Mirrors the deck that reached a phone: three terms, one shared definition. + const repeated = "Jedna z hlavních postav v knížkách lidového čtení"; + const deckReply = { + title: "Lidové čtení", + subject: "Literatura", + language: "Czech", + summary: "Deck for the repeated-wording guards.", + activities: [ + { type: "flashcard", topic: "Postavy", front: "Kdo je Bruncvík?", back: "Rytíř se lvem ve znaku.", explanation: "x" }, + { type: "flashcard", topic: "Postavy", front: "Kdo je Štilfrid?", back: "Bruncvíkův otec.", explanation: "x" }, + { type: "multiple_choice", topic: "Postavy", question: "Kdo je hlavní postava?", options: ["Bruncvík", "Paleček", "Štilfrid", "Karel"], correctIndex: 0, explanation: "x" }, + { type: "multiple_choice", topic: "Postavy", question: "Kdo patří k lidovému čtení?", options: ["Paleček", "Paleček", "Bruncvík", "Štilfrid"], correctIndex: 2, explanation: "x" }, + { + type: "matching", + topic: "Postavy", + instruction: "Přiřaď postavy k jejich popisu.", + pairs: [ + { term: "Bruncvík", definition: "Rytíř se lvem ve znaku" }, + { term: "Štilfrid", definition: "Bruncvíkův otec" }, + { term: "Paleček", definition: "Chytrý sluha" }, + ], + explanation: "x", + }, + { + type: "matching", + topic: "Postavy", + instruction: "Přiřaď postavy k jejich zařazení.", + pairs: [ + { term: "Bruncvík", definition: repeated }, + { term: "Štilfrid", definition: repeated }, + { term: "Paleček", definition: repeated }, + ], + explanation: "x", + }, + { + type: "matching", + topic: "Postavy", + instruction: "Přiřaď postavy k vlastnostem.", + pairs: [ + { term: "A", definition: "statečný" }, + { term: "B", definition: "statečný" }, + { term: "C", definition: "moudrý" }, + { term: "D", definition: "věrný" }, + ], + explanation: "x", + }, + ], + }; + const stub = http.createServer((request, response) => { + if (request.url?.startsWith("/api/tags")) { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ models: [{ name: "stub", model: "stub" }] })); + return; + } + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ message: { content: JSON.stringify(deckReply) } })); + }); + await new Promise((resolve) => stub.listen(11998, "127.0.0.1", resolve)); + + const app = spawnApp(3210, { + OLLAMA_BASE_URL: "http://127.0.0.1:11998", + OLLAMA_AUTH: "local", + OLLAMA_VERIFY_KEY: "false", + }); + + try { + const up = await waitForHttp("http://127.0.0.1:3210/api/health"); + check("guard: stub app is up", up); + if (up) { + const response = await fetch("http://127.0.0.1:3210/api/generate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: "Lidové čtení: Bruncvík, Štilfrid a Paleček.", difficulty: "easy", count: 6 }), + }); + const events = (await response.text()) + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const deck = events.find((event) => event.type === "deck")?.deck; + const activities = deck?.activities ?? []; + const matchings = activities.filter((activity) => activity.type === "matching"); + const choices = activities.filter((activity) => activity.type === "multiple_choice"); + check("guard: the good cards still ship", activities.length === 5, `got ${activities.length}: ${activities.map((a) => a.type).join(",")}`); + check("guard: repeated-option card dropped", choices.length === 1 && new Set(choices[0].options).size === 4); + check("guard: repeated-definition card dropped", matchings.length === 2, `matchings: ${matchings.length}`); + check( + "guard: every matching card keeps its definitions distinct", + matchings.every((activity) => new Set(activity.pairs.map((pair) => pair.definition)).size === activity.pairs.length), + ); + const salvaged = matchings.find((activity) => activity.pairs.some((pair) => pair.term === "A")); + check("guard: a salvageable card is trimmed, not dropped", salvaged?.pairs.length === 3, `pairs: ${salvaged?.pairs?.length}`); + } + } finally { + app.stop(); + stub.close(); + } +} + await browser.close(); finish();