import type { ServerConfig } from "./config"; import { translate, type Locale } from "./i18n-dict"; import { acquireSlot, clientKey, rateLimit, rateLimitResponse, releaseSlot } from "./rate-limit"; /** * One gate in front of every route that spends an Ollama call: * * - a **global budget** per window (all clients together) so no flood can * burn through the account — the thing that gets a key throttled or banned; * - a **per-IP concurrency cap** and a **global one**, so a burst of parallel * requests can't pile up model calls (or sockets and memory). * * The slot is released once, when the response is finished — including a * client that disconnects mid-stream, and including routes that throw. */ export type ModelGate = { release: () => void }; export function acquireModelGate( request: Request, config: ServerConfig, locale: Locale, ): { gate: ModelGate } | { response: Response } { const perIpKey = `model-ip:${clientKey(request)}`; const totalKey = "model-total"; if (!acquireSlot(totalKey, config.rateLimits.concurrentTotal)) { return { response: busyResponse(locale) }; } if (!acquireSlot(perIpKey, config.rateLimits.concurrent)) { releaseSlot(totalKey); return { response: busyResponse(locale) }; } const budget = rateLimit("model-budget", config.rateLimits.modelTotal, config.rateLimits.windowMs); if (!budget.allowed) { releaseSlot(perIpKey); releaseSlot(totalKey); return { response: rateLimitResponse(budget, locale) }; } let released = false; return { gate: { release: () => { if (released) return; released = true; releaseSlot(perIpKey); releaseSlot(totalKey); }, }, }; } /** 429 for "something is already running" — not a rate-limit punishment. */ function busyResponse(locale: Locale): Response { return Response.json( { error: translate(locale, "server.busy"), kind: "busy", retryAfter: 5 }, { status: 429, headers: { "Retry-After": "5", "Cache-Control": "no-store" } }, ); }