diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 0000000..16c41f4 --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,53 @@ +# Agent Notes + +Operational knowledge recorded by agents working on this repo. Update as things change. + +## LLM API (llm.elimelt.com → Ollama) + +### Endpoints +- Public: `https://llm.elimelt.com` — both Ollama-native (`/api/chat`) and OpenAI-compatible (`/v1/chat/completions`, `/v1/models`) APIs. +- Internal (from host): `http://172.27.0.11:11434`. +- Model management endpoints (`/api/pull`, `/api/delete`, `/api/copy`, `/api/push`, `/api/create`) are blocked at the edge by Caddy. Manage models via `docker exec ollama ollama pull|rm `. + +### CORS / rate limiting (Caddyfile) +- Allowed origins: `https://*.elimelt.com` and `http://localhost:5173` (exact spelling — `127.0.0.1` and other ports do NOT match). +- Requests from allowed origins are exempt from rate limiting; all other traffic is limited to 100 req/min per IP. +- Gotcha: Ollama validates the `Origin` header itself and returns 403 unless `OLLAMA_ORIGINS=*` is set. Origin policy is enforced by Caddy at the edge instead. + +### Ollama container tuning (docker-compose.yml) +- `OLLAMA_NUM_PARALLEL=4`: concurrency benchmarks (`infra/scripts/bench_ollama_concurrency.py`) showed throughput plateaus around 4 concurrent requests on the i9-13900HK (CPU-bound). +- `OLLAMA_MAX_QUEUE=20`: rejects excess requests instead of unbounded queueing. +- `OLLAMA_MAX_LOADED_MODELS=2`: requesting a non-resident model triggers a cold load (up to ~13s for 18GB models) and may evict a warm one. +- Memory limit 50G: 26B-class models OOM-killed the container at the old 6G limit. + +## Model lineup (as of 2026-08-02) + +Chosen to be Pareto-optimal per use case. `gemma3:27b` and `llama3.2:3b` were +deleted as dominated (see benchmarks below). + +| Model | Size | Role | Warm 1st content | Decode | +|---|---|---|---|---| +| `gemma2:2b` | 1.6GB | speed / TTS-rewrite | 206ms | 27.4 t/s | +| `gemma4:e4b` | 9.6GB | mid quality, fast cold load (4.4s) | 439ms (`think: false`) | 15.4 t/s | +| `gemma4:26b` | 18GB | best quality-per-second (MoE, 3.8B active) | 448ms (`think: false`) | 16.7 t/s | +| `gpt-oss:20b` | 13.8GB | reasoning / tool calling (MoE, 3.6B active) | 1.5s (`think: "low"`) | 12.8 t/s | +| `qwen2.5-coder:7b` | 4.7GB | code / FIM | 200ms | 11.2 t/s | + +### Thinking models — critical gotchas +- `gemma4:*` think **by default**. Pass `"think": false` on `/api/chat` or first content is delayed ~10s (e4b) to ~37s (26b). With a small `num_predict`, the entire budget can be consumed by thinking and no content ever arrives. +- `gpt-oss` cannot disable thinking, only effort: `"think": "low" | "medium" | "high"`. Low ≈ 1.5s to first content; high ≈ 28s. +- The OpenAI `/v1` endpoint has **no way to pass `think`** — gemma4 models think by default there. If a `/v1` client needs gemma4 without thinking, create a Modelfile variant (not yet done; see issues). +- When benchmarking, distinguish *first token* (may be thinking) from *first content*. `infra/scripts/bench_ollama.py` handles this. + +### Benchmark/eval scripts (`infra/scripts/`) +- `bench_ollama.py [runs]` — TTFT (first token vs first content), decode/prompt t/s, per think-mode. +- `bench_ollama_concurrency.py` — throughput vs concurrent requests. +- `bench_ollama_prefill.py` — prompt-processing latency vs input length. +- `eval_tts_rewrite.py ` — quality eval for the notes→TTS rewrite task (math excerpts from notes.elimelt.com, checks for leftover LaTeX/symbols, preambles, dropped terms). +- `eval_tts_prompts.py` — prompt-variant ablations for the same task. + +### Misc findings +- MoE models (`gemma4:26b`, `gpt-oss:20b`) decode at small-model speeds on CPU despite large total size — only active params matter per token. +- Small dense models: optimal `num_thread` is 10–14 on the i9-13900HK. +- `gemma2:2b` passed all automated TTS-rewrite checks but made a semantic error (conflated largest/smallest singular value) that `gemma4:e4b`/`26b` got right. Automated checks don't catch meaning errors. +- `llama3.2:3b` left raw LaTeX in TTS output; `gemma4` spells code identifiers letter-by-letter ("n p dot ..."). diff --git a/infra/Caddyfile b/infra/Caddyfile index b88e254..6aed084 100644 --- a/infra/Caddyfile +++ b/infra/Caddyfile @@ -25,7 +25,7 @@ rate_limit { zone llm_zone { key {remote_host} - events 20 + events 100 window 1m } } @@ -103,29 +103,53 @@ # llm.elimelt.com → Ollama (OpenAI-compatible LLM API) @llm host llm.elimelt.com handle @llm { - # Rate limit: 20 requests per minute per IP - import rate_limit_llm + # Set CORS headers for allowed origins (non-terminal, always runs) + @cors_allowed { + expression {http.request.header.Origin}.matches("^https://[a-zA-Z0-9-]+\\.elimelt\\.com$") || {http.request.header.Origin} == "http://localhost:5173" + } + header @cors_allowed Access-Control-Allow-Origin "{http.request.header.Origin}" + + # Handle CORS preflight + @cors_preflight method OPTIONS + handle @cors_preflight { + @cors_allowed_preflight { + expression {http.request.header.Origin}.matches("^https://[a-zA-Z0-9-]+\\.elimelt\\.com$") || {http.request.header.Origin} == "http://localhost:5173" + } + header @cors_allowed_preflight Access-Control-Allow-Methods "GET, POST, OPTIONS" + header @cors_allowed_preflight Access-Control-Allow-Headers "Content-Type, Authorization" + header @cors_allowed_preflight Access-Control-Max-Age "86400" + respond @cors_allowed_preflight 204 + respond "Forbidden" 403 + } - # CORS for *.elimelt.com and localhost - import cors_elimelt + # Block model management endpoints + @blocked path /api/pull /api/delete /api/copy /api/push /api/create + handle @blocked { + respond "Forbidden" 403 + } - # Block model management - pull/delete/copy/push - @blocked_pull path /api/pull - @blocked_delete path /api/delete - @blocked_copy path /api/copy - @blocked_push path /api/push - @blocked_create path /api/create - respond @blocked_pull "Forbidden" 403 - respond @blocked_delete "Forbidden" 403 - respond @blocked_copy "Forbidden" 403 - respond @blocked_push "Forbidden" 403 - respond @blocked_create "Forbidden" 403 - - reverse_proxy ollama:11434 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Access-Control-Allow-Credentials + # Internal origins: no rate limit, proxy to Ollama + @internal_origin { + expression {http.request.header.Origin}.matches("^https://[a-zA-Z0-9-]+\\.elimelt\\.com$") || {http.request.header.Origin} == "http://localhost:5173" + } + handle @internal_origin { + reverse_proxy ollama:11434 { + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Allow-Credentials + } + } + + # External/no origin: rate limited, proxy to Ollama + handle { + import rate_limit_llm + reverse_proxy ollama:11434 { + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Allow-Credentials + } } } diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 75bd1ff..08550e3 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -262,6 +262,7 @@ services: # Ollama - Local LLM inference with OpenAI-compatible API # Models are stored in ollama_data volume, download via: docker exec ollama ollama pull # API: http://ollama:11434/v1/chat/completions (OpenAI-compatible) + # Backpressure: OLLAMA_NUM_PARALLEL controls concurrent inference; excess requests queue. ollama: image: ollama/ollama:latest container_name: ollama @@ -270,10 +271,23 @@ services: - ollama_data:/root/.ollama environment: - OLLAMA_HOST=0.0.0.0 - - OLLAMA_NUM_PARALLEL=${OLLAMA_NUM_PARALLEL:-2} + # Accept any Origin header; CORS/origin policy is enforced by Caddy at the edge + - OLLAMA_ORIGINS=* + # Concurrent requests per model (queues excess). Keep low for backpressure. + - OLLAMA_NUM_PARALLEL=${OLLAMA_NUM_PARALLEL:-4} + # Max models in memory simultaneously - OLLAMA_MAX_LOADED_MODELS=${OLLAMA_MAX_LOADED_MODELS:-2} + # Queue timeout: reject if queued too long (prevents infinite queue buildup) + - OLLAMA_MAX_QUEUE=${OLLAMA_MAX_QUEUE:-20} networks: - dev_network + # Memory limits to prevent OOM + deploy: + resources: + limits: + memory: 50G + reservations: + memory: 2G healthcheck: test: ["CMD", "curl", "-f", "http://localhost:11434/api/tags"] interval: 30s diff --git a/infra/scripts/bench_ollama.py b/infra/scripts/bench_ollama.py new file mode 100644 index 0000000..c5f29fd --- /dev/null +++ b/infra/scripts/bench_ollama.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Benchmark TTFT and tok/sec for all models on an Ollama server. + +Usage: python3 bench_ollama.py [host] [runs] + host: Ollama base URL (default http://localhost:11434) + runs: measured runs per model after 1 warm-up (default 3) + +Stdlib only. Streams /api/chat to measure wall-clock TTFT; decode tok/s +comes from Ollama's own eval_count/eval_duration stats. +""" +import json +import statistics +import sys +import time +import urllib.request + +HOST = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11434" +RUNS = int(sys.argv[2]) if len(sys.argv) > 2 else 3 +PROMPT = "Explain what a hash table is in about one paragraph." +NUM_PREDICT = 100 + + +def list_models(): + with urllib.request.urlopen(f"{HOST}/api/tags", timeout=30) as r: + return [m["name"] for m in json.load(r)["models"]] + + +def bench_once(model, think=None): + """One streamed request. + + Returns dict with: + first_tok_s: time to first streamed token (thinking or content) + first_content_s: time to first content token (None if none arrived) + think_tokens: approx thinking tokens emitted (chunk count) + decode_tok_s, prompt_tok_s, eval_count: from Ollama's stats + """ + # Thinking modes need a larger budget so content tokens actually arrive + budget = NUM_PREDICT if not think else 600 + payload = { + "model": model, + "messages": [{"role": "user", "content": PROMPT}], + "stream": True, + "options": {"num_predict": budget, "temperature": 0}, + } + if think is not None: + payload["think"] = think + req = urllib.request.Request( + f"{HOST}/api/chat", data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + ) + start = time.perf_counter() + first_tok = first_content = None + think_tokens = 0 + final = None + with urllib.request.urlopen(req, timeout=1800) as r: + for line in r: + chunk = json.loads(line) + msg = chunk.get("message", {}) + if msg.get("thinking"): + think_tokens += 1 + if first_tok is None: + first_tok = time.perf_counter() - start + if msg.get("content"): + now = time.perf_counter() - start + if first_tok is None: + first_tok = now + if first_content is None: + first_content = now + if chunk.get("done"): + final = chunk + return { + "first_tok_s": first_tok, + "first_content_s": first_content, + "think_tokens": think_tokens, + "decode_tok_s": final["eval_count"] / (final["eval_duration"] / 1e9), + "prompt_tok_s": final["prompt_eval_count"] + / max(final["prompt_eval_duration"] / 1e9, 1e-9), + "eval_count": final["eval_count"], + } + + +def fmt_ms(seconds): + return f"{seconds*1000:.0f}ms" if seconds is not None else "n/a" + + +def bench_model(model, think=None): + label = f"{model}" + (f" (think={think})" if think is not None else "") + print(f"\n=== {label} ===") + # Warm-up: loads model into memory (cold first-token reported separately) + try: + cold = bench_once(model, think) + except Exception as e: + print(f" FAILED to load/run: {e}") + return None + print(f" cold first token (incl. model load): {fmt_ms(cold['first_tok_s'])}") + + runs = [] + for i in range(RUNS): + r = bench_once(model, think) + runs.append(r) + extra = (f" thinking={r['think_tokens']} tok" if r["think_tokens"] else "") + print(f" run {i+1}: first_tok={fmt_ms(r['first_tok_s'])} " + f"first_content={fmt_ms(r['first_content_s'])} " + f"decode={r['decode_tok_s']:.1f} tok/s " + f"prompt={r['prompt_tok_s']:.1f} tok/s " + f"({r['eval_count']} tokens){extra}") + + def med(key): + vals = [r[key] for r in runs if r[key] is not None] + return statistics.median(vals) if vals else None + + return { + "label": label, + "cold_first_tok_s": cold["first_tok_s"], + "first_tok_ms": med("first_tok_s") * 1000 if med("first_tok_s") else None, + "first_content_ms": (med("first_content_s") * 1000 + if med("first_content_s") else None), + "decode_tok_s": med("decode_tok_s"), + "prompt_tok_s": med("prompt_tok_s"), + } + + +# Thinking-capable models get benchmarked in both modes; gpt-oss cannot +# fully disable thinking, only lower effort. +THINK_MODES = { + "gemma4": [False, True], + "gpt-oss": ["low", "high"], +} + + +def think_modes_for(model): + for prefix, modes in THINK_MODES.items(): + if model.startswith(prefix): + return modes + return [None] + + +def main(): + models = list_models() + print(f"Ollama @ {HOST} | models: {', '.join(models)} | {RUNS} runs each") + results = [r for m in models for t in think_modes_for(m) + if (r := bench_model(m, t))] + + def cell(val, fmt): + return format(val, fmt) if val is not None else " n/a" + + print(f"\n{'Model':<28} {'Cold 1st tok':>12} {'1st tok':>9} " + f"{'1st content':>12} {'Decode':>10} {'Prompt':>10}") + print("-" * 86) + for r in results: + print(f"{r['label']:<28} {cell(r['cold_first_tok_s'], '>11.2f')}s " + f"{cell(r['first_tok_ms'], '>7.0f')}ms " + f"{cell(r['first_content_ms'], '>10.0f')}ms " + f"{cell(r['decode_tok_s'], '>6.1f')} t/s " + f"{cell(r['prompt_tok_s'], '>6.1f')} t/s") + + +if __name__ == "__main__": + main() diff --git a/infra/scripts/bench_ollama_concurrency.py b/infra/scripts/bench_ollama_concurrency.py new file mode 100644 index 0000000..2c8565a --- /dev/null +++ b/infra/scripts/bench_ollama_concurrency.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Concurrency sweep benchmark for Ollama. + +Fires N simultaneous streaming /api/chat requests and measures aggregate +throughput, per-request TTFT (includes queue wait), and errors. + +Usage: python3 bench_ollama_concurrency.py [host] [model] [levels] + host: Ollama base URL (default http://localhost:11434) + model: model name (default llama3.2:3b) + levels: comma-separated concurrency levels (default 1,2,4,8) +""" +import json +import statistics +import sys +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +HOST = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11434" +MODEL = sys.argv[2] if len(sys.argv) > 2 else "llama3.2:3b" +LEVELS = [int(x) for x in (sys.argv[3] if len(sys.argv) > 3 else "1,2,4,8").split(",")] +PROMPT = "Explain what a hash table is in about one paragraph." +NUM_PREDICT = 100 + + +def one_request(i): + body = json.dumps({ + "model": MODEL, + "messages": [{"role": "user", "content": f"(req {i}) {PROMPT}"}], + "stream": True, + "options": {"num_predict": NUM_PREDICT, "temperature": 0}, + }).encode() + req = urllib.request.Request( + f"{HOST}/api/chat", data=body, headers={"Content-Type": "application/json"} + ) + start = time.perf_counter() + ttft = None + tokens = 0 + try: + with urllib.request.urlopen(req, timeout=1800) as r: + for line in r: + chunk = json.loads(line) + if ttft is None and chunk.get("message", {}).get("content"): + ttft = time.perf_counter() - start + if chunk.get("done"): + tokens = chunk["eval_count"] + return {"ok": True, "ttft": ttft, "tokens": tokens, + "total": time.perf_counter() - start} + except Exception as e: + return {"ok": False, "error": str(e), "total": time.perf_counter() - start} + + +def run_level(n): + wall_start = time.perf_counter() + with ThreadPoolExecutor(max_workers=n) as pool: + results = list(pool.map(one_request, range(n))) + wall = time.perf_counter() - wall_start + + oks = [r for r in results if r["ok"]] + errs = [r for r in results if not r["ok"]] + total_tokens = sum(r["tokens"] for r in oks) + agg = total_tokens / wall if wall > 0 else 0 + + print(f"\n--- concurrency {n} ---") + if oks: + ttfts = sorted(r["ttft"] for r in oks) + per_req = statistics.median(r["tokens"] / (r["total"] - r["ttft"]) for r in oks) + print(f" ok={len(oks)} err={len(errs)} wall={wall:.1f}s " + f"aggregate={agg:.1f} tok/s per-req decode={per_req:.1f} tok/s") + print(f" TTFT min/med/max = {ttfts[0]*1000:.0f} / " + f"{statistics.median(ttfts)*1000:.0f} / {ttfts[-1]*1000:.0f} ms") + else: + print(f" ALL {len(errs)} FAILED wall={wall:.1f}s") + for e in {r["error"] for r in errs}: + cnt = sum(1 for r in errs if r["error"] == e) + print(f" error x{cnt}: {e}") + return {"n": n, "ok": len(oks), "err": len(errs), "wall": wall, "agg": agg} + + +def main(): + print(f"Ollama @ {HOST} | model={MODEL} | levels={LEVELS}") + one_request(0) # warm-up / model load + rows = [run_level(n) for n in LEVELS] + print(f"\n{'Conc':>5} {'OK':>4} {'Err':>4} {'Wall':>8} {'Aggregate':>12}") + print("-" * 38) + for r in rows: + print(f"{r['n']:>5} {r['ok']:>4} {r['err']:>4} {r['wall']:>7.1f}s " + f"{r['agg']:>8.1f} t/s") + + +if __name__ == "__main__": + main() diff --git a/infra/scripts/bench_ollama_prefill.py b/infra/scripts/bench_ollama_prefill.py new file mode 100644 index 0000000..404c607 --- /dev/null +++ b/infra/scripts/bench_ollama_prefill.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Prefill latency vs prompt size benchmark for Ollama. + +Sends prompts of increasing size and measures prompt processing (prefill) +latency and throughput from Ollama's prompt_eval_* stats. Each prompt gets +a unique random prefix so Ollama's prefix cache can't skip the prefill. + +Usage: python3 bench_ollama_prefill.py [host] [model] [sizes] [num_thread] + host: Ollama base URL (default http://localhost:11434) + model: model name (default llama3.2:3b) + sizes: comma-separated approx token counts (default 128,256,512,1024,2048,4096) + num_thread: optional thread override (default: model default) +""" +import json +import random +import string +import sys +import urllib.request + +HOST = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11434" +MODEL = sys.argv[2] if len(sys.argv) > 2 else "llama3.2:3b" +SIZES = [int(x) for x in (sys.argv[3] if len(sys.argv) > 3 + else "128,256,512,1024,2048,4096").split(",")] +NUM_THREAD = int(sys.argv[4]) if len(sys.argv) > 4 else None + +FILLER = ("The quick brown fox jumps over the lazy dog while considering " + "the computational complexity of various sorting algorithms. ") + + +def make_prompt(approx_tokens): + """Build a prompt of roughly approx_tokens tokens (~1.3 tokens/word).""" + prefix = "".join(random.choices(string.ascii_lowercase, k=12)) + words_needed = int(approx_tokens / 1.3) + filler_words = FILLER.split() + body = " ".join(filler_words[i % len(filler_words)] + for i in range(words_needed)) + return (f"[session {prefix}] Summarize the following text in one " + f"sentence:\n\n{body}") + + +def bench(model, approx_tokens): + options = {"num_predict": 1, "temperature": 0} + if NUM_THREAD: + options["num_thread"] = NUM_THREAD + body = json.dumps({ + "model": model, + "messages": [{"role": "user", "content": make_prompt(approx_tokens)}], + "stream": False, + "options": options, + }).encode() + req = urllib.request.Request( + f"{HOST}/api/chat", data=body, headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(req, timeout=3600) as r: + d = json.load(r) + n = d["prompt_eval_count"] + secs = d["prompt_eval_duration"] / 1e9 + return n, secs + + +def main(): + thread_note = f" num_thread={NUM_THREAD}" if NUM_THREAD else "" + print(f"Ollama @ {HOST} | model={MODEL}{thread_note}") + bench(MODEL, 32) # warm-up / model load + print(f"{'Target':>8} {'Actual tok':>11} {'Prefill':>10} {'tok/s':>9}") + print("-" * 42) + for size in SIZES: + n, secs = bench(MODEL, size) + print(f"{size:>8} {n:>11} {secs:>9.2f}s {n/secs:>9.1f}") + + +if __name__ == "__main__": + main() diff --git a/infra/scripts/eval_tts_prompts.py b/infra/scripts/eval_tts_prompts.py new file mode 100644 index 0000000..8e11d6c --- /dev/null +++ b/infra/scripts/eval_tts_prompts.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Prompt-variant eval for gemma2:2b on the notes-TTS rewrite task. + +Compares system prompt variants on math-heavy excerpts, with checks for +leftover notation, preambles, dropped/garbled content, and known semantic +traps (e.g. conflating sigma_1 with sigma_r). + +Usage: python3 eval_tts_prompts.py [host] [model] [variant_names_csv] +""" +import json +import re +import sys +import time +import urllib.request + +HOST = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11434" +MODEL = sys.argv[2] if len(sys.argv) > 2 else "gemma2:2b" + +BASE = ("You rewrite excerpts from technical notes so they can be read aloud " + "naturally by a text-to-speech engine. Expand math, symbols, code " + "identifiers, and abbreviations into spoken words (for example " + '"O(n log n)" becomes "order n log n", "x_i" becomes "x sub i"). ' + "LaTeX between \\( and \\) delimiters is inline math: speak it as a " + "person would read the formula aloud and drop the delimiters. Smooth " + "awkward notation into plain sentences but keep the meaning and " + "technical content identical. Do not add, explain, or summarize. " + "Output only the rewritten text with no preamble.") + +RULES = ( + "You convert technical notes into text that a text-to-speech engine reads aloud.\n" + "Rules:\n" + "1. Go sentence by sentence, in order. Produce exactly one spoken sentence per " + "input sentence. Never merge, split, reorder, or drop sentences.\n" + "2. Replace every math symbol with words: \\(x_i\\) -> 'x sub i', \\(A^T\\) -> " + "'A transpose', \\(A^{-1}\\) -> 'A inverse', \\(A^+\\) -> 'A plus', " + "\\(\\sigma_1\\) -> 'sigma one', \\(\\geq\\) -> 'is greater than or equal to', " + "\\(\\|A\\|_2\\) -> 'the two norm of A', \\(\\ldots\\) -> 'through', " + "O(n log n) -> 'order n log n'.\n" + "3. Code identifiers are spelled out in words: np.linalg.pinv -> " + "'numpy's pinv function'.\n" + "4. Output must contain no backslashes, underscores, carets, braces, or " + "math symbols of any kind.\n" + "5. Keep every claim exactly as written. Never swap which quantity a " + "property belongs to. If the input says 'sigma r is the smallest', the " + "output must also attach 'smallest' to sigma r.\n" + "6. Output only the rewritten text. No preamble, no quotes, no commentary." +) + +FEWSHOT_EXAMPLES = ( + "\n\nExample input: The eigenvalues satisfy \\(\\lambda_1 \\geq \\cdots \\geq " + "\\lambda_n \\geq 0\\), and \\(A^T A v_i = \\lambda_i v_i\\).\n" + "Example output: The eigenvalues satisfy lambda one is greater than or equal " + "to, down through, lambda n, which is at least zero, and A transpose A times " + "v sub i equals lambda i times v sub i.\n" + "\nExample input: Since \\(\\kappa = \\sigma_1 / \\sigma_r\\), a large ratio " + "means \\(Ax = b\\) is ill-conditioned; np.linalg.cond computes it.\n" + "Example output: Since kappa equals sigma one divided by sigma r, a large " + "ratio means the system A x equals b is ill-conditioned; numpy's cond " + "function computes it." +) + +BASE_PLUS = BASE + ( + "\n\nAdditional instructions:\n" + "- \\(\\|A\\|_2\\) is 'the two norm of A' (never 'norm of A squared').\n" + "- \\(\\ldots\\) or \\(\\cdots\\) inside a list or chain reads as 'down " + "through'; never output literal dots.\n" + "- Keep every property attached to the same quantity as the input: if the " + "input says sigma r is the smallest, do not attach 'smallest' to sigma one.\n" + "- Copy every sentence's meaning exactly; expand notation only, never " + "rephrase claims.\n" + "- The output must contain no backslashes, underscores, carets, braces, " + "digits attached to letters (write 'v one' not 'v1'), or literal '...'." +) + +VARIANTS = { + "baseline": BASE, + "base+direct": BASE_PLUS, + "base+fewshot": BASE + FEWSHOT_EXAMPLES, + "base+direct+fs": BASE_PLUS + FEWSHOT_EXAMPLES, + "rules": RULES, + "rules+fewshot": RULES + FEWSHOT_EXAMPLES, +} + +CASES = [ + {"text": r"Let \(A \in R^{m \times n}\). The matrix \(A^T A\) is symmetric " + r"and positive semidefinite, so it has an orthonormal eigenbasis " + r"\(v_1, \ldots, v_n\) with eigenvalues \(\lambda_1 \geq \cdots " + r"\geq \lambda_n \geq 0\). Define the singular values " + r"\(\sigma_i = \sqrt{\lambda_i}\).", + "keywords": ["symmetric", "positive semidefinite", "orthonormal", + "eigenvalues", "singular values"], + "traps": []}, + {"text": r"\(\sigma_1 = \|A\|_2\) is the largest stretch factor any unit " + r"vector experiences, and \(\sigma_r\) the smallest nonzero one; " + r"their ratio \(\sigma_1 / \sigma_r\) is the condition number.", + "keywords": ["largest", "stretch", "unit vector", "condition number"], + # semantic traps: property attached to the wrong quantity, or misread norm + "traps": [(r"sigma one (is|equals)[^.;]*smallest", "sigma1 called smallest"), + (r"(this|which|it) is (equal to )?the smallest", "sigma1 called smallest"), + (r"sigma r (is|equals)[^.;]*largest", "sigmar called largest"), + (r"norm of A,? squared", "misread ||A||_2 as squared")]}, + {"text": r"When A has full column rank, \(A^+ = (A^T A)^{-1} A^T\), the " + r"least-squares operator from the normal equations. The general " + r"statement: \(\hat{x} = A^+ b\) is always a least-squares " + r"solution of \(Ax = b\).", + "keywords": ["full column rank", "least-squares", "normal equations"], + "traps": []}, + {"text": "This is exactly what np.linalg.lstsq returns, since its " + "SVD-based driver applies a truncated pseudoinverse. " + "np.linalg.pinv applies an rcond cutoff before inverting " + "singular values.", + "keywords": ["cutoff", "singular values"], + "traps": []}, +] + +BAD_PATTERNS = [ + (r"\\\(|\\\)|\\[a-zA-Z]+\{|\\sigma|\\lambda|\\geq|\\in", "latex"), + (r"[_^]\{?[a-zA-Z0-9]", "sub/superscript"), + (r"[≥≤∈∑√σλΣ×]", "math symbol"), + (r"\^T|\^\+|\^\{-1\}|\^-1|\(A\)|\{|\}", "notation"), + (r"(?i)^\s*(here is|here's|sure|certainly|rewritten|okay|below is)", "preamble"), +] + + +def chat(system, text): + body = json.dumps({ + "model": MODEL, "stream": False, + "options": {"temperature": 0, "num_predict": 400}, + "messages": [{"role": "system", "content": system}, + {"role": "user", "content": text}], + }).encode() + req = urllib.request.Request(f"{HOST}/api/chat", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.perf_counter() + with urllib.request.urlopen(req, timeout=3600) as r: + out = json.load(r)["message"]["content"].strip() + return out, time.perf_counter() - t0 + + +def score(case, output): + issues = [name for pat, name in BAD_PATTERNS if re.search(pat, output)] + issues += [f"dropped: {k}" for k in case["keywords"] + if k.lower() not in output.lower()] + issues += [name for pat, name in case["traps"] + if re.search(pat, output, re.I)] + ratio = len(output) / len(case["text"]) + if ratio < 0.6: + issues.append(f"too short ({ratio:.1f}x)") + elif ratio > 2.5: + issues.append(f"too long ({ratio:.1f}x)") + return issues + + +def main(): + names = (sys.argv[3].split(",") if len(sys.argv) > 3 else list(VARIANTS)) + results = {} + for name in names: + print(f"\n{'='*60}\nvariant: {name}\n{'='*60}") + total, elapsed = 0, 0.0 + for i, case in enumerate(CASES): + out, dt = chat(VARIANTS[name], case["text"]) + issues = score(case, out) + total += len(issues) + elapsed += dt + print(f"[case {i+1}] {'OK ' if not issues else 'BAD'} ({dt:.1f}s) " + f"{'; '.join(issues)}") + print(f" {out[:220]}{'...' if len(out) > 220 else ''}") + results[name] = (total, elapsed) + + print(f"\n{'Variant':<15} {'Issues':>7} {'Time':>8}") + print("-" * 32) + for n, (t, e) in sorted(results.items(), key=lambda kv: kv[1][0]): + print(f"{n:<15} {t:>7} {e:>7.1f}s") + + +if __name__ == "__main__": + main() diff --git a/infra/scripts/eval_tts_rewrite.py b/infra/scripts/eval_tts_rewrite.py new file mode 100644 index 0000000..f180e15 --- /dev/null +++ b/infra/scripts/eval_tts_rewrite.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Quick eval: which model best rewrites technical notes for TTS? + +Runs each model on math-heavy excerpts (from notes.elimelt.com) with the +production system prompt, then scores outputs on: + - no leftover unspeakable notation (LaTeX, sub/superscripts, symbols) + - no preamble/meta-text ("Here is the rewritten...") + - content preserved (length ratio sane, key terms kept) + +Usage: python3 eval_tts_rewrite.py [host] [models_csv] +""" +import json +import re +import sys +import time +import urllib.request + +HOST = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11434" +MODELS = (sys.argv[2] if len(sys.argv) > 2 + else "gemma2:2b,llama3.2:3b,gemma3:27b").split(",") + +SYSTEM = ("You rewrite excerpts from technical notes so they can be read aloud " + "naturally by a text-to-speech engine. Expand math, symbols, code " + "identifiers, and abbreviations into spoken words (for example " + '"O(n log n)" becomes "order n log n", "x_i" becomes "x sub i"). ' + "LaTeX between \\( and \\) delimiters is inline math: speak it as a " + "person would read the formula aloud and drop the delimiters. Smooth " + "awkward notation into plain sentences but keep the meaning and " + "technical content identical. Do not add, explain, or summarize. " + "Output only the rewritten text with no preamble.") + +# Excerpts from notes.elimelt.com/math/linear-algebra/svd-and-pseudoinverse +CASES = [ + {"text": r"Let \(A \in R^{m \times n}\). The matrix \(A^T A\) is symmetric " + r"and positive semidefinite, so it has an orthonormal eigenbasis " + r"\(v_1, \ldots, v_n\) with eigenvalues \(\lambda_1 \geq \cdots " + r"\geq \lambda_n \geq 0\). Define the singular values " + r"\(\sigma_i = \sqrt{\lambda_i}\).", + "keywords": ["symmetric", "positive semidefinite", "orthonormal", + "eigenvalues", "singular values"]}, + {"text": r"\(\sigma_1 = \|A\|_2\) is the largest stretch factor any unit " + r"vector experiences, and \(\sigma_r\) the smallest nonzero one; " + r"their ratio \(\sigma_1 / \sigma_r\) is the condition number.", + "keywords": ["largest", "stretch", "unit vector", "condition number"]}, + {"text": r"When A has full column rank, \(A^+ = (A^T A)^{-1} A^T\), the " + r"least-squares operator from the normal equations. The general " + r"statement: \(\hat{x} = A^+ b\) is always a least-squares " + r"solution of \(Ax = b\).", + "keywords": ["full column rank", "least-squares", "normal equations"]}, + {"text": "This is exactly what np.linalg.lstsq returns, since its " + "SVD-based driver applies a truncated pseudoinverse. " + "np.linalg.pinv applies an rcond cutoff before inverting " + "singular values.", + "keywords": ["pseudoinverse", "cutoff", "singular values"]}, +] + +BAD_PATTERNS = [ + (r"\\\(|\\\)|\\[a-zA-Z]+\{|\\sigma|\\lambda|\\geq|\\in", "latex"), + (r"[_^]\{?[a-zA-Z0-9]", "sub/superscript"), + (r"[≥≤∈∑√σλΣ×]", "math symbol"), + (r"\^T|\^\+|\^\{-1\}|\^-1", "operator notation"), + (r"(?i)^\s*(here is|here's|sure|certainly|rewritten|okay|below is)", "preamble"), +] + + +def chat(model, text): + body = json.dumps({ + "model": model, "stream": False, + "options": {"temperature": 0.2, "num_predict": 400}, + "messages": [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": text}], + }).encode() + req = urllib.request.Request(f"{HOST}/api/chat", data=body, + headers={"Content-Type": "application/json"}) + t0 = time.perf_counter() + with urllib.request.urlopen(req, timeout=3600) as r: + out = json.load(r)["message"]["content"].strip() + return out, time.perf_counter() - t0 + + +def score(case, output): + issues = [] + for pat, name in BAD_PATTERNS: + if re.search(pat, output): + issues.append(name) + kw_missing = [k for k in case["keywords"] + if k.lower() not in output.lower()] + if kw_missing: + issues.append(f"dropped: {', '.join(kw_missing)}") + ratio = len(output) / len(case["text"]) + if ratio < 0.6: + issues.append(f"too short ({ratio:.1f}x)") + elif ratio > 2.5: + issues.append(f"too long ({ratio:.1f}x)") + return issues + + +def main(): + results = {} + for model in MODELS: + print(f"\n{'='*60}\n{model}\n{'='*60}") + total_issues, elapsed = 0, 0.0 + for i, case in enumerate(CASES): + try: + out, dt = chat(model, case["text"]) + except Exception as e: + print(f"[case {i+1}] FAILED: {e}") + total_issues += 10 + continue + issues = score(case, out) + total_issues += len(issues) + elapsed += dt + flag = "OK " if not issues else "BAD" + print(f"[case {i+1}] {flag} ({dt:.1f}s) " + f"{'; '.join(issues) if issues else ''}") + print(f" {out[:200]}{'...' if len(out) > 200 else ''}") + results[model] = (total_issues, elapsed) + + print(f"\n{'Model':<15} {'Issues':>7} {'Time':>8}") + print("-" * 32) + for m, (n, t) in sorted(results.items(), key=lambda kv: kv[1][0]): + print(f"{m:<15} {n:>7} {t:>7.1f}s") + + +if __name__ == "__main__": + main()