diff --git a/.gitignore b/.gitignore index b95131d..ae039b2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,9 +13,9 @@ web-ext.config.ts !.env.example *.local -# Eval harness artifacts (generated reports/caches) +# Eval harness artifacts (generated checkpoints/pages; reports (.md) are committed) eval/.cache/ -eval/results/*.json +eval/results/* !eval/results/.gitkeep # Editor / OS diff --git a/CHANGELOG.md b/CHANGELOG.md index b0efc7c..ad2171c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,44 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.0] - 2026-07-10 + +Research-grade evaluation: live model benchmark, judge calibration, and a +structured-output study — which caught and fixed a product-breaking +reasoning-model bug. + +### Added + +- Live translation benchmark (`pnpm bench`): 4 local models × 2 prompt + conditions × 27 curated EN→zh-TW fixtures with Taiwan-convention references; + reference-based chrF (sacrebleu-cross-validated), artifact rates, + TTFT-net/TTFT-UI latency, tokens/s, and a JSON-schema-constrained LLM judge + → `eval/BENCHMARK-RESULTS.md` +- Judge calibration workflow (`pnpm bench:agreement`): seeded blind labeling + page + Cohen's κ (plain & quadratic-weighted) judge↔human → `eval/AGREEMENT.md` +- Structured-output study (`pnpm eval:structured`): prompt-only vs + schema-constrained decoding across 4 models × 16 realistic capture excerpts, + with a failure-shape taxonomy → `eval/STRUCTURED-RESULTS.md` +- `docs/BENCHMARK.md` — methodology, found-bug case study, limitations + +### Changed + +- **Ollama client migrated from OpenAI-compat `/v1/chat/completions` to native + `/api/chat` with `think: false`** (now requires Ollama ≥ 0.9) +- Default model `qwen2.5` → `qwen3:latest` — chosen by the benchmark (best + chrF at the lowest TTFT of the models measured) +- Capture enrichment now uses schema-constrained decoding (Ollama `format`) — + the study measured it closing the last unreliable tail at zero latency + cost; the tolerant parser remains as content hygiene + +### Fixed + +- Reasoning models (qwen3 family, deepseek-r1) produced **no visible output** + through the compat endpoint — chain-of-thought consumed the entire + generation in a separate `reasoning` field (measured: 99 s / 4,055 tokens / + 0 visible characters on one fixture; 1.6 s after the fix). Found by the + benchmark harness. + ## [2.1.0] - 2026-07-09 Switch to local Ollama backend. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcd9df9..61098bc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,10 +48,14 @@ The translation reliability harness runs offline against a fixed dataset: pnpm eval ``` -The optional live LLM-as-judge (`eval/judge.ts`) calls a real model instead of -the fixed dataset. It needs a local [Ollama](https://ollama.com/) server — -install Ollama, then `ollama pull qwen2.5` — and reads the server URL from the -`OLLAMA_URL` env var (default `http://localhost:11434`); no API key required. +The live evaluation suite — `pnpm bench` (model × prompt benchmark), +`pnpm eval:structured` (structured-output study), `pnpm bench:agreement` +(judge↔human calibration), and the older `eval/judge.ts` — calls real models +instead of fixed datasets. It needs a local [Ollama](https://ollama.com/) +server (≥ 0.9) — install Ollama, then `ollama pull qwen3` — and reads the +server URL from the `OLLAMA_URL` env var (default `http://localhost:11434`); +no API key required. See [`docs/BENCHMARK.md`](docs/BENCHMARK.md) for the +methodology. ## Project layout diff --git a/README.md b/README.md index 3d35345..5f2d612 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,40 @@ _Measured over 23 curated fixtures (21 Traditional-Chinese targets). Regenerate with `pnpm eval`; full report in [`eval/RESULTS.md`](eval/RESULTS.md)._ The pure core carries **100% function coverage and ~94% line coverage** across -85 unit tests (`pnpm test:cov`). +120 unit tests (`pnpm test:cov`). + +## Which local model? — live benchmark + +The offline eval freezes model output to score the pipeline; `pnpm bench` +asks the opposite question against live models: **which model should you run, +and what does each design choice cost?** 27 curated EN→zh-TW fixtures with +Taiwan-convention references × 4 models × 2 prompt conditions, streamed +through the exact shipped pipeline and scored with sacrebleu-cross-validated +chrF, artifact detectors, latency probes, and a schema-constrained LLM judge +(itself calibrated against human labels — Cohen's κ workflow included). + +| Model | chrF ↑ | TTFT-UI p50 | Tokens/s | Verdict | +| --- | --- | --- | --- | --- | +| **qwen3** (default) | **46.3** | **451 ms** | 48 | best quality/latency balance | +| qwen3.5 | 43.4 | 730 ms | 42 | no chrF edge, 1.6× the wait | +| llama3.1 | 31.6 | 532 ms | 49 | fast, but ~13 chrF behind | +| deepseek-r1:8b | 36.6 | 6,353 ms | — | 6-second "thinking tax" — wrong workload | + +_Engineered-prompt condition, seed 42; full tables in +[`eval/BENCHMARK-RESULTS.md`](eval/BENCHMARK-RESULTS.md), methodology and +limitations in [`docs/BENCHMARK.md`](docs/BENCHMARK.md)._ + +Two findings worth calling out: + +- **The benchmark caught a product-breaking bug.** Through Ollama's + OpenAI-compat endpoint, reasoning models can spend the *entire* generation + on hidden chain-of-thought — one fixture: 99 s, 4,055 tokens, zero visible + characters. The client now uses the native `/api/chat` with `think: false` + (same fixture: 1.6 s). +- **The reliability layer is a measured tradeoff, not free.** It zeroes + preamble on dirty outputs and halves deepseek-r1's Simplified leakage, but + costs ~200 ms of first paint and a fraction of a chrF point on clean + outputs — numbers, not vibes, either way. ## How it works @@ -87,8 +120,13 @@ The pure core carries **100% function coverage and ~94% line coverage** across - **Cancellation-safe streaming** ([`src/api/ollama.ts`](src/api/ollama.ts) + [`src/entrypoints/background.ts`](src/entrypoints/background.ts)) — each - request owns an `AbortController`; a new selection or a closed panel aborts the - in-flight SSE with no shared mutable state to race on. + request owns an `AbortController`; a new selection or a closed panel aborts + the in-flight stream with no shared mutable state to race on. +- **Reasoning-model safe** — the client uses Ollama's native `/api/chat` with + `think: false` because the benchmark caught the OpenAI-compat endpoint + burning entire generations as hidden reasoning with zero visible output on + qwen3-family and deepseek-r1 models + ([`docs/BENCHMARK.md`](docs/BENCHMARK.md) §6). Requires Ollama ≥ 0.9. - **Fully local** — no cloud, no key, no telemetry. The selected text is sent only to a local Ollama server on your machine; nothing leaves your device. The server URL lives in `chrome.storage` and is read only by the background @@ -121,20 +159,20 @@ cheap, reliable part on-device and defers the expensive part — rather than re-implementing a knowledge base it has no business owning. Optionally, a small local model can pre-label a capture with a title, summary, -and tags. Small models are unreliable at structured output, so this is strictly -best-effort — and _measured_: `parseEnrichResponse` salvages usable metadata -from fenced, preamble-wrapped, and trailing-prose replies that a naive -`JSON.parse` drops. +and tags. That pipeline is defense-in-depth, and every layer is _measured_: -| Metric | Rate | -| --------------------------------------------------- | ---------- | -| Naive `JSON.parse` yields an object | 42.9% | -| Robust `parseEnrichResponse` yields usable metadata | **71.4%** | +| Layer | Evidence | +| --- | --- | +| Schema-constrained decoding (Ollama `format`) | took the one imperfect model (deepseek-r1) from 93.3% → **100%** usable metadata at zero latency cost — live study over 4 models × 16 excerpts ([`eval/STRUCTURED-RESULTS.md`](eval/STRUCTURED-RESULTS.md)) | +| Tolerant `parseEnrichResponse` | salvages 71.4% vs naive parsing's 42.9% on an archive of 14 hostile reply shapes from older/thinking models ([`eval/CAPTURE-RESULTS.md`](eval/CAPTURE-RESULTS.md)); today it mostly does content hygiene — length caps, tag normalisation | +| `status: raw` handoff | even a perfect-looking label is garnish; the raw capture stays the source of truth | -_Offline, deterministic run over 14 real small-model reply shapes; regenerate -with `pnpm eval:capture`, full report in -[`eval/CAPTURE-RESULTS.md`](eval/CAPTURE-RESULTS.md). This is why enrichment is -off by default — the raw capture is always the source of truth._ +The live study also produced an honest negative result: with the shipped +prompt, temperature 0, and thinking disabled, **modern small models emit +clean JSON ~100% of the time** — the dramatic salvage rates belong to older +model generations. Enrichment stays off by default anyway: it adds a +model round-trip per capture, and reasoning-class models take ~45 s to label +a paragraph (measured), which no capture UX survives. Set your vault, capture folder, and the enrichment toggle in the popup; leave the vault blank to use whichever vault is currently open. @@ -155,7 +193,8 @@ OpenRead translates through a local [Ollama](https://ollama.com/) server — no API key required. 1. [Install Ollama](https://ollama.com/). -2. Pull a model: `ollama pull qwen2.5`. +2. Pull a model: `ollama pull qwen3` (the benchmarked default; see + [`docs/BENCHMARK.md`](docs/BENCHMARK.md) for how it was chosen). 3. Start the server: `ollama serve`. 4. Allow the extension's origin through Ollama's CORS by setting `OLLAMA_ORIGINS=chrome-extension://*` before starting it: @@ -164,7 +203,7 @@ API key required. - **Windows**: set a user environment variable `OLLAMA_ORIGINS=chrome-extension://*`, then restart Ollama Open the toolbar popup and set the Ollama server URL (default -`http://localhost:11434`) and model (default `qwen2.5`), plus a target +`http://localhost:11434`) and model (default `qwen3:latest`), plus a target language. ## Usage diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8acff33..4bdf1a8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,9 +34,19 @@ public/ eval/ dataset/fixtures.json curated translation failure-mode fixtures dataset/capture-fixtures.json curated small-model enrichment reply shapes + dataset/bench-fixtures.json 27 EN→zh-TW segments with Taiwan-convention references + dataset/enrich-inputs.json 16 realistic capture excerpts (EN/zh-TW/mixed) detectors.ts preamble / Simplified / echo detectors (reuse core) - run.ts translation before→after runner → eval/RESULTS.md - capture-run.ts enrichment-parser runner → eval/CAPTURE-RESULTS.md + run.ts offline before→after runner → eval/RESULTS.md (CI gate) + capture-run.ts offline enrichment-parser runner → eval/CAPTURE-RESULTS.md + bench/ live model × prompt benchmark (needs Ollama, not CI) + chrf.ts chrF metric, sacrebleu-cross-validated + kappa.ts Cohen's κ (plain + quadratic-weighted) + run.ts matrix runner + LLM judge → eval/BENCHMARK-RESULTS.md + agreement.ts human-labeling page + judge↔human κ → eval/AGREEMENT.md + structured/ live structured-output study (needs Ollama, not CI) + taxonomy.ts failure-shape classifier for small-model replies + run.ts prompt vs schema-constrained → eval/STRUCTURED-RESULTS.md ``` ### Why this split @@ -56,14 +66,22 @@ eval/ A selection triggers a long-lived port to the background worker, which owns the network call and streams cleaned chunks back. Each request has its own `AbortController`, so a new selection (or a closed panel) cancels the previous -SSE cleanly. +stream cleanly. + +The client talks to Ollama's **native `/api/chat`** endpoint (NDJSON streaming) +with `think: false`. The benchmark found that the OpenAI-compat `/v1` endpoint +routes reasoning-model chain-of-thought into a separate field and can leave +`content` empty for the entire generation (qwen3.5: 99 s, 4,055 tokens, zero +visible output) — the native endpoint disables thinking on hybrid models and +keeps it out of `content` on models that cannot stop (deepseek-r1). Requires +Ollama ≥ 0.9. See [`BENCHMARK.md`](BENCHMARK.md) §6. ```mermaid sequenceDiagram participant U as User participant C as Content script (ui/selection.ts) participant B as Background broker - participant O as Ollama (local, SSE) + participant O as Ollama (local, /api/chat NDJSON) U->>C: select text, click 文 C->>C: shouldBypassAI(text, target)? @@ -72,14 +90,14 @@ sequenceDiagram else needs translation C->>B: connect "stream-translate" + START_STREAM {text, target, model} B->>B: load Ollama base URL from storage; new AbortController - B->>O: POST /chat/completions (stream:true, signal) - loop each SSE delta - O-->>B: data: {delta.content} + B->>O: POST /api/chat (stream:true, think:false, signal) + loop each NDJSON chunk + O-->>B: {message:{content}, done:false} B->>B: StreamAssembler.push (buffer opening,
strip preamble, SC→TC) B-->>C: {status:"streaming", chunk} C-->>U: append chunk to panel end - O-->>B: data: [DONE] + O-->>B: {done:true, eval_count} B-->>C: {status:"done"} end @@ -95,5 +113,7 @@ sequenceDiagram | OpenCC `s2twp` over a hand-rolled map | v1's unconditional character map corrupted common words (`界面→界麵`). Phrase-level conversion is correct and maintained. | | Ollama base URL read in background from storage; request goes to a local server | No secret rides the message bus — and since the request never leaves the machine, there's nothing to leak either way. | | Offline, deterministic eval | Before/after numbers are reproducible in CI without a running Ollama server, so they are honest to cite. | +| Native `/api/chat` + `think: false`, not the OpenAI-compat `/v1` | On reasoning models `/v1` can burn the whole generation as hidden `reasoning` with `content` empty — the user sees nothing. Found by the benchmark; see `docs/BENCHMARK.md` §6. | +| Live benchmark imports the shipped modules | `pnpm bench` scores `buildMessages`/`extractChunk`/`StreamAssembler` themselves, so its numbers describe the product, not a re-implementation. | | Capture writes via `obsidian://new` from the content script, not a new API | Keeps least-privilege intact (no `downloads`/native-host permission) and reuses the user gesture; oversized notes fall back to the clipboard. | | Captures are `status: raw`; heavy synthesis deferred to a downstream model | On-device small models fail at structured output (measured), so OpenRead ships a reliable raw note and lets a stronger "second brain" process it. | diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md new file mode 100644 index 0000000..f70dd68 --- /dev/null +++ b/docs/BENCHMARK.md @@ -0,0 +1,238 @@ +# OpenRead Benchmark — Methodology & Findings + +A local, reproducible evaluation of streaming-LLM translation quality, +reliability, and latency across four on-device models — plus a controlled +study of structured-output strategies for small models. Everything runs +against the **exact shipped code path**, so the numbers describe the product, +not a lab approximation of it. + +> Headline results live in the generated reports: +> [`eval/BENCHMARK-RESULTS.md`](../eval/BENCHMARK-RESULTS.md), +> [`eval/STRUCTURED-RESULTS.md`](../eval/STRUCTURED-RESULTS.md), and +> [`eval/AGREEMENT.md`](../eval/AGREEMENT.md). This document explains how they +> were produced and what they mean. + +## 1. Why benchmark a translator extension? + +OpenRead's thesis is that streaming-LLM output is an engineering material with +measurable failure modes — preamble, input echo, Simplified-character leakage, +hidden reasoning — not just "AI magic". The offline eval (`pnpm eval`) proves +the sanitizer removes artifacts from a *fixed* fixture set; this benchmark +asks the live questions the offline harness cannot: + +1. **Quality** — which local model translates EN→zh-TW best, and how much does + prompt engineering move it? +2. **Reliability** — how often do artifacts appear in *fresh* generations, and + does the shipped pipeline still catch them? +3. **Latency** — what does the user actually wait, and what does the + reliability layer cost in time-to-first-paint? +4. **Trust** — is the LLM judge itself calibrated against a human? + +## 2. Setup + +| | | +| --- | --- | +| Hardware | NVIDIA GeForce RTX 4060 Laptop GPU, 8 GB VRAM | +| Server | Ollama 0.31.1, Windows 11 | +| Models | `qwen3.5:latest` (6.6 GB), `qwen3:latest` (5.2 GB), `llama3.1:latest` (4.9 GB), `deepseek-r1:8b` (5.2 GB) | +| Decoding | temperature 0.3 (the extension's first-attempt setting), fixed seed 42 | +| Endpoint | native `/api/chat`, `think: false` — the shipped client path | +| Judge | `qwen3.5:latest`, temperature 0, JSON-schema-constrained output | + +**Product fidelity.** The runner imports the extension's own modules — +`buildMessages` (prompt), `extractChunk` (stream parsing), `StreamAssembler` +(reluctant buffer), OpenCC `s2twp` — so a benchmark cell exercises the same +bytes a user's selection does. The only deviations are a fixed seed and timing +probes. + +## 3. Dataset + +[`eval/dataset/bench-fixtures.json`](../eval/dataset/bench-fixtures.json): +**27 EN→zh-TW segments** across six domains — news (4), tech-docs (5), +academic (4), UI strings (4), colloquial (4), and adversarial "tricky" items +(6): idioms that must not be translated literally, units/numbers/inline code +that must survive verbatim, and a multi-sentence streaming stress paragraph. + +Each fixture carries a reference translation written in deliberate **Taiwan +conventions** — 升息一碼, 執行緒, 連接埠, 工作階段, 晶片 — so the metric can +distinguish "Traditional script" from "actually Taiwanese usage", which is +exactly the distinction v1's hand-rolled converter failed at. + +*Provenance & limitation:* references were drafted with a frontier LLM and +human-reviewed; they represent one good translation, not the only one. chrF +against a single reference therefore under-credits legitimate paraphrases — +fine for *comparing systems on the same footing*, not an absolute quality +scale. + +## 4. Metrics + +- **chrF** ([`eval/bench/chrf.ts`](../eval/bench/chrf.ts)) — character + n-gram F-score (n=1–6, β=2), the standard reference-based surface metric + for Chinese targets because it needs no word segmenter. Our implementation + is cross-validated against Python `sacrebleu` 2.6.0 to six decimal places + (see `chrf.test.ts`); corpus scores aggregate n-gram statistics, not + per-segment averages. +- **Artifact rates** — the same detectors the offline eval and the production + pipeline share (`hasPreamble`, `hasEcho`, `hasSimplifiedLeak`), applied to + both the raw model output and the shipped-pipeline output of every cell. +- **TTFT-net vs TTFT-UI** — ms from request start to the first content token, + vs to the first text the panel would actually paint. The gap prices the + reluctant buffer; for reasoning models the wait is dominated by hidden + thinking, which the runner tracks separately (`thinkingChars`). +- **Tokens/s** — generated tokens over generation time, from the final + NDJSON chunk's `eval_count`. +- **LLM judge** — reference-based grading on three 1–5 axes: adequacy, + fluency, Taiwan localization. Constrained decoding (`format` = JSON schema) + + `think: false` + temperature 0 make it cheap and deterministic. Judged + end-to-end experiences: the `naive` baseline's raw output vs the + `engineered` condition's shipped-pipeline output. + +### Judge calibration + +A judge is only evidence if it tracks human judgement. +`pnpm bench:agreement -- --make-page` samples 40 judged items (seeded, +stratified across model × condition, model identity hidden) into a +self-contained labeling page; a human rates them on the same rubric, and +`pnpm bench:agreement` reports raw agreement plus unweighted and +quadratically-weighted Cohen's κ per axis +([`eval/bench/kappa.ts`](../eval/bench/kappa.ts), verified against textbook +cases). Results land in [`eval/AGREEMENT.md`](../eval/AGREEMENT.md). + +## 5. Conditions + +| Condition | Prompt | Pipeline | +| --- | --- | --- | +| `naive` | one bare user instruction, no system prompt, no few-shot | scored raw *and* through the pipeline | +| `engineered` | the shipped `buildMessages`: role/rules system prompt + anti-echo few-shot | scored raw *and* through the pipeline | + +Two generations per model per fixture; the pipeline variants are free +(post-processing the same stream), so the matrix separates *prompting* gains +from *pipeline* gains. + +## 6. Found bug: reasoning models × the OpenAI-compat endpoint + +The first full run produced a product-breaking discovery. Through Ollama's +OpenAI-compat `/v1/chat/completions` — the endpoint v2.1 shipped with — +**reasoning models can emit their entire generation as `reasoning` while +`content` stays empty**: `qwen3.5` spent 99 s and 4,055 tokens on one fixture +and returned zero visible characters. Passing `think: false` through `/v1` +did not disable it. The extension would have shown users a spinner and then +nothing. + +The fix, shipped in the same change as this benchmark: migrate the client to +the **native `/api/chat`** endpoint with `think: false` — + +- hybrid thinkers (qwen3 family) actually stop thinking: the same qwen3 + fixture went from **99 s / empty** to **1.6 s / TTFT 282 ms**; +- non-thinkers (llama3.1) tolerate the flag as a no-op; +- models that cannot stop thinking (deepseek-r1) still keep reasoning out of + `content` — they pay a "thinking tax" in TTFT, which the latency table + quantifies, but the answer arrives clean. + +This is the benchmark working as intended: the harness exists to catch +exactly the class of failure a quick manual test misses. + +## 7. Results + +See [`eval/BENCHMARK-RESULTS.md`](../eval/BENCHMARK-RESULTS.md) for the full +tables (quality, artifacts, latency, judge scores). Summary of what to look +for: + + +Run of 2026-07-10 — 216/216 generations, 0 errors, all cells judged: + +- **`qwen3` (engineered prompt) is the quality/latency sweet spot**: corpus + chrF 46.3, TTFT-UI p50 451 ms, ~48 tokens/s. `qwen3.5` matches it on judge + scores but is no better on chrF and pays ~1.6× the TTFT. It is now the + extension's default model. +- **Prompt engineering is model-dependent, not free quality**: the shipped + system-prompt + few-shot lifts qwen3 (+1.7 chrF raw) and deepseek-r1 + (+3.6), does nothing for qwen3.5, and nothing for llama3.1 — which sits + ~13 chrF below the qwen family regardless of prompting. +- **The reliability layer's value concentrates on dirty outputs.** It zeroed + llama3.1's naive-prompt preamble (7.4% → 0%) and halved deepseek-r1's + Simplified leakage (18.5% → 7.4%), while on already-clean outputs it costs + a small amount of legitimate text (llama3.1: −0.3 to −0.5 chrF) plus + ~200 ms of TTFT — the reluctant buffer's measured price. +- **Residual Simplified leakage (~7%) survives the pipeline** on several + models. Consistent with the per-delta OpenCC transform being unable to + convert a phrase split across two stream chunks — a concrete, testable + follow-up for the streaming layer. +- **The thinking tax is disqualifying for interactive use**: deepseek-r1's + TTFT is 6.4–6.6 **seconds** (vs 0.3–0.8 s for everything else) because it + cannot stop reasoning even with `think: false`; its tokens/s figure is + inflated for the same reason (`eval_count` includes hidden thinking + tokens). Quality-wise it also trails the qwen family. Fine model, wrong + workload. +- **Judge scores rank models the same way chrF does** (qwen family > + llama3.1 ≈ deepseek-r1 on adequacy), with one caveat: `naive` conditions + are judged on raw output and `engineered` on pipeline output, so the small, + consistent adequacy dip under `engineered` partly reflects the pipeline's + clipping, and qwen3.5 grading its own family invites bias — hence the human + calibration step (§4). + + +## 8. Structured-output study + +The capture feature optionally asks a small model for `{title, summary, +tags}` metadata. The earlier offline eval showed a robust salvage parser +lifts usable-metadata rate from 42.9% to 71.4% over 14 canned reply shapes; +the live study ([`eval/STRUCTURED-RESULTS.md`](../eval/STRUCTURED-RESULTS.md)) +asks the next question: **does schema-constrained decoding beat prompt +engineering + robust parsing on fresh generations?** + +Design: 16 realistic capture excerpts (EN / zh-TW / mixed) × 4 models × 2 +generation conditions (`prompt` = shipped prompt rules; `schema` = the same +request with Ollama's `format` JSON schema), each raw reply scored by naive +`JSON.parse`, by the shipped `parseEnrichResponse`, and by a failure-shape +taxonomy ([`eval/structured/taxonomy.ts`](../eval/structured/taxonomy.ts)) — +clean / fenced / prose-wrapped / thinking-contaminated / truncated / no-JSON. + + +Run of 2026-07-10 — 128 generations, and the result is an honest surprise: + +- **The 2024-era failure modes did not reproduce.** With the shipped prompt, + temperature 0, and `think: false` on the native endpoint, all four models + returned clean, directly-parseable JSON on essentially every reply (naive + `JSON.parse`: 100%). The fenced/preamble/trailing-prose shapes that + motivated the salvage parser — and its offline 42.9% → 71.4% rescue rate on + archived reply shapes — belong mostly to older models and to + thinking-contaminated output, which the client migration itself eliminated. +- **Schema-constrained decoding still closes the last tail at zero cost**: + deepseek-r1 was the only imperfect model (93.3% all-three-fields, one + timed-out cell) and `format` took it to 100%/0 errors with identical median + latency. The extension now sends the schema on every enrichment request; + `parseEnrichResponse` remains as the content-hygiene layer (length caps, + tag normalisation) rather than a JSON rescue. +- **The thinking tax again**: deepseek-r1's median enrichment is ~45 s + (thinking through the whole labeling task) vs 1.1–1.8 s for the others — + reinforcing the benchmark's conclusion that reasoning models are the wrong + tool for interactive assist features, independent of output shape. + + +## 9. Limitations + +- **Single seed, one generation per cell** — differences of a point or two of + chrF are noise; read the big gaps. +- **Judge family bias** — qwen3.5 judges its own relatives; mitigated by + reference-based grading and the human-calibration protocol, not eliminated. +- **Single-reference chrF** under-credits paraphrase (§3). +- **Latency is machine-specific** (RTX 4060 Laptop, 8 GB); relative + comparisons are the point. +- **n = 27 fixtures** — domain-level breakdowns are indicative, not + significant. + +## 10. Reproduce + +```bash +ollama serve # any Ollama ≥ 0.9 with the four models pulled +pnpm bench # matrix + judge + eval/BENCHMARK-RESULTS.md (~30–60 min) +pnpm eval:structured # structured-output study (~10 min) +pnpm bench:agreement -- --make-page # then rate eval/results/labeling.html +pnpm bench:agreement # κ judge↔human -> eval/AGREEMENT.md +``` + +Both runners checkpoint after every cell (`eval/results/*.json`) and resume; +`--report-only` regenerates reports without touching the GPU. The offline +gates (`pnpm eval`, `pnpm eval:capture`, `pnpm test`) stay network-free and +deterministic for CI. diff --git a/eval/BENCHMARK-RESULTS.md b/eval/BENCHMARK-RESULTS.md new file mode 100644 index 0000000..9f902de --- /dev/null +++ b/eval/BENCHMARK-RESULTS.md @@ -0,0 +1,62 @@ +# OpenRead — Translation Benchmark + +Live model × prompt matrix over **27** curated EN→zh-TW fixtures (216 generations recorded). Generations run through the exact shipped pipeline (`buildMessages` → native `/api/chat` NDJSON, `think: false` → `extractChunk` → `StreamAssembler` + OpenCC). Decoding: temperature 0.3, seed 42. Judge: `qwen3.5:latest` (native `/api/chat`, JSON-schema constrained, temperature 0). Regenerate with `pnpm bench`. + +## Quality — corpus chrF against references + +| Model | Prompt | chrF raw | chrF shipped pipeline | Δ pipeline | +| --- | --- | --- | --- | --- | +| qwen3.5:latest | naive | 44.1 | 44.7 | 0.6 | +| qwen3.5:latest | engineered | 42.9 | 43.4 | 0.4 | +| qwen3:latest | naive | 44.2 | 44.4 | 0.1 | +| qwen3:latest | engineered | 45.9 | 46.3 | 0.5 | +| llama3.1:latest | naive | 32.5 | 32.1 | -0.3 | +| llama3.1:latest | engineered | 32.1 | 31.6 | -0.5 | +| deepseek-r1:8b | naive | 32.8 | 35.2 | 2.4 | +| deepseek-r1:8b | engineered | 36.4 | 36.6 | 0.1 | + +## Streaming artifacts — raw vs shipped pipeline + +| Model | Prompt | Preamble raw→piped | Echo raw→piped | Simplified raw→piped | +| --- | --- | --- | --- | --- | +| qwen3.5:latest | naive | 0.0% → 0.0% | 0.0% → 0.0% | 7.4% → 7.4% | +| qwen3.5:latest | engineered | 0.0% → 0.0% | 0.0% → 0.0% | 7.4% → 7.4% | +| qwen3:latest | naive | 0.0% → 0.0% | 0.0% → 0.0% | 7.4% → 7.4% | +| qwen3:latest | engineered | 0.0% → 0.0% | 0.0% → 0.0% | 11.1% → 7.4% | +| llama3.1:latest | naive | 7.4% → 0.0% | 0.0% → 0.0% | 7.4% → 3.7% | +| llama3.1:latest | engineered | 0.0% → 0.0% | 0.0% → 0.0% | 7.4% → 3.7% | +| deepseek-r1:8b | naive | 0.0% → 0.0% | 0.0% → 0.0% | 18.5% → 7.4% | +| deepseek-r1:8b | engineered | 0.0% → 0.0% | 0.0% → 0.0% | 11.1% → 11.1% | + +## Latency + +_TTFT-net = first SSE content token; TTFT-UI = first text the panel paints (after the reluctant buffer). The gap is the price of preamble filtering; for reasoning models the wait is dominated by hidden thinking._ + +| Model | Prompt | TTFT-net p50 (ms) | TTFT-UI p50 (ms) | Tokens/s | Errors | +| --- | --- | --- | --- | --- | --- | +| qwen3.5:latest | naive | 554 | 766 | 13.7 | 0/27 | +| qwen3.5:latest | engineered | 545 | 730 | 42.4 | 0/27 | +| qwen3:latest | naive | 258 | 453 | 47.8 | 0/27 | +| qwen3:latest | engineered | 255 | 451 | 48.0 | 0/27 | +| llama3.1:latest | naive | 301 | 534 | 48.6 | 0/27 | +| llama3.1:latest | engineered | 299 | 532 | 49.1 | 0/27 | +| deepseek-r1:8b | naive | 6568 | 6568 | 457.0 | 0/27 | +| deepseek-r1:8b | engineered | 6353 | 6353 | 477.0 | 0/27 | + +## LLM-judge quality (1–5) + +_Judged end-to-end experiences: `naive` = raw baseline output, `engineered` = shipped pipeline output. Reference-based grading; see `docs/BENCHMARK.md` for judge calibration against human labels._ + +| Model | Prompt | Adequacy | Fluency | TW localization | Judged | +| --- | --- | --- | --- | --- | --- | +| qwen3.5:latest | naive | 4.74 | 4.89 | 4.78 | 27/27 | +| qwen3.5:latest | engineered | 4.67 | 4.93 | 4.78 | 27/27 | +| qwen3:latest | naive | 4.70 | 4.78 | 4.33 | 27/27 | +| qwen3:latest | engineered | 4.48 | 4.70 | 4.37 | 27/27 | +| llama3.1:latest | naive | 4.44 | 4.70 | 4.37 | 27/27 | +| llama3.1:latest | engineered | 4.30 | 4.63 | 4.41 | 27/27 | +| deepseek-r1:8b | naive | 4.41 | 4.81 | 4.22 | 27/27 | +| deepseek-r1:8b | engineered | 4.19 | 4.85 | 4.41 | 27/27 | + +_Hardware: local Ollama (http://localhost:11434). Latency numbers are machine-specific; relative comparisons are the point._ + diff --git a/eval/STRUCTURED-RESULTS.md b/eval/STRUCTURED-RESULTS.md new file mode 100644 index 0000000..9fadf65 --- /dev/null +++ b/eval/STRUCTURED-RESULTS.md @@ -0,0 +1,37 @@ +# OpenRead — Structured-Output Study + +Small-model enrichment replies over **16** realistic capture excerpts × **4** local models, generated once per condition (temperature 0, seed 42) and scored offline. `prompt` = the shipped prompt-rules-only path; `schema` = decoding constrained to the EnrichResult JSON schema. Regenerate with `pnpm eval:structured`. + +## Headline — usable metadata rate by strategy + +| Generation | Parse | Usable rate | All 3 fields | +| --- | --- | --- | --- | +| prompt | naive `JSON.parse` | 100.0% | — | +| prompt | robust `parseEnrichResponse` | 100.0% | 98.4% | +| schema | naive `JSON.parse` | 100.0% | — | +| schema | robust `parseEnrichResponse` | 100.0% | 100.0% | + +## Per model + +| Model | Gen | Naive parse | Robust parse | All fields | Median ms | Errors | +| --- | --- | --- | --- | --- | --- | --- | +| qwen3.5:latest | prompt | 100.0% | 100.0% | 100.0% | 1838 | 0/16 | +| qwen3.5:latest | schema | 100.0% | 100.0% | 100.0% | 1845 | 0/16 | +| qwen3:latest | prompt | 100.0% | 100.0% | 100.0% | 1665 | 0/16 | +| qwen3:latest | schema | 100.0% | 100.0% | 100.0% | 1729 | 0/16 | +| llama3.1:latest | prompt | 100.0% | 100.0% | 100.0% | 1126 | 0/16 | +| llama3.1:latest | schema | 100.0% | 100.0% | 100.0% | 1109 | 0/16 | +| deepseek-r1:8b | prompt | 100.0% | 100.0% | 93.3% | 46300 | 1/16 | +| deepseek-r1:8b | schema | 100.0% | 100.0% | 100.0% | 45309 | 0/16 | + +## Failure shapes — unconstrained (`prompt`) replies + +| Model | clean-json | fenced-json | thinking-then-json | json-with-prose | truncated-json | no-json | empty | +| --- | --- | --- | --- | --- | --- | --- | --- | +| qwen3.5:latest | 16 | 0 | 0 | 0 | 0 | 0 | 0 | +| qwen3:latest | 16 | 0 | 0 | 0 | 0 | 0 | 0 | +| llama3.1:latest | 16 | 0 | 0 | 0 | 0 | 0 | 0 | +| deepseek-r1:8b | 15 | 0 | 0 | 0 | 0 | 0 | 0 | + +_Both conditions run on the native `/api/chat` endpoint with `think: false` (the shipped client path); `schema` adds the `format` parameter. Latency medians include any hidden reasoning time._ + diff --git a/eval/bench/agreement.ts b/eval/bench/agreement.ts new file mode 100644 index 0000000..697d3bb --- /dev/null +++ b/eval/bench/agreement.ts @@ -0,0 +1,262 @@ +/** + * Judge calibration — how much can the LLM judge be trusted? + * + * An LLM judge is only evidence if its ratings track human judgement, so this + * script closes the loop in two steps: + * + * 1. `pnpm bench:agreement -- --make-page [n]` + * Samples n judged items (default 40, seeded PRNG, stratified across + * model × condition cells) into `eval/results/labeling.html` — a + * self-contained page where a human rates the same outputs on the same + * 1–5 rubric, blind to the judge's scores and to which model produced + * what. The page exports `human-labels.json`. + * + * 2. `pnpm bench:agreement` (labels saved to `eval/dataset/human-labels.json`) + * Reports per-axis agreement between judge and human: raw percent, + * unweighted Cohen's kappa, and quadratically weighted kappa (ordinal + * scales should forgive 4-vs-5 more than 1-vs-5). Writes + * `eval/AGREEMENT.md`. + */ +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { cohenKappa, weightedKappa } from './kappa'; + +interface GenerationRecord { + model: string; + condition: 'naive' | 'engineered'; + fixtureId: string; + raw: string; + piped: string; + error?: string; +} +interface JudgeScore { + adequacy: number; + fluency: number; + localization: number; +} +interface Checkpoint { + generations: Record; + judgements: Record; +} +interface Fixture { + id: string; + source: string; + reference: string; +} +interface HumanLabel extends JudgeScore { + key: string; +} + +const here = dirname(fileURLToPath(import.meta.url)); +const checkpointPath = join(here, '..', 'results', 'bench-raw.json'); +const labelsPath = join(here, '..', 'dataset', 'human-labels.json'); +const pagePath = join(here, '..', 'results', 'labeling.html'); +const reportPath = join(here, '..', 'AGREEMENT.md'); + +const AXES = ['adequacy', 'fluency', 'localization'] as const; +const CATEGORIES = [1, 2, 3, 4, 5]; + +function loadCheckpoint(): Checkpoint { + return JSON.parse(readFileSync(checkpointPath, 'utf8')) as Checkpoint; +} +function loadFixtures(): Map { + const fixtures = JSON.parse( + readFileSync(join(here, '..', 'dataset', 'bench-fixtures.json'), 'utf8'), + ) as Fixture[]; + return new Map(fixtures.map((f) => [f.id, f])); +} + +/** The same output the judge graded: raw for naive, piped for engineered. */ +function judgedOutput(record: GenerationRecord): string { + return record.condition === 'naive' ? record.raw : record.piped; +} + +/** Deterministic PRNG so the sampled page is reproducible. */ +function mulberry32(seed: number): () => number { + let state = seed; + return () => { + state |= 0; + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function makePage(sampleSize: number): void { + const checkpoint = loadCheckpoint(); + const fixtures = loadFixtures(); + + const judged = Object.entries(checkpoint.judgements) + .filter(([, j]) => j.score) + .map(([key]) => key) + .sort(); + if (judged.length === 0) { + console.error('No judged items in the checkpoint — run `pnpm bench` first.'); + process.exitCode = 1; + return; + } + + // Stratify: round-robin across model|condition cells, random within each. + const random = mulberry32(42); + const byCell = new Map(); + for (const key of judged) { + const [model, condition] = key.split('|'); + const cell = `${model}|${condition}`; + byCell.set(cell, [...(byCell.get(cell) ?? []), key]); + } + for (const keys of byCell.values()) { + keys.sort(() => random() - 0.5); + } + const sampled: string[] = []; + const cells = [...byCell.values()]; + for (let round = 0; sampled.length < Math.min(sampleSize, judged.length); round++) { + let advanced = false; + for (const keys of cells) { + const key = keys[round]; + if (key === undefined) continue; + advanced = true; + sampled.push(key); + if (sampled.length >= Math.min(sampleSize, judged.length)) break; + } + if (!advanced) break; + } + // Shuffle presentation order so consecutive items don't share a model. + sampled.sort(() => random() - 0.5); + + const items = sampled.map((key) => { + const record = checkpoint.generations[key]; + const fixture = record ? fixtures.get(record.fixtureId) : undefined; + return { + key, + source: fixture?.source ?? '', + reference: fixture?.reference ?? '', + candidate: record ? judgedOutput(record) : '', + }; + }); + + const html = ` + +OpenRead — human labeling (${items.length} items) + +

OpenRead — human labels

+

Rate each CANDIDATE 1–5 per axis (adequacy = meaning preserved; +fluency = natural Traditional Chinese; localization = Taiwan script & +terminology, 1 if Simplified characters appear). Model identities are hidden. +When done, click Export and save the file as +eval/dataset/human-labels.json, then run +pnpm bench:agreement.

+ +
+ +`; + writeFileSync(pagePath, html, 'utf8'); + console.log(`Wrote ${pagePath} with ${items.length} items.`); +} + +function computeAgreement(): void { + if (!existsSync(labelsPath)) { + console.error( + `No human labels at ${labelsPath}.\n` + + 'Generate the labeling page with `pnpm bench:agreement -- --make-page`, ' + + 'rate the items, save the export there, then re-run.', + ); + process.exitCode = 1; + return; + } + const checkpoint = loadCheckpoint(); + const labels = JSON.parse(readFileSync(labelsPath, 'utf8')) as HumanLabel[]; + + const lines: string[] = []; + lines.push('# OpenRead — Judge ↔ Human Agreement'); + lines.push(''); + lines.push( + `Cohen's kappa between the LLM judge and **${labels.length}** human-rated ` + + 'items (1–5 scales). Quadratic weighting is the headline number for ' + + 'ordinal ratings; ≥0.4 = moderate, ≥0.6 = substantial agreement.', + ); + lines.push(''); + lines.push('| Axis | Raw agreement | Cohen κ | Weighted κ (quadratic) | n |'); + lines.push('| --- | --- | --- | --- | --- |'); + + for (const axis of AXES) { + const human: number[] = []; + const judge: number[] = []; + for (const label of labels) { + const score = checkpoint.judgements[label.key]?.score; + if (!score) continue; + human.push(label[axis]); + judge.push(score[axis]); + } + if (human.length === 0) continue; + const exact = human.filter((h, i) => h === judge[i]).length / human.length; + lines.push( + `| ${axis} | ${(exact * 100).toFixed(1)}% ` + + `| ${cohenKappa(human, judge, CATEGORIES).toFixed(3)} ` + + `| ${weightedKappa(human, judge, CATEGORIES).toFixed(3)} ` + + `| ${human.length} |`, + ); + } + lines.push(''); + const report = lines.join('\n'); + writeFileSync(reportPath, report + '\n', 'utf8'); + console.log(report); +} + +const pageFlag = process.argv.indexOf('--make-page'); +if (pageFlag >= 0) { + makePage(Number(process.argv[pageFlag + 1]) || 40); +} else { + computeAgreement(); +} diff --git a/eval/bench/chrf.test.ts b/eval/bench/chrf.test.ts new file mode 100644 index 0000000..7e2320a --- /dev/null +++ b/eval/bench/chrf.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest'; +import { addStats, chrfFromStats, chrfScore, chrfStats, corpusChrf } from './chrf'; + +/** + * Expected values cross-validated against Python sacrebleu 2.6.0 + * (`sacrebleu.sentence_chrf(hyp, [ref])`, default chrF2: char order 6, β=2, + * whitespace removed, no eps smoothing) — identical to 6 decimal places. + */ +const SACREBLEU_CASES: Array<{ + id: string; + hyp: string; + ref: string; + score: number; +}> = [ + { id: 'hand-tiny', hyp: 'ab', ref: 'ac', score: 25.0 }, + { + id: 'identical-zh', + hyp: '快取會將經常存取的資料儲存在記憶體中。', + ref: '快取會將經常存取的資料儲存在記憶體中。', + score: 100.0, + }, + { + id: 'close-zh', + hyp: '快取把經常存取的資料存放在記憶體中，減少往返資料庫的次數。', + ref: '快取會將經常存取的資料儲存在記憶體中，減少往返資料庫的次數。', + score: 71.339874, + }, + { + id: 'far-zh', + hyp: '緩存把常用數據放內存里，少跑幾趟數據庫。', + ref: '快取會將經常存取的資料儲存在記憶體中，減少往返資料庫的次數。', + score: 4.761905, + }, + { + id: 'english', + hyp: 'The cat sat on the mat.', + ref: 'A cat was sitting on the mat.', + score: 39.784852, + }, + { + id: 'short-zh', + hyp: '你好世界', + ref: '哈囉世界', + score: 20.833333, + }, +]; + +describe('chrfScore', () => { + for (const c of SACREBLEU_CASES) { + it(`matches sacrebleu on ${c.id}`, () => { + expect(chrfScore(c.hyp, c.ref)).toBeCloseTo(c.score, 5); + }); + } + + it('ignores whitespace differences, like sacrebleu', () => { + expect( + chrfScore( + '新晶片速度快了 3.2 倍，滿載時功耗 15 瓦。', + '新晶片速度快了3.2倍，滿載時功耗15瓦。', + ), + ).toBe(100); + }); + + it('scores an empty hypothesis 0', () => { + expect(chrfScore('', '你好')).toBe(0); + }); + + it('scores an empty reference 0', () => { + expect(chrfScore('你好', '')).toBe(0); + }); +}); + +describe('chrfStats', () => { + it('produces per-order clipped counts', () => { + // hyp "ab": unigrams a,b; bigram ab. ref "ac": unigrams a,c; bigram ac. + const stats = chrfStats('ab', 'ac'); + expect(stats[0]).toEqual({ match: 1, hyp: 2, ref: 2 }); + expect(stats[1]).toEqual({ match: 0, hyp: 1, ref: 1 }); + expect(stats[2]).toEqual({ match: 0, hyp: 0, ref: 0 }); + }); + + it('counts by code point, not UTF-16 code unit', () => { + // Surrogate-pair characters must count as one character each. + const stats = chrfStats('𝒂𝒃', '𝒂𝒄'); + expect(stats[0]).toEqual({ match: 1, hyp: 2, ref: 2 }); + }); +}); + +describe('corpusChrf', () => { + it('matches sacrebleu corpus_chrf on the validation set', () => { + // sacrebleu 2.6.0 corpus_chrf over the seven non-empty cases: 58.391739 + const pairs = [ + ...SACREBLEU_CASES.map((c) => ({ hypothesis: c.hyp, reference: c.ref })), + { + hypothesis: '新晶片速度快了 3.2 倍，滿載時功耗 15 瓦。', + reference: '新晶片速度快了3.2倍，滿載時功耗15瓦。', + }, + ]; + expect(corpusChrf(pairs)).toBeCloseTo(58.391739, 5); + }); + + it('aggregates stats rather than averaging segment scores', () => { + const first = { hypothesis: 'ab', reference: 'ac' }; + const second = { hypothesis: '你好世界', reference: '哈囉世界' }; + const summed = addStats( + chrfStats(first.hypothesis, first.reference), + chrfStats(second.hypothesis, second.reference), + ); + expect(corpusChrf([first, second])).toBeCloseTo(chrfFromStats(summed), 10); + }); +}); diff --git a/eval/bench/chrf.ts b/eval/bench/chrf.ts new file mode 100644 index 0000000..bafb4eb --- /dev/null +++ b/eval/bench/chrf.ts @@ -0,0 +1,112 @@ +/** + * chrF — character n-gram F-score (Popović, WMT 2015), the standard surface + * metric for machine-translation quality against a reference. + * + * Chosen over BLEU/chrF++ deliberately: Chinese has no whitespace word + * boundaries, so word-n-gram metrics need a segmenter (a dependency and a + * noise source); chrF operates on characters and is the widely recommended + * surface metric for zh targets. Parameters match sacrebleu defaults + * (char order 6, β = 2, whitespace removed) and the implementation is + * cross-validated against sacrebleu in `chrf.test.ts`. + * + * Pure and dependency-free so the benchmark scores are reproducible anywhere. + */ + +export const CHAR_ORDER = 6; +const BETA = 2; + +export interface ChrfOrderStats { + match: number; + hyp: number; + ref: number; +} + +/** Count character n-grams of one order, whitespace removed, by code point. */ +function charNgrams(text: string, n: number): Map { + const chars = Array.from(text.replace(/\s+/g, '')); + const grams = new Map(); + for (let i = 0; i + n <= chars.length; i++) { + const gram = chars.slice(i, i + n).join(''); + grams.set(gram, (grams.get(gram) ?? 0) + 1); + } + return grams; +} + +/** Per-order clipped match / hypothesis / reference counts (orders 1..6). */ +export function chrfStats( + hypothesis: string, + reference: string, +): ChrfOrderStats[] { + const stats: ChrfOrderStats[] = []; + for (let n = 1; n <= CHAR_ORDER; n++) { + const hypGrams = charNgrams(hypothesis, n); + const refGrams = charNgrams(reference, n); + let match = 0; + let hyp = 0; + for (const [gram, count] of hypGrams) { + hyp += count; + match += Math.min(count, refGrams.get(gram) ?? 0); + } + let ref = 0; + for (const count of refGrams.values()) ref += count; + stats.push({ match, hyp, ref }); + } + return stats; +} + +/** Element-wise sum of per-order stats — for corpus-level aggregation. */ +export function addStats( + a: ChrfOrderStats[], + b: ChrfOrderStats[], +): ChrfOrderStats[] { + return a.map((s, i) => ({ + match: s.match + (b[i]?.match ?? 0), + hyp: s.hyp + (b[i]?.hyp ?? 0), + ref: s.ref + (b[i]?.ref ?? 0), + })); +} + +/** + * chrF from aggregated stats, 0..100. Precision and recall are macro-averaged + * across the "effective" orders (orders where both sides produced n-grams), + * then combined into a single Fβ — the chrF-paper formulation, numerically + * identical to sacrebleu's non-smoothed scoring. + */ +export function chrfFromStats(stats: ChrfOrderStats[]): number { + const betaSq = BETA * BETA; + let precSum = 0; + let recSum = 0; + let effectiveOrders = 0; + for (const { match, hyp, ref } of stats) { + if (hyp === 0 || ref === 0) continue; + effectiveOrders++; + precSum += match / hyp; + recSum += match / ref; + } + if (effectiveOrders === 0) return 0; + const avgPrec = precSum / effectiveOrders; + const avgRec = recSum / effectiveOrders; + const denom = betaSq * avgPrec + avgRec; + if (denom === 0) return 0; + return (((1 + betaSq) * avgPrec * avgRec) / denom) * 100; +} + +/** Sentence-level chrF, 0..100. */ +export function chrfScore(hypothesis: string, reference: string): number { + return chrfFromStats(chrfStats(hypothesis, reference)); +} + +/** Corpus-level chrF: per-order stats summed over all pairs, then one F. */ +export function corpusChrf( + pairs: Array<{ hypothesis: string; reference: string }>, +): number { + let total: ChrfOrderStats[] = Array.from({ length: CHAR_ORDER }, () => ({ + match: 0, + hyp: 0, + ref: 0, + })); + for (const { hypothesis, reference } of pairs) { + total = addStats(total, chrfStats(hypothesis, reference)); + } + return chrfFromStats(total); +} diff --git a/eval/bench/kappa.test.ts b/eval/bench/kappa.test.ts new file mode 100644 index 0000000..46bea04 --- /dev/null +++ b/eval/bench/kappa.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; +import { cohenKappa, weightedKappa } from './kappa'; + +/** Expand a confusion-matrix spec into two aligned rating lists. */ +function expand( + cells: Array<[a: number, b: number, count: number]>, +): [number[], number[]] { + const a: number[] = []; + const b: number[] = []; + for (const [ra, rb, count] of cells) { + for (let i = 0; i < count; i++) { + a.push(ra); + b.push(rb); + } + } + return [a, b]; +} + +describe('cohenKappa', () => { + it('matches the textbook 2x2 example (po=0.7, pe=0.5 → κ=0.4)', () => { + // 20 yes/yes, 15 no/no, 10 yes/no, 5 no/yes over 50 items. + const [a, b] = expand([ + [1, 1, 20], + [0, 0, 15], + [1, 0, 10], + [0, 1, 5], + ]); + expect(cohenKappa(a, b, [0, 1])).toBeCloseTo(0.4, 10); + }); + + it('is 1 for perfect agreement', () => { + expect(cohenKappa([1, 2, 3, 1], [1, 2, 3, 1], [1, 2, 3])).toBe(1); + }); + + it('is 0 for chance-level agreement', () => { + // Rater B says 1 half the time regardless of A: po = pe. + const [a, b] = expand([ + [1, 1, 5], + [1, 2, 5], + [2, 1, 5], + [2, 2, 5], + ]); + expect(cohenKappa(a, b, [1, 2])).toBeCloseTo(0, 10); + }); + + it('can be negative for systematic disagreement', () => { + const [a, b] = expand([ + [1, 2, 10], + [2, 1, 10], + ]); + expect(cohenKappa(a, b, [1, 2])).toBeLessThan(0); + }); + + it('throws on mismatched lengths', () => { + expect(() => cohenKappa([1], [1, 2], [1, 2])).toThrow(/differ in length/); + }); + + it('throws on empty input', () => { + expect(() => cohenKappa([], [], [1, 2])).toThrow(/zero items/); + }); + + it('throws on out-of-category ratings', () => { + expect(() => cohenKappa([9], [1], [1, 2])).toThrow(/outside categories/); + }); +}); + +describe('weightedKappa (quadratic)', () => { + it('is 1 for perfect agreement', () => { + expect(weightedKappa([1, 3, 5], [1, 3, 5], [1, 2, 3, 4, 5])).toBe(1); + }); + + it('is -1 for perfectly reversed ratings on a uniform 3-scale', () => { + // Hand-computed: Σd·O = 2, Σd·E = 1 → κw = 1 - 2/1 = -1. + expect(weightedKappa([1, 2, 3], [3, 2, 1], [1, 2, 3])).toBeCloseTo(-1, 10); + }); + + it('penalises near-misses less than the unweighted kappa', () => { + // All disagreements are adjacent (4 vs 5): unweighted sees pure + // disagreement, quadratic weighting forgives most of it. + const [a, b] = expand([ + [5, 4, 5], + [4, 5, 5], + [1, 1, 5], + [2, 2, 5], + ]); + const cats = [1, 2, 3, 4, 5]; + expect(weightedKappa(a, b, cats)).toBeGreaterThan(cohenKappa(a, b, cats)); + }); + + it('hand-checked mixed example', () => { + // Ratings over categories 1..3, N=4: + // items: (1,1) (2,3) (3,3) (3,2) + // O-cells: (0,0)=1 (1,2)=1 (2,2)=1 (2,1)=1; d(i,j)=(i-j)^2/4 + // Σd·O = 0 + 0.25 + 0 + 0.25 = 0.5 + // row marg = [1,1,2], col marg = [1,1,2] + // Σd·E = Σ d(i,j)·row_i·col_j/4 + // d01·1·1 + d02·1·2 + d10·1·1 + d12·1·2 + d20·2·1 + d21·2·1 (all /4) + // = (0.25 + 2 + 0.25 + 0.5 + 2 + 0.5)/4 = 5.5/4 = 1.375 + // κw = 1 - 0.5/1.375 = 0.636363... + expect(weightedKappa([1, 2, 3, 3], [1, 3, 3, 2], [1, 2, 3])).toBeCloseTo( + 1 - 0.5 / 1.375, + 10, + ); + }); +}); diff --git a/eval/bench/kappa.ts b/eval/bench/kappa.ts new file mode 100644 index 0000000..392c921 --- /dev/null +++ b/eval/bench/kappa.ts @@ -0,0 +1,103 @@ +/** + * Cohen's kappa — inter-rater agreement corrected for chance. Used to + * calibrate the LLM judge against human labels: raw percent agreement is + * inflated by chance on skewed rating distributions, kappa is not. + * + * Two variants: + * - cohenKappa: unweighted, for categorical labels. + * - weightedKappa: quadratic weights, for ordinal scales (1–5 ratings), + * where a 4-vs-5 disagreement should cost less than 1-vs-5. + * + * Pure and dependency-free; verified against hand-computed textbook examples. + */ + +function buildConfusion( + a: number[], + b: number[], + categories: number[], +): number[][] { + if (a.length !== b.length) { + throw new Error(`Rating lists differ in length: ${a.length} vs ${b.length}`); + } + if (a.length === 0) throw new Error('Cannot compute kappa on zero items'); + const index = new Map(categories.map((c, i) => [c, i])); + const k = categories.length; + const matrix = Array.from({ length: k }, () => new Array(k).fill(0)); + for (let i = 0; i < a.length; i++) { + const va = a[i]; + const vb = b[i]; + const ra = va === undefined ? undefined : index.get(va); + const rb = vb === undefined ? undefined : index.get(vb); + const row = ra === undefined ? undefined : matrix[ra]; + if (row === undefined || rb === undefined) { + throw new Error( + `Rating outside categories at item ${i}: ${String(va)}/${String(vb)}`, + ); + } + row[rb] = (row[rb] ?? 0) + 1; + } + return matrix; +} + +function marginals(matrix: number[][]): { row: number[]; col: number[] } { + const k = matrix.length; + const row = new Array(k).fill(0); + const col = new Array(k).fill(0); + for (let i = 0; i < k; i++) { + for (let j = 0; j < k; j++) { + const value = matrix[i]?.[j] ?? 0; + row[i] = (row[i] ?? 0) + value; + col[j] = (col[j] ?? 0) + value; + } + } + return { row, col }; +} + +/** Unweighted Cohen's kappa over two raters' labels. */ +export function cohenKappa( + a: number[], + b: number[], + categories: number[], +): number { + const matrix = buildConfusion(a, b, categories); + const n = a.length; + const { row, col } = marginals(matrix); + const k = categories.length; + + let observed = 0; + for (let i = 0; i < k; i++) observed += matrix[i]?.[i] ?? 0; + const po = observed / n; + + let pe = 0; + for (let i = 0; i < k; i++) pe += ((row[i] ?? 0) * (col[i] ?? 0)) / (n * n); + + if (pe === 1) return po === 1 ? 1 : 0; + return (po - pe) / (1 - pe); +} + +/** Quadratically weighted Cohen's kappa for ordinal categories (in order). */ +export function weightedKappa( + a: number[], + b: number[], + categories: number[], +): number { + const matrix = buildConfusion(a, b, categories); + const n = a.length; + const { row, col } = marginals(matrix); + const k = categories.length; + if (k < 2) return 1; + + const denomScale = (k - 1) * (k - 1); + let weightedObserved = 0; + let weightedExpected = 0; + for (let i = 0; i < k; i++) { + for (let j = 0; j < k; j++) { + const disagreement = ((i - j) * (i - j)) / denomScale; + weightedObserved += disagreement * (matrix[i]?.[j] ?? 0); + weightedExpected += disagreement * (((row[i] ?? 0) * (col[j] ?? 0)) / n); + } + } + + if (weightedExpected === 0) return weightedObserved === 0 ? 1 : 0; + return 1 - weightedObserved / weightedExpected; +} diff --git a/eval/bench/run.ts b/eval/bench/run.ts new file mode 100644 index 0000000..1c2fd14 --- /dev/null +++ b/eval/bench/run.ts @@ -0,0 +1,681 @@ +/** + * Live translation benchmark — model × prompt matrix over the curated + * EN→Traditional-Chinese fixture set, scored with reference-based chrF, + * artifact detectors, latency, and an LLM judge. + * + * Product fidelity: generations go through the SAME code path the extension + * ships — `buildMessages` for the prompt, `/v1/chat/completions` SSE parsed by + * `extractDelta`, assembled by `StreamAssembler` with the OpenCC transform — + * so every number describes shipped behaviour, not a lab approximation. The + * only additions are a fixed seed (reproducibility) and timing probes. + * + * Judge calls use Ollama's native `/api/chat` with a constrained JSON-schema + * `format` and `think: false` — the judge is infrastructure, not the system + * under test, so it does not need the product code path. + * + * Requires a local Ollama server; NOT wired into CI (the offline `pnpm eval` + * stays the deterministic gate). Results checkpoint after every cell, so an + * interrupted run resumes where it stopped: + * + * pnpm bench # full matrix + judge + report + * pnpm bench -- --models qwen3:latest --fixtures news-01 --skip-judge + * pnpm bench -- --report-only # regenerate the report from checkpoints + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { buildMessages, extractChunk } from '../../src/api/ollama'; +import { StreamAssembler } from '../../src/core/stream'; +import { toTraditionalTW } from '../../src/core/zh-convert'; +import { hasEcho, hasPreamble, hasSimplifiedLeak } from '../detectors'; +import { chrfStats, addStats, chrfFromStats, CHAR_ORDER } from './chrf'; +import type { ChatMessage } from '../../src/core/types'; + +// --- configuration ----------------------------------------------------------- + +const TARGET_LANG = 'Traditional Chinese'; +const TEMPERATURE = 0.3; // the extension's first-attempt temperature +const SEED = 42; +const REQUEST_TIMEOUT_MS = 300_000; + +const DEFAULT_MODELS = [ + 'qwen3.5:latest', + 'qwen3:latest', + 'llama3.1:latest', + 'deepseek-r1:8b', +]; + +export type ConditionId = 'naive' | 'engineered'; +const CONDITIONS: ConditionId[] = ['naive', 'engineered']; + +/** + * The baseline any first implementation would use: a bare instruction, no + * system prompt, no few-shot, no output-format rules. + */ +function naiveMessages(text: string): ChatMessage[] { + return [ + { + role: 'user', + content: `Translate the following text to Traditional Chinese:\n\n${text}`, + }, + ]; +} + +function messagesFor(condition: ConditionId, text: string): ChatMessage[] { + return condition === 'naive' + ? naiveMessages(text) + : buildMessages(text, TARGET_LANG); +} + +// --- types ------------------------------------------------------------------- + +interface Fixture { + id: string; + domain: string; + source: string; + reference: string; + note: string; +} + +interface GenerationRecord { + model: string; + condition: ConditionId; + fixtureId: string; + /** Concatenated raw content deltas — what the model actually emitted. */ + raw: string; + /** Characters of hidden chain-of-thought (kept out of content by Ollama). */ + thinkingChars: number; + /** Output of the shipped streaming pipeline (StreamAssembler + OpenCC). */ + piped: string; + /** ms from request start to the first SSE content delta. */ + ttftNetMs: number | null; + /** ms from request start to the first text the UI would paint. */ + ttftUiMs: number | null; + totalMs: number; + completionTokens: number | null; + error?: string; +} + +interface JudgeScore { + adequacy: number; + fluency: number; + localization: number; +} + +interface JudgeRecord { + key: string; + score?: JudgeScore; + error?: string; +} + +interface Checkpoint { + meta: { + baseUrl: string; + targetLang: string; + temperature: number; + seed: number; + judgeModel: string; + startedAt: string; + }; + generations: Record; + judgements: Record; +} + +// --- CLI / files ------------------------------------------------------------- + +const here = dirname(fileURLToPath(import.meta.url)); +const resultsDir = join(here, '..', 'results'); +const checkpointPath = join(resultsDir, 'bench-raw.json'); +const reportPath = join(here, '..', 'BENCHMARK-RESULTS.md'); + +const baseUrl = (process.env.OLLAMA_URL ?? 'http://localhost:11434').replace( + /\/+$/, + '', +); +const judgeModel = process.env.JUDGE_MODEL ?? 'qwen3.5:latest'; + +function argList(flag: string): string[] | null { + const index = process.argv.indexOf(flag); + const value = index < 0 ? undefined : process.argv[index + 1]; + if (!value) return null; + return value.split(',').map((s) => s.trim()); +} +const onlyModels = argList('--models'); +const onlyFixtures = argList('--fixtures'); +const skipJudge = process.argv.includes('--skip-judge'); +const reportOnly = process.argv.includes('--report-only'); + +const fixtures = ( + JSON.parse( + readFileSync(join(here, '..', 'dataset', 'bench-fixtures.json'), 'utf8'), + ) as Fixture[] +).filter((f) => !onlyFixtures || onlyFixtures.includes(f.id)); + +const models = (onlyModels ?? DEFAULT_MODELS).slice(); + +function loadCheckpoint(): Checkpoint { + try { + return JSON.parse(readFileSync(checkpointPath, 'utf8')) as Checkpoint; + } catch { + return { + meta: { + baseUrl, + targetLang: TARGET_LANG, + temperature: TEMPERATURE, + seed: SEED, + judgeModel, + startedAt: new Date().toISOString(), + }, + generations: {}, + judgements: {}, + }; + } +} + +function saveCheckpoint(checkpoint: Checkpoint): void { + mkdirSync(resultsDir, { recursive: true }); + writeFileSync(checkpointPath, JSON.stringify(checkpoint, null, 1), 'utf8'); +} + +const cellKey = (model: string, condition: string, fixtureId: string) => + `${model}|${condition}|${fixtureId}`; + +// --- generation -------------------------------------------------------------- + +/** + * Stream one translation exactly the way the extension does, with timing + * probes around the shipped parsing/assembly path. + */ +async function generate( + model: string, + condition: ConditionId, + fixture: Fixture, +): Promise { + const record: GenerationRecord = { + model, + condition, + fixtureId: fixture.id, + raw: '', + thinkingChars: 0, + piped: '', + ttftNetMs: null, + ttftUiMs: null, + totalMs: 0, + completionTokens: null, + }; + + const assembler = new StreamAssembler({ transform: toTraditionalTW }); + const emitted: string[] = []; + const t0 = performance.now(); + + try { + const response = await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + messages: messagesFor(condition, fixture.source), + stream: true, + think: false, + options: { temperature: TEMPERATURE, seed: SEED }, + }), + }); + if (!response.ok || !response.body) { + throw new Error(`HTTP ${response.status}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder('utf-8'); + let lineBuffer = ''; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + lineBuffer += decoder.decode(value, { stream: true }); + const lines = lineBuffer.split('\n'); + lineBuffer = lines.pop() ?? ''; + for (const line of lines) { + const chunk = extractChunk(line); + if (chunk === null) continue; + if (chunk.evalCount !== undefined) { + record.completionTokens = chunk.evalCount; + } + record.thinkingChars += chunk.thinking.length; + if (chunk.content === '') continue; + if (record.ttftNetMs === null) { + record.ttftNetMs = performance.now() - t0; + } + record.raw += chunk.content; + const emit = assembler.push(chunk.content); + if (emit) { + if (record.ttftUiMs === null) { + record.ttftUiMs = performance.now() - t0; + } + emitted.push(emit); + } + } + } + const tail = assembler.end(); + if (tail) { + if (record.ttftUiMs === null) record.ttftUiMs = performance.now() - t0; + emitted.push(tail); + } + } catch (error) { + record.error = error instanceof Error ? error.message : String(error); + } + + record.totalMs = performance.now() - t0; + record.piped = emitted.join(''); + return record; +} + +// --- judge ------------------------------------------------------------------- + +const JUDGE_SCHEMA = { + type: 'object', + properties: { + adequacy: { type: 'integer', minimum: 1, maximum: 5 }, + fluency: { type: 'integer', minimum: 1, maximum: 5 }, + localization: { type: 'integer', minimum: 1, maximum: 5 }, + }, + required: ['adequacy', 'fluency', 'localization'], +} as const; + +function judgePrompt(fixture: Fixture, candidate: string): string { + return [ + 'You are grading one machine translation from English into Traditional Chinese (Taiwan).', + '', + `SOURCE (English): ${fixture.source}`, + `REFERENCE (a good human translation): ${fixture.reference}`, + `CANDIDATE (the translation to grade): ${candidate}`, + '', + 'Score the CANDIDATE on three axes, each an integer 1-5:', + '- adequacy: meaning preserved. 5 = complete and correct; 3 = noticeable omissions or errors; 1 = wrong or unrelated. Non-translation text (explanations, the English source echoed back, reasoning) lowers adequacy.', + '- fluency: natural Traditional Chinese. 5 = reads like a native text; 3 = understandable but awkward; 1 = broken or not Chinese.', + '- localization: Taiwan conventions. 5 = Traditional script with Taiwan terminology throughout; 3 = Traditional script but mainland terminology; 1 = Simplified characters present.', + '', + 'Grade only what is in CANDIDATE. Reply with the JSON object only.', + ].join('\n'); +} + +/** Judge one candidate via native /api/chat with a constrained JSON schema. */ +async function judge( + fixture: Fixture, + candidate: string, +): Promise { + const body: Record = { + model: judgeModel, + messages: [{ role: 'user', content: judgePrompt(fixture, candidate) }], + stream: false, + format: JUDGE_SCHEMA, + think: false, + options: { temperature: 0, seed: SEED }, + }; + let response = await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!response.ok) { + // Some models reject the think flag — retry without it. + delete body.think; + response = await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + } + if (!response.ok) throw new Error(`judge HTTP ${response.status}`); + + const data = (await response.json()) as { + message?: { content?: string }; + }; + const parsed = JSON.parse(data.message?.content ?? '') as JudgeScore; + for (const key of ['adequacy', 'fluency', 'localization'] as const) { + const value = parsed[key]; + if (!Number.isInteger(value) || value < 1 || value > 5) { + throw new Error(`judge returned invalid ${key}: ${String(value)}`); + } + } + return parsed; +} + +/** The two end-to-end experiences worth judging per (model, fixture). */ +function judgedOutput(record: GenerationRecord): string | null { + if (record.error) return null; + return record.condition === 'naive' ? record.raw : record.piped; +} + +// --- scoring / report ---------------------------------------------------------- + +interface CellAggregate { + model: string; + condition: ConditionId; + count: number; + errors: number; + chrfRaw: number; + chrfPiped: number; + preambleRaw: number; + preamblePiped: number; + echoRaw: number; + echoPiped: number; + simplifiedRaw: number; + simplifiedPiped: number; + ttftNetP50: number | null; + ttftUiP50: number | null; + tokensPerSec: number | null; + judgeMeans: { adequacy: number; fluency: number; localization: number } | null; + judged: number; +} + +function percentile(values: number[], p: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil(p * sorted.length) - 1), + ); + return sorted[index] ?? null; +} + +function aggregate(checkpoint: Checkpoint): CellAggregate[] { + const byFixture = new Map(fixtures.map((f) => [f.id, f])); + const cells: CellAggregate[] = []; + + for (const model of models) { + for (const condition of CONDITIONS) { + const records = Object.values(checkpoint.generations).filter( + (r) => + r.model === model && + r.condition === condition && + byFixture.has(r.fixtureId), + ); + if (records.length === 0) continue; + const ok = records.filter((r) => !r.error); + + let statsRaw = Array.from({ length: CHAR_ORDER }, () => ({ + match: 0, + hyp: 0, + ref: 0, + })); + let statsPiped = statsRaw.map((s) => ({ ...s })); + const ttftNet: number[] = []; + const ttftUi: number[] = []; + let tokenSum = 0; + let tokenTimeMs = 0; + let preambleRaw = 0; + let preamblePiped = 0; + let echoRaw = 0; + let echoPiped = 0; + let simplifiedRaw = 0; + let simplifiedPiped = 0; + + for (const r of ok) { + const fixture = byFixture.get(r.fixtureId); + if (!fixture) continue; + statsRaw = addStats(statsRaw, chrfStats(r.raw, fixture.reference)); + statsPiped = addStats(statsPiped, chrfStats(r.piped, fixture.reference)); + if (hasPreamble(r.raw)) preambleRaw++; + if (hasPreamble(r.piped)) preamblePiped++; + if (hasEcho(fixture.source, r.raw)) echoRaw++; + if (hasEcho(fixture.source, r.piped)) echoPiped++; + if (hasSimplifiedLeak(r.raw)) simplifiedRaw++; + if (hasSimplifiedLeak(r.piped)) simplifiedPiped++; + if (r.ttftNetMs !== null) ttftNet.push(r.ttftNetMs); + if (r.ttftUiMs !== null) ttftUi.push(r.ttftUiMs); + if (r.completionTokens !== null && r.ttftNetMs !== null) { + tokenSum += r.completionTokens; + tokenTimeMs += r.totalMs - r.ttftNetMs; + } + } + + const judgeScores = ok + .map( + (r) => + checkpoint.judgements[cellKey(r.model, r.condition, r.fixtureId)] + ?.score, + ) + .filter((s): s is JudgeScore => Boolean(s)); + + cells.push({ + model, + condition, + count: records.length, + errors: records.length - ok.length, + chrfRaw: chrfFromStats(statsRaw), + chrfPiped: chrfFromStats(statsPiped), + preambleRaw, + preamblePiped, + echoRaw, + echoPiped, + simplifiedRaw, + simplifiedPiped, + ttftNetP50: percentile(ttftNet, 0.5), + ttftUiP50: percentile(ttftUi, 0.5), + tokensPerSec: tokenTimeMs > 0 ? (tokenSum / tokenTimeMs) * 1000 : null, + judgeMeans: + judgeScores.length > 0 + ? { + adequacy: + judgeScores.reduce((s, j) => s + j.adequacy, 0) / + judgeScores.length, + fluency: + judgeScores.reduce((s, j) => s + j.fluency, 0) / + judgeScores.length, + localization: + judgeScores.reduce((s, j) => s + j.localization, 0) / + judgeScores.length, + } + : null, + judged: judgeScores.length, + }); + } + } + return cells; +} + +function fmtMs(ms: number | null): string { + return ms === null ? '—' : `${Math.round(ms)}`; +} +function fmtRate(count: number, total: number): string { + if (total === 0) return '—'; + return `${((count / total) * 100).toFixed(1)}%`; +} +function fmt(n: number | null | undefined, digits = 1): string { + return n === null || n === undefined ? '—' : n.toFixed(digits); +} + +function writeReport(checkpoint: Checkpoint): void { + const cells = aggregate(checkpoint); + const total = Object.values(checkpoint.generations).length; + const lines: string[] = []; + + lines.push('# OpenRead — Translation Benchmark'); + lines.push(''); + lines.push( + `Live model × prompt matrix over **${fixtures.length}** curated EN→zh-TW fixtures ` + + `(${total} generations recorded). Generations run through the exact shipped ` + + 'pipeline (`buildMessages` → native `/api/chat` NDJSON, `think: false` → ' + + '`extractChunk` → `StreamAssembler` + OpenCC). Decoding: temperature ' + + `${checkpoint.meta.temperature}, seed ${checkpoint.meta.seed}. ` + + `Judge: \`${checkpoint.meta.judgeModel}\` (native \`/api/chat\`, JSON-schema ` + + 'constrained, temperature 0). Regenerate with `pnpm bench`.', + ); + lines.push(''); + + lines.push('## Quality — corpus chrF against references'); + lines.push(''); + lines.push('| Model | Prompt | chrF raw | chrF shipped pipeline | Δ pipeline |'); + lines.push('| --- | --- | --- | --- | --- |'); + for (const c of cells) { + lines.push( + `| ${c.model} | ${c.condition} | ${fmt(c.chrfRaw)} | ${fmt(c.chrfPiped)} | ` + + `${fmt(c.chrfPiped - c.chrfRaw)} |`, + ); + } + lines.push(''); + + lines.push('## Streaming artifacts — raw vs shipped pipeline'); + lines.push(''); + lines.push( + '| Model | Prompt | Preamble raw→piped | Echo raw→piped | Simplified raw→piped |', + ); + lines.push('| --- | --- | --- | --- | --- |'); + for (const c of cells) { + const okCount = c.count - c.errors; + lines.push( + `| ${c.model} | ${c.condition} ` + + `| ${fmtRate(c.preambleRaw, okCount)} → ${fmtRate(c.preamblePiped, okCount)} ` + + `| ${fmtRate(c.echoRaw, okCount)} → ${fmtRate(c.echoPiped, okCount)} ` + + `| ${fmtRate(c.simplifiedRaw, okCount)} → ${fmtRate(c.simplifiedPiped, okCount)} |`, + ); + } + lines.push(''); + + lines.push('## Latency'); + lines.push(''); + lines.push( + '_TTFT-net = first content token off the wire; TTFT-UI = first text the panel paints ' + + '(after the reluctant buffer). The gap is the price of preamble filtering; ' + + 'for reasoning models the wait is dominated by hidden thinking._', + ); + lines.push(''); + lines.push( + '| Model | Prompt | TTFT-net p50 (ms) | TTFT-UI p50 (ms) | Tokens/s | Errors |', + ); + lines.push('| --- | --- | --- | --- | --- | --- |'); + for (const c of cells) { + lines.push( + `| ${c.model} | ${c.condition} | ${fmtMs(c.ttftNetP50)} | ${fmtMs(c.ttftUiP50)} ` + + `| ${fmt(c.tokensPerSec)} | ${c.errors}/${c.count} |`, + ); + } + lines.push(''); + + lines.push('## LLM-judge quality (1–5)'); + lines.push(''); + lines.push( + '_Judged end-to-end experiences: `naive` = raw baseline output, `engineered` = ' + + 'shipped pipeline output. Reference-based grading; see `docs/BENCHMARK.md` ' + + 'for judge calibration against human labels._', + ); + lines.push(''); + lines.push('| Model | Prompt | Adequacy | Fluency | TW localization | Judged |'); + lines.push('| --- | --- | --- | --- | --- | --- |'); + for (const c of cells) { + lines.push( + `| ${c.model} | ${c.condition} | ${fmt(c.judgeMeans?.adequacy, 2)} ` + + `| ${fmt(c.judgeMeans?.fluency, 2)} | ${fmt(c.judgeMeans?.localization, 2)} ` + + `| ${c.judged}/${c.count - c.errors} |`, + ); + } + lines.push(''); + lines.push( + `_Hardware: local Ollama (${checkpoint.meta.baseUrl}). Latency numbers are ` + + 'machine-specific; relative comparisons are the point._', + ); + lines.push(''); + + const report = lines.join('\n'); + writeFileSync(reportPath, report + '\n', 'utf8'); + console.log(report); +} + +// --- main ---------------------------------------------------------------------- + +/** One throwaway 1-token call so model-load time never lands in a cell. */ +async function warmup(model: string): Promise { + try { + await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: 'Hi' }], + stream: false, + think: false, + options: { num_predict: 1 }, + }), + }); + } catch { + /* best-effort */ + } +} + +async function main(): Promise { + const checkpoint = loadCheckpoint(); + + if (!reportOnly) { + const totalCells = models.length * CONDITIONS.length * fixtures.length; + let done = 0; + for (const model of models) { + const modelPending = CONDITIONS.some((condition) => + fixtures.some((f) => { + const cell = checkpoint.generations[cellKey(model, condition, f.id)]; + return !cell || cell.error; + }), + ); + if (modelPending) await warmup(model); + for (const condition of CONDITIONS) { + for (const fixture of fixtures) { + const key = cellKey(model, condition, fixture.id); + done++; + const existing = checkpoint.generations[key]; + if (existing && !existing.error) continue; + process.stdout.write( + `[${done}/${totalCells}] ${key} ... `, + ); + const record = await generate(model, condition, fixture); + checkpoint.generations[key] = record; + saveCheckpoint(checkpoint); + console.log( + record.error + ? `ERROR ${record.error}` + : `${Math.round(record.totalMs)}ms ttft=${fmtMs(record.ttftNetMs)}ms`, + ); + } + } + } + + if (!skipJudge) { + const pending = Object.values(checkpoint.generations).filter( + (r) => + judgedOutput(r) !== null && + fixtures.some((f) => f.id === r.fixtureId) && + !checkpoint.judgements[cellKey(r.model, r.condition, r.fixtureId)] + ?.score, + ); + let judgedCount = 0; + for (const record of pending) { + const key = cellKey(record.model, record.condition, record.fixtureId); + judgedCount++; + process.stdout.write(`[judge ${judgedCount}/${pending.length}] ${key} ... `); + try { + const score = await judge( + fixtures.find((f) => f.id === record.fixtureId)!, + judgedOutput(record)!, + ); + checkpoint.judgements[key] = { key, score }; + console.log( + `a=${score.adequacy} f=${score.fluency} l=${score.localization}`, + ); + } catch (error) { + checkpoint.judgements[key] = { + key, + error: error instanceof Error ? error.message : String(error), + }; + console.log(`ERROR ${checkpoint.judgements[key].error}`); + } + saveCheckpoint(checkpoint); + } + } + } + + writeReport(checkpoint); +} + +void main(); diff --git a/eval/dataset/bench-fixtures.json b/eval/dataset/bench-fixtures.json new file mode 100644 index 0000000..e9962f9 --- /dev/null +++ b/eval/dataset/bench-fixtures.json @@ -0,0 +1,191 @@ +[ + { + "id": "news-01", + "domain": "news", + "source": "The central bank raised interest rates by a quarter point on Thursday, citing persistent inflation and a tight labor market.", + "reference": "中央銀行週四宣布升息一碼，理由是通膨持續居高不下，且勞動市場依然緊俏。", + "note": "Taiwan financial jargon: 升息一碼 for a quarter-point hike." + }, + { + "id": "news-02", + "domain": "news", + "source": "Wildfires forced thousands of residents to evacuate coastal towns overnight, and officials warned that strong winds could spread the flames further inland.", + "reference": "野火迫使數千名居民連夜撤離沿海城鎮，官員警告強風可能使火勢進一步向內陸蔓延。", + "note": "Compound sentence with reported speech." + }, + { + "id": "news-03", + "domain": "news", + "source": "The company's shares fell nearly eight percent after it cut its annual revenue forecast for the second time this year.", + "reference": "該公司今年第二度下修年度營收預測後，股價下跌近百分之八。", + "note": "Taiwan financial term 下修; clause reordering required for natural zh." + }, + { + "id": "news-04", + "domain": "news", + "source": "Negotiators reached a provisional agreement early Saturday, though both sides cautioned that several key issues remain unresolved.", + "reference": "談判代表於週六凌晨達成臨時協議，但雙方均提醒，數項關鍵議題仍懸而未決。", + "note": "Concessive clause; formal register." + }, + { + "id": "tech-01", + "domain": "tech-docs", + "source": "The cache stores frequently accessed data in memory, reducing the number of round trips to the database.", + "reference": "快取會將經常存取的資料儲存在記憶體中，減少往返資料庫的次數。", + "note": "Taiwan IT terms: 快取, 存取, 資料, 記憶體, 資料庫." + }, + { + "id": "tech-02", + "domain": "tech-docs", + "source": "If the request fails with a timeout, the client retries with exponential backoff, up to a maximum of five attempts.", + "reference": "若請求因逾時而失敗，用戶端會以指數退避方式重試，最多五次。", + "note": "Taiwan terms: 逾時 (not 超时), 用戶端." + }, + { + "id": "tech-03", + "domain": "tech-docs", + "source": "This function returns a promise that resolves when all pending writes have been flushed to disk.", + "reference": "此函式會回傳一個 Promise，當所有待處理的寫入都已寫入磁碟時便會完成。", + "note": "Taiwan terms: 函式 (not 函数), 回傳 (not 返回); Promise conventionally untranslated." + }, + { + "id": "tech-04", + "domain": "tech-docs", + "source": "Deprecated: use the async variant instead. This method blocks the main thread and will be removed in the next major release.", + "reference": "已棄用：請改用非同步版本。此方法會阻塞主執行緒，並將在下一個主要版本中移除。", + "note": "Taiwan terms: 非同步 (not 异步), 執行緒 (not 线程)." + }, + { + "id": "tech-05", + "domain": "tech-docs", + "source": "By default, the server listens on port 8080; set the PORT environment variable to override this behavior.", + "reference": "伺服器預設在連接埠 8080 上監聽；設定 PORT 環境變數即可覆寫此行為。", + "note": "Taiwan terms: 伺服器, 預設, 連接埠 (not 端口); PORT must stay verbatim." + }, + { + "id": "acad-01", + "domain": "academic", + "source": "We propose a novel attention mechanism that reduces memory consumption by forty percent while preserving accuracy on downstream tasks.", + "reference": "我們提出一種新穎的注意力機制，可在維持下游任務準確率的同時，將記憶體用量降低百分之四十。", + "note": "ML paper abstract style; spelled-out percentage." + }, + { + "id": "acad-02", + "domain": "academic", + "source": "The results suggest that the observed correlation is largely driven by confounding variables rather than a causal relationship.", + "reference": "結果顯示，觀察到的相關性主要來自干擾變數，而非因果關係。", + "note": "Statistics register: 干擾變數, 因果關係." + }, + { + "id": "acad-03", + "domain": "academic", + "source": "Participants were randomly assigned to one of three conditions, and the experiment was conducted double-blind.", + "reference": "受試者被隨機分派至三種實驗條件之一，實驗採雙盲方式進行。", + "note": "Methods-section phrasing: 受試者, 隨機分派, 雙盲." + }, + { + "id": "acad-04", + "domain": "academic", + "source": "These findings should be interpreted with caution, as the sample size was small and drawn from a single institution.", + "reference": "由於樣本數少且僅來自單一機構，這些發現應謹慎解讀。", + "note": "Hedging language; cause-first reordering for natural zh." + }, + { + "id": "ui-01", + "domain": "ui-strings", + "source": "Your session has expired. Please sign in again to continue.", + "reference": "您的工作階段已逾期，請重新登入以繼續。", + "note": "Taiwan Microsoft-glossary terms: 工作階段 (not 会话), 登入 (not 登录)." + }, + { + "id": "ui-02", + "domain": "ui-strings", + "source": "Are you sure you want to delete this file? This action cannot be undone.", + "reference": "確定要刪除這個檔案嗎？此動作無法復原。", + "note": "Taiwan terms: 檔案 (not 文件), 復原." + }, + { + "id": "ui-03", + "domain": "ui-strings", + "source": "Settings saved. Restart the application for the changes to take effect.", + "reference": "設定已儲存。請重新啟動應用程式，變更才會生效。", + "note": "Taiwan terms: 儲存 (not 保存), 應用程式 (not 应用程序)." + }, + { + "id": "ui-04", + "domain": "ui-strings", + "source": "Drag and drop files here, or click to browse.", + "reference": "將檔案拖曳至此，或按一下以瀏覽。", + "note": "Taiwan UI conventions: 拖曳, 按一下 (not 单击/点击)." + }, + { + "id": "coll-01", + "domain": "colloquial", + "source": "Honestly, the movie wasn't as bad as everyone said — I'd probably watch it again.", + "reference": "老實說，這部電影沒有大家說的那麼糟——我搞不好還會再看一次。", + "note": "Casual register; hedged future." + }, + { + "id": "coll-02", + "domain": "colloquial", + "source": "Sorry for the late reply! Things have been crazy at work this week.", + "reference": "抱歉這麼晚才回覆！這週工作忙翻了。", + "note": "Chat register; idiomatic 忙翻了." + }, + { + "id": "coll-03", + "domain": "colloquial", + "source": "That restaurant is totally worth the wait, but go early or you'll be standing in line forever.", + "reference": "那間餐廳絕對值得等，但要早點去，不然會排隊排到天荒地老。", + "note": "Hyperbole; measure word 間 for restaurants." + }, + { + "id": "coll-04", + "domain": "colloquial", + "source": "I can't believe he actually pulled it off — everyone thought the deadline was impossible.", + "reference": "真不敢相信他居然辦到了——大家都以為那個期限根本不可能達成。", + "note": "Phrasal verb 'pull it off'; exclamative." + }, + { + "id": "tricky-01", + "domain": "tricky", + "source": "It's not rocket science — just read the manual before you start.", + "reference": "這又不是什麼高深的學問——開始前先把說明書讀一讀就好。", + "note": "Idiom must not be translated literally (火箭科學)." + }, + { + "id": "tricky-02", + "domain": "tricky", + "source": "The new chip is 3.2 times faster, draws 15 watts under load, and costs $249.", + "reference": "新晶片速度快了 3.2 倍，滿載時功耗 15 瓦，售價 249 美元。", + "note": "Numbers and units must survive; Taiwan term 晶片 (not 芯片)." + }, + { + "id": "tricky-03", + "domain": "tricky", + "source": "Run `npm install` first, then set `DEBUG=true` in your .env file to enable verbose logging.", + "reference": "請先執行 `npm install`，接著在 .env 檔案中設定 `DEBUG=true`，即可啟用詳細記錄。", + "note": "Inline code and filenames must stay verbatim." + }, + { + "id": "tricky-04", + "domain": "tricky", + "source": "The Taipei Metro carries more than two million passengers a day, and the fare depends on the distance traveled.", + "reference": "台北捷運每天載運超過兩百萬人次，票價依乘車距離而定。", + "note": "Proper noun 台北捷運; passenger-trips measure 人次." + }, + { + "id": "tricky-05", + "domain": "tricky", + "source": "Large language models generate text one token at a time, which means the user interface receives a stream of small fragments rather than a complete answer. A streaming UI must decide when to display each fragment: showing tokens immediately feels responsive but risks exposing artifacts, while buffering the entire response hides artifacts at the cost of a long blank wait. Most production systems settle on a compromise between the two.", + "reference": "大型語言模型逐一產生文字 token，也就是說，使用者介面收到的是一連串的小片段，而不是一次完整的答案。串流介面必須決定何時顯示每個片段：立即顯示雖然感覺反應靈敏，卻可能讓瑕疵直接暴露在畫面上；把整段回應先緩衝起來雖能遮住瑕疵，代價卻是漫長的空白等待。多數正式上線的系統會在兩者之間取得折衷。", + "note": "Multi-sentence paragraph; stresses streaming TTFT and long-output behaviour." + }, + { + "id": "tricky-06", + "domain": "tricky", + "source": "Break a leg tonight! You've rehearsed this a hundred times, so trust yourself and enjoy the stage.", + "reference": "今晚祝你演出順利！這齣戲你都排練一百次了，相信自己，好好享受舞台吧。", + "note": "Idiom 'break a leg' must not become 斷腿; encouragement register." + } +] diff --git a/eval/dataset/enrich-inputs.json b/eval/dataset/enrich-inputs.json new file mode 100644 index 0000000..47664c2 --- /dev/null +++ b/eval/dataset/enrich-inputs.json @@ -0,0 +1,114 @@ +[ + { + "id": "mdn-fetch", + "url": "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API", + "pageTitle": "Fetch API - Web APIs | MDN", + "lang": "en", + "text": "The Fetch API provides an interface for fetching resources (including across the network). It is a more powerful and flexible replacement for XMLHttpRequest. The fetch() method takes one mandatory argument, the path to the resource you want to fetch, and returns a Promise that resolves to the Response to that request — as soon as the server responds with headers — even if the server response is an HTTP error status. You can also optionally pass in an init options object as the second argument. Once a Response is retrieved, there are a number of methods available to define what the body content is and how it should be handled." + }, + { + "id": "arxiv-attention", + "url": "https://arxiv.org/abs/1706.03762", + "pageTitle": "Attention Is All You Need", + "lang": "en", + "text": "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train." + }, + { + "id": "news-chipact", + "url": "https://example-news.com/semiconductor-subsidies", + "pageTitle": "Government unveils new semiconductor subsidies", + "lang": "en", + "text": "The government announced a fresh round of subsidies for domestic semiconductor manufacturing on Tuesday, earmarking twelve billion dollars for new fabrication plants and workforce training programs. Officials framed the package as a national-security measure, arguing that dependence on overseas chip production leaves critical industries exposed to supply-chain shocks. Industry groups welcomed the funding but cautioned that permitting delays, not capital, remain the biggest obstacle to breaking ground on new fabs." + }, + { + "id": "blog-burnout", + "url": "https://example-blog.dev/on-burnout", + "pageTitle": "What I learned from six months of burnout", + "lang": "en", + "text": "The strangest part of burnout was that it did not feel like exhaustion at first. It felt like indifference. Code reviews I used to enjoy became chores, and side projects sat untouched for weeks. What finally helped was not a vacation — it was renegotiating what I was responsible for, saying no to two recurring meetings, and rediscovering a hobby that had nothing to do with computers. If you are waiting for a dramatic collapse before taking burnout seriously, you are waiting too long." + }, + { + "id": "wiki-photosynthesis", + "url": "https://en.wikipedia.org/wiki/Photosynthesis", + "pageTitle": "Photosynthesis - Wikipedia", + "lang": "en", + "text": "Photosynthesis is a system of biological processes by which photosynthetic organisms, such as most plants, algae, and cyanobacteria, convert light energy, typically from sunlight, into the chemical energy necessary to fuel their metabolism. The term usually refers to oxygenic photosynthesis, where oxygen is produced as a byproduct and some of the chemical energy produced is stored in energy-rich organic compounds such as sugars, which are synthesized from carbon dioxide and water." + }, + { + "id": "docs-docker", + "url": "https://docs.docker.com/get-started/overview/", + "pageTitle": "Docker overview", + "lang": "en", + "text": "Docker is an open platform for developing, shipping, and running applications. Docker enables you to separate your applications from your infrastructure so you can deliver software quickly. With Docker, you can manage your infrastructure in the same ways you manage your applications. Docker provides the ability to package and run an application in a loosely isolated environment called a container. The isolation and security lets you run many containers simultaneously on a given host." + }, + { + "id": "forum-keyboard", + "url": "https://example-forum.net/t/mechanical-keyboards", + "pageTitle": "Are mechanical keyboards worth it?", + "lang": "en", + "text": "Honestly after three years on a mechanical board I can't go back. It's not about typing speed — mine made me maybe 2% faster at best. It's that typing stopped being something I noticed. Get hot-swap sockets for your first board so you can try different switches without soldering, ignore anyone who tells you there's one correct switch, and budget for a decent keycap set because the stock ones on cheap boards feel like wet cardboard." + }, + { + "id": "readme-sqlite", + "url": "https://github.com/example/sqlite-backup", + "pageTitle": "sqlite-backup — streaming online backups for SQLite", + "lang": "en", + "text": "sqlite-backup performs online backups of live SQLite databases without blocking writers. It uses the SQLite Online Backup API to copy pages incrementally, retries automatically when the source database is busy, and verifies the resulting snapshot with a full integrity check. Backups can be streamed to local disk, S3-compatible object storage, or any writable stream. Requires Node 18+. Install with npm install sqlite-backup, then run npx sqlite-backup ./data.db ./snapshots/." + }, + { + "id": "paper-sleep", + "url": "https://example-journal.org/sleep-memory", + "pageTitle": "Sleep-dependent memory consolidation", + "lang": "en", + "text": "A growing body of evidence indicates that sleep plays a causal role in memory consolidation. During slow-wave sleep, hippocampal replay of recently encoded experiences coordinates with neocortical slow oscillations and thalamocortical spindles, a dialogue thought to transfer labile memory traces into stable cortical networks. Targeted memory reactivation experiments, in which learning-associated cues are re-presented during sleep, reliably bias which memories are strengthened, providing causal evidence for the replay hypothesis." + }, + { + "id": "tutorial-git", + "url": "https://example-tutorials.io/git-rebase", + "pageTitle": "A practical guide to git rebase", + "lang": "en", + "text": "Rebase rewrites history by replaying your commits on top of a new base commit. Used well, it produces a linear, readable history; used carelessly, it rewrites commits your teammates already depend on. The golden rule: never rebase commits that have been pushed to a shared branch. For local cleanup before opening a pull request, interactive rebase lets you squash fixup commits, reword messages, and reorder changes so reviewers see a coherent story instead of your actual chaotic process." + }, + { + "id": "zh-news-rain", + "url": "https://example-news.tw/typhoon-rain", + "pageTitle": "颱風外圍環流影響 北部山區豪雨特報", + "lang": "zh-TW", + "text": "受到颱風外圍環流影響，氣象署今日針對北部山區發布豪雨特報，提醒民眾防範坍方與落石。氣象署指出，這波降雨預計持續到週五清晨，山區累積雨量可能達三百毫米，河川上游地區民眾應避免前往溪邊活動。市政府已預先開設應變中心，並要求各區公所檢查抽水站與側溝，避免低窪地區重演積水災情。" + }, + { + "id": "zh-tech-battery", + "url": "https://example-tech.tw/solid-state-battery", + "pageTitle": "固態電池量產進度", + "lang": "zh-TW", + "text": "固態電池被視為下一代電動車的關鍵技術，以固態電解質取代傳統鋰電池的液態電解液，理論上能同時提升能量密度與安全性。不過量產仍面臨兩大瓶頸：一是固態電解質與電極介面的阻抗問題，二是製程良率遠低於現有產線。多家廠商宣稱將在二〇二七年前後啟動小規模量產，但分析師普遍認為，成本要降到與液態鋰電池相當，至少還需要十年。" + }, + { + "id": "zh-essay-reading", + "url": "https://example-essays.tw/slow-reading", + "pageTitle": "慢讀的必要", + "lang": "zh-TW", + "text": "資訊爆炸的年代，我們習慣用瀏覽取代閱讀，用摘要取代思考。慢讀不是懷舊，而是一種抵抗：抵抗演算法替你決定什麼值得注意，抵抗把知識壓縮成三行重點的誘惑。真正改變一個人的書，往往需要在某一頁停下來，反覆讀同一段話，讓它跟你既有的經驗互相摩擦。這種摩擦無法加速，也不該加速。" + }, + { + "id": "zh-recipe-braise", + "url": "https://example-food.tw/braised-pork", + "pageTitle": "家常滷肉的三個關鍵", + "lang": "zh-TW", + "text": "滷肉要好吃，第一是選肉，帶皮五花切成兩公分見方，肥瘦相間才有口感；第二是炒糖色，冰糖小火炒到琥珀色再下肉，顏色自然紅亮，不必靠老抽；第三是火候，大火煮滾後轉小火慢滷四十分鐘，關火再燜二十分鐘讓肉吸回湯汁。醬油選純釀造的，加一點米酒和青蔥結，不要放八角以外太多香料，家常味重點在肉香不在藥膳味。" + }, + { + "id": "mixed-api-error", + "url": "https://example-status.io/incident-2077", + "pageTitle": "Incident report: elevated API error rates", + "lang": "mixed", + "text": "本次事故起因於一次資料庫連線池設定變更。Between 14:02 and 14:47 UTC, the API returned elevated 503 rates (peak 18%) because the new pool size exceeded the database's max_connections limit, causing connection storms on failover. 我們已將連線池上限調回安全值，並新增部署前的組態檢查。Action items include adding a canary stage for configuration-only changes and alerting on connection-refused errors from the database, not just on HTTP error rates." + }, + { + "id": "mixed-release-notes", + "url": "https://github.com/example/app/releases/v3.2.0", + "pageTitle": "v3.2.0 release notes", + "lang": "mixed", + "text": "v3.2.0 主要更新：新增離線模式，文件在無網路時仍可編輯，重新連線後自動同步。Breaking change: the plugins API now requires an explicit permissions manifest; plugins without one will fail to load with error PLUGIN_NO_MANIFEST. 效能方面，大型文件的開啟速度提升約 40%，記憶體用量降低 25%。See the migration guide for details on updating existing plugins before upgrading." + } +] diff --git a/eval/judge.ts b/eval/judge.ts index 43dcce3..2163be3 100644 --- a/eval/judge.ts +++ b/eval/judge.ts @@ -29,7 +29,7 @@ interface Score { const baseUrl = process.env.OLLAMA_URL ?? 'http://localhost:11434'; const endpoint = `${baseUrl.replace(/\/+$/, '')}/v1/chat/completions`; -const judgeModel = process.argv[2] ?? 'qwen2.5'; +const judgeModel = process.argv[2] ?? 'qwen3.5:latest'; const here = dirname(fileURLToPath(import.meta.url)); const fixtures = JSON.parse( diff --git a/eval/structured/run.ts b/eval/structured/run.ts new file mode 100644 index 0000000..86bca45 --- /dev/null +++ b/eval/structured/run.ts @@ -0,0 +1,351 @@ +/** + * Structured-output study — can schema-constrained decoding replace (or must + * it complement) robust parsing for small-model enrichment? + * + * Conditions per model × input: + * prompt — the shipped path: `buildEnrichMessages` prompt rules only + * (native `/api/chat`, `think: false`, temperature 0), the model + * free to misbehave. + * schema — same request, decoding constrained to the EnrichResult JSON + * schema via the native `format` parameter. + * + * Each raw reply is then scored three ways at report time (no extra calls): + * naive parse — strict JSON.parse of the whole reply. + * robust parse — the shipped `parseEnrichResponse` salvage parser. + * field quality — title/summary/tags all present after cleaning. + * plus a failure-shape taxonomy label (`taxonomy.ts`). + * + * Requires local Ollama; NOT in CI. Checkpoints after every cell: + * pnpm eval:structured [-- --models a,b] [--report-only] + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { + buildEnrichMessages, + parseEnrichResponse, + ENRICH_SCHEMA, +} from '../../src/core/enrich'; +import { classifyReply, REPLY_SHAPES, type ReplyShape } from './taxonomy'; + +const TARGET_LANG = 'Traditional Chinese'; +const SEED = 42; +const REQUEST_TIMEOUT_MS = 300_000; + +const DEFAULT_MODELS = [ + 'qwen3.5:latest', + 'qwen3:latest', + 'llama3.1:latest', + 'deepseek-r1:8b', +]; + +type GenCondition = 'prompt' | 'schema'; +const GEN_CONDITIONS: GenCondition[] = ['prompt', 'schema']; + +interface EnrichInput { + id: string; + url: string; + pageTitle: string; + lang: string; + text: string; +} + +interface CellRecord { + model: string; + condition: GenCondition; + inputId: string; + raw: string; + totalMs: number; + error?: string; +} + +interface Checkpoint { + meta: { + baseUrl: string; + targetLang: string; + seed: number; + startedAt: string; + }; + cells: Record; +} + +const here = dirname(fileURLToPath(import.meta.url)); +const resultsDir = join(here, '..', 'results'); +const checkpointPath = join(resultsDir, 'structured-raw.json'); +const reportPath = join(here, '..', 'STRUCTURED-RESULTS.md'); + +const baseUrl = (process.env.OLLAMA_URL ?? 'http://localhost:11434').replace( + /\/+$/, + '', +); + +function argList(flag: string): string[] | null { + const index = process.argv.indexOf(flag); + const value = index < 0 ? undefined : process.argv[index + 1]; + if (!value) return null; + return value.split(',').map((s) => s.trim()); +} +const onlyModels = argList('--models'); +const reportOnly = process.argv.includes('--report-only'); + +const inputs = JSON.parse( + readFileSync(join(here, '..', 'dataset', 'enrich-inputs.json'), 'utf8'), +) as EnrichInput[]; +const models = (onlyModels ?? DEFAULT_MODELS).slice(); + +const cellKey = (model: string, condition: string, inputId: string) => + `${model}|${condition}|${inputId}`; + +function loadCheckpoint(): Checkpoint { + try { + return JSON.parse(readFileSync(checkpointPath, 'utf8')) as Checkpoint; + } catch { + return { + meta: { + baseUrl, + targetLang: TARGET_LANG, + seed: SEED, + startedAt: new Date().toISOString(), + }, + cells: {}, + }; + } +} +function saveCheckpoint(checkpoint: Checkpoint): void { + mkdirSync(resultsDir, { recursive: true }); + writeFileSync(checkpointPath, JSON.stringify(checkpoint, null, 1), 'utf8'); +} + +// --- generation --------------------------------------------------------------- + +async function generate( + model: string, + condition: GenCondition, + input: EnrichInput, +): Promise { + const record: CellRecord = { + model, + condition, + inputId: input.id, + raw: '', + totalMs: 0, + }; + const body: Record = { + model, + messages: buildEnrichMessages(input.text, TARGET_LANG), + stream: false, + think: false, + options: { temperature: 0, seed: SEED }, + }; + if (condition === 'schema') body.format = ENRICH_SCHEMA; + + const t0 = performance.now(); + try { + const response = await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = (await response.json()) as { message?: { content?: string } }; + record.raw = data.message?.content ?? ''; + } catch (error) { + record.error = error instanceof Error ? error.message : String(error); + } + record.totalMs = performance.now() - t0; + return record; +} + +async function warmup(model: string): Promise { + try { + await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: 'Hi' }], + stream: false, + think: false, + options: { num_predict: 1 }, + }), + }); + } catch { + /* best-effort */ + } +} + +// --- scoring / report ----------------------------------------------------------- + +interface ParseOutcome { + naive: boolean; + robust: boolean; + allFields: boolean; + shape: ReplyShape; +} + +function scoreReply(raw: string): ParseOutcome { + let naive = false; + try { + const parsed: unknown = JSON.parse(raw.trim()); + naive = + typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed); + } catch { + naive = false; + } + const robustResult = parseEnrichResponse(raw); + return { + naive, + robust: robustResult !== null, + allFields: Boolean( + robustResult?.title && robustResult.summary && robustResult.tags?.length, + ), + shape: classifyReply(raw), + }; +} + +function pct(count: number, total: number): string { + return total === 0 ? '—' : `${((count / total) * 100).toFixed(1)}%`; +} +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor((sorted.length - 1) / 2)] ?? null; +} + +function writeReport(checkpoint: Checkpoint): void { + const lines: string[] = []; + const all = Object.values(checkpoint.cells).filter((c) => + models.includes(c.model), + ); + + lines.push('# OpenRead — Structured-Output Study'); + lines.push(''); + lines.push( + `Small-model enrichment replies over **${inputs.length}** realistic capture ` + + `excerpts × **${models.length}** local models, generated once per condition ` + + '(temperature 0, seed 42) and scored offline. `prompt` = the shipped ' + + 'prompt-rules-only path; `schema` = decoding constrained to the ' + + 'EnrichResult JSON schema. Regenerate with `pnpm eval:structured`.', + ); + lines.push(''); + + lines.push('## Headline — usable metadata rate by strategy'); + lines.push(''); + lines.push('| Generation | Parse | Usable rate | All 3 fields |'); + lines.push('| --- | --- | --- | --- |'); + for (const condition of GEN_CONDITIONS) { + const ok = all.filter((c) => c.condition === condition && !c.error); + const scores = ok.map((c) => scoreReply(c.raw)); + const naive = scores.filter((s) => s.naive).length; + const robust = scores.filter((s) => s.robust).length; + const fields = scores.filter((s) => s.allFields).length; + lines.push( + `| ${condition} | naive \`JSON.parse\` | ${pct(naive, scores.length)} | — |`, + ); + lines.push( + `| ${condition} | robust \`parseEnrichResponse\` | ${pct(robust, scores.length)} | ${pct(fields, scores.length)} |`, + ); + } + lines.push(''); + + lines.push('## Per model'); + lines.push(''); + lines.push( + '| Model | Gen | Naive parse | Robust parse | All fields | Median ms | Errors |', + ); + lines.push('| --- | --- | --- | --- | --- | --- | --- |'); + for (const model of models) { + for (const condition of GEN_CONDITIONS) { + const cells = all.filter( + (c) => c.model === model && c.condition === condition, + ); + if (cells.length === 0) continue; + const ok = cells.filter((c) => !c.error); + const scores = ok.map((c) => scoreReply(c.raw)); + const med = median(ok.map((c) => c.totalMs)); + lines.push( + `| ${model} | ${condition} ` + + `| ${pct(scores.filter((s) => s.naive).length, scores.length)} ` + + `| ${pct(scores.filter((s) => s.robust).length, scores.length)} ` + + `| ${pct(scores.filter((s) => s.allFields).length, scores.length)} ` + + `| ${med === null ? '—' : Math.round(med)} ` + + `| ${cells.length - ok.length}/${cells.length} |`, + ); + } + } + lines.push(''); + + lines.push('## Failure shapes — unconstrained (`prompt`) replies'); + lines.push(''); + lines.push(`| Model | ${REPLY_SHAPES.join(' | ')} |`); + lines.push(`| --- |${' --- |'.repeat(REPLY_SHAPES.length)}`); + for (const model of models) { + const ok = all.filter( + (c) => c.model === model && c.condition === 'prompt' && !c.error, + ); + const counts = new Map(); + for (const c of ok) { + const shape = classifyReply(c.raw); + counts.set(shape, (counts.get(shape) ?? 0) + 1); + } + lines.push( + `| ${model} | ${REPLY_SHAPES.map((s) => counts.get(s) ?? 0).join(' | ')} |`, + ); + } + lines.push(''); + + lines.push( + '_Both conditions run on the native `/api/chat` endpoint with `think: false` ' + + '(the shipped client path); `schema` adds the `format` parameter. Latency ' + + 'medians include any hidden reasoning time._', + ); + lines.push(''); + + const report = lines.join('\n'); + writeFileSync(reportPath, report + '\n', 'utf8'); + console.log(report); +} + +// --- main ----------------------------------------------------------------------- + +async function main(): Promise { + const checkpoint = loadCheckpoint(); + + if (!reportOnly) { + const total = models.length * GEN_CONDITIONS.length * inputs.length; + let done = 0; + for (const model of models) { + const pending = GEN_CONDITIONS.some((condition) => + inputs.some((input) => { + const cell = checkpoint.cells[cellKey(model, condition, input.id)]; + return !cell || cell.error; + }), + ); + if (pending) await warmup(model); + for (const condition of GEN_CONDITIONS) { + for (const input of inputs) { + const key = cellKey(model, condition, input.id); + done++; + const existing = checkpoint.cells[key]; + if (existing && !existing.error) continue; + process.stdout.write(`[${done}/${total}] ${key} ... `); + const record = await generate(model, condition, input); + checkpoint.cells[key] = record; + saveCheckpoint(checkpoint); + console.log( + record.error + ? `ERROR ${record.error}` + : `${Math.round(record.totalMs)}ms ${classifyReply(record.raw)}`, + ); + } + } + } + } + + writeReport(checkpoint); +} + +void main(); diff --git a/eval/structured/taxonomy.test.ts b/eval/structured/taxonomy.test.ts new file mode 100644 index 0000000..9eb9bfb --- /dev/null +++ b/eval/structured/taxonomy.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest'; +import { classifyReply } from './taxonomy'; + +const OBJ = '{"title": "固態電池", "summary": "量產瓶頸與時程。", "tags": ["電池"]}'; + +describe('classifyReply', () => { + it('classifies clean JSON', () => { + expect(classifyReply(OBJ)).toBe('clean-json'); + expect(classifyReply(` ${OBJ}\n`)).toBe('clean-json'); + }); + + it('classifies fenced JSON', () => { + expect(classifyReply('```json\n' + OBJ + '\n```')).toBe('fenced-json'); + expect(classifyReply('```\n' + OBJ + '\n```')).toBe('fenced-json'); + }); + + it('classifies JSON after visible thinking', () => { + expect( + classifyReply(`The user wants metadata.\n${OBJ}`), + ).toBe('thinking-then-json'); + }); + + it('classifies JSON wrapped in prose', () => { + expect(classifyReply(`Here is the metadata you asked for:\n${OBJ}`)).toBe( + 'json-with-prose', + ); + expect(classifyReply(`${OBJ}\nLet me know if you need anything else!`)).toBe( + 'json-with-prose', + ); + }); + + it('classifies truncated JSON', () => { + expect(classifyReply('{"title": "固態電池", "summary": "量產')).toBe( + 'truncated-json', + ); + // Balanced braces but unparseable content is still a truncation-class defect. + expect(classifyReply('{"title": broken}')).toBe('truncated-json'); + }); + + it('classifies pure prose and refusals as no-json', () => { + expect(classifyReply('I cannot label this text.')).toBe('no-json'); + expect(classifyReply('固態電池是下一代電動車關鍵技術。')).toBe('no-json'); + }); + + it('classifies empty and whitespace-only replies', () => { + expect(classifyReply('')).toBe('empty'); + expect(classifyReply(' \n ')).toBe('empty'); + }); + + it('prefers fence over prose when both would match', () => { + expect( + classifyReply('Sure! Here you go:\n```json\n' + OBJ + '\n```\nEnjoy!'), + ).toBe('fenced-json'); + }); +}); diff --git a/eval/structured/taxonomy.ts b/eval/structured/taxonomy.ts new file mode 100644 index 0000000..5de98ad --- /dev/null +++ b/eval/structured/taxonomy.ts @@ -0,0 +1,74 @@ +/** + * Failure taxonomy for small-model structured-output replies. + * + * Classifies a raw enrichment reply into the shape it arrived in — the study's + * unit of analysis. Categories are mutually exclusive and checked in priority + * order; each maps to a recovery strategy (or the lack of one): + * + * clean-json parses as-is — nothing to recover. + * fenced-json valid JSON inside a ``` fence — markdown habit. + * thinking-then-json valid JSON after visible chain-of-thought. + * json-with-prose valid JSON wrapped in preamble/trailing prose. + * truncated-json an object starts but never closes / cannot parse — + * usually context or token-limit exhaustion. + * no-json prose or a refusal with no object at all. + * empty nothing usable was emitted. + * + * Pure and dependency-free so it is unit-testable and CI-safe. + */ + +export type ReplyShape = + | 'clean-json' + | 'fenced-json' + | 'thinking-then-json' + | 'json-with-prose' + | 'truncated-json' + | 'no-json' + | 'empty'; + +export const REPLY_SHAPES: readonly ReplyShape[] = [ + 'clean-json', + 'fenced-json', + 'thinking-then-json', + 'json-with-prose', + 'truncated-json', + 'no-json', + 'empty', +]; + +function parsesAsObject(text: string): boolean { + try { + const parsed: unknown = JSON.parse(text); + return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed); + } catch { + return false; + } +} + +/** First `{` … last `}` span, the same recovery window the parser uses. */ +function objectSpan(text: string): string | null { + const start = text.indexOf('{'); + const end = text.lastIndexOf('}'); + if (start < 0 || end <= start) return null; + return text.slice(start, end + 1); +} + +export function classifyReply(reply: string): ReplyShape { + const trimmed = reply.trim(); + if (!trimmed) return 'empty'; + if (parsesAsObject(trimmed)) return 'clean-json'; + + const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(trimmed)?.[1]; + if (fenced !== undefined && parsesAsObject(fenced.trim())) { + return 'fenced-json'; + } + + const span = objectSpan(trimmed); + if (span && parsesAsObject(span)) { + const thinking = /^/i.test(trimmed) || /<\/think>/i.test(trimmed); + return thinking ? 'thinking-then-json' : 'json-with-prose'; + } + + if (trimmed.includes('{')) return 'truncated-json'; + return 'no-json'; +} diff --git a/package.json b/package.json index d22eb10..5469896 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openread", - "version": "2.1.0", + "version": "2.2.0", "description": "Local-first, evaluation-driven LLM translation for web & PDF via Ollama — a case study in making streaming LLM output reliable.", "type": "module", "license": "MIT", @@ -21,6 +21,9 @@ "format": "prettier --write .", "eval": "tsx eval/run.ts", "eval:capture": "tsx eval/capture-run.ts", + "eval:structured": "tsx eval/structured/run.ts", + "bench": "tsx eval/bench/run.ts", + "bench:agreement": "tsx eval/bench/agreement.ts", "postinstall": "wxt prepare" }, "devDependencies": { diff --git a/src/api/ollama.test.ts b/src/api/ollama.test.ts index ef8df92..0adb5a7 100644 --- a/src/api/ollama.test.ts +++ b/src/api/ollama.test.ts @@ -1,43 +1,63 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; -import { extractDelta, translateStream } from './ollama'; +import { extractChunk, translateStream } from './ollama'; + +describe('extractChunk', () => { + it('extracts content from a native NDJSON line', () => { + const line = '{"message":{"role":"assistant","content":"你好"},"done":false}'; + expect(extractChunk(line)).toEqual({ + content: '你好', + thinking: '', + done: false, + evalCount: undefined, + }); + }); -describe('extractDelta', () => { - it('extracts delta content from a data line', () => { - const line = 'data: {"choices":[{"delta":{"content":"你好"}}]}'; - expect(extractDelta(line)).toBe('你好'); + it('keeps thinking separate from content', () => { + const line = + '{"message":{"role":"assistant","content":"","thinking":"Let me see"},"done":false}'; + expect(extractChunk(line)).toEqual({ + content: '', + thinking: 'Let me see', + done: false, + evalCount: undefined, + }); }); - it('returns null for the [DONE] sentinel and keep-alives', () => { - expect(extractDelta('data: [DONE]')).toBeNull(); - expect(extractDelta(': keep-alive')).toBeNull(); - expect(extractDelta('')).toBeNull(); + it('surfaces the token count on the done chunk', () => { + const line = + '{"message":{"role":"assistant","content":""},"done":true,"eval_count":42}'; + expect(extractChunk(line)).toEqual({ + content: '', + thinking: '', + done: true, + evalCount: 42, + }); }); - it('returns null for a role-only opening delta', () => { - expect( - extractDelta('data: {"choices":[{"delta":{"role":"assistant"}}]}'), - ).toBeNull(); + it('returns null for blank lines', () => { + expect(extractChunk('')).toBeNull(); + expect(extractChunk(' ')).toBeNull(); }); it('swallows malformed JSON rather than throwing', () => { - expect(extractDelta('data: {not json')).toBeNull(); + expect(extractChunk('{not json')).toBeNull(); }); }); -/** Build a Response whose body streams the given SSE text as one or more chunks. */ -function sseResponse(sseChunks: string[]): Response { +/** Build a Response whose body streams the given NDJSON text chunks. */ +function ndjsonResponse(chunks: string[]): Response { const encoder = new TextEncoder(); const body = new ReadableStream({ start(controller) { - for (const chunk of sseChunks) controller.enqueue(encoder.encode(chunk)); + for (const chunk of chunks) controller.enqueue(encoder.encode(chunk)); controller.close(); }, }); return new Response(body, { status: 200 }); } -function sseLine(content: string): string { - return `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}\n`; +function ndjsonLine(content: string, done = false): string { + return `${JSON.stringify({ message: { role: 'assistant', content }, done })}\n`; } const BASE_URL = 'http://localhost:11434'; @@ -45,17 +65,17 @@ const BASE_URL = 'http://localhost:11434'; describe('translateStream', () => { afterEach(() => vi.unstubAllGlobals()); - it('assembles cleaned chunks from an SSE stream', async () => { + it('assembles cleaned chunks from an NDJSON stream', async () => { vi.stubGlobal( 'fetch', vi .fn() .mockResolvedValue( - sseResponse([ - sseLine('你好'), - sseLine('，世界'), - sseLine('！這是測試'), - 'data: [DONE]\n', + ndjsonResponse([ + ndjsonLine('你好'), + ndjsonLine('，世界'), + ndjsonLine('！這是測試'), + ndjsonLine('', true), ]), ), ); @@ -72,16 +92,70 @@ describe('translateStream', () => { expect(chunks.join('')).toBe('你好，世界！這是測試'); }); + it('requests the native endpoint with thinking disabled', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(ndjsonResponse([ndjsonLine('你好', true)])); + vi.stubGlobal('fetch', fetchMock); + + await translateStream({ + text: 'Hello', + baseUrl: BASE_URL, + model: 'qwen3:latest', + targetLang: 'Traditional Chinese', + onChunk: () => {}, + }); + + expect(fetchMock).toHaveBeenCalledWith( + `${BASE_URL}/api/chat`, + expect.objectContaining({ method: 'POST' }), + ); + const init = fetchMock.mock.calls[0]?.[1] as RequestInit | undefined; + const body = JSON.parse((init?.body as string) ?? '{}') as { + think?: boolean; + stream?: boolean; + }; + expect(body.think).toBe(false); + expect(body.stream).toBe(true); + }); + + it('never emits thinking-only chunks', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + ndjsonResponse([ + `${JSON.stringify({ + message: { role: 'assistant', content: '', thinking: '推理中……' }, + done: false, + })}\n`, + ndjsonLine('你好世界'), + ndjsonLine('', true), + ]), + ), + ); + + const chunks: string[] = []; + await translateStream({ + text: 'Hello world', + baseUrl: BASE_URL, + model: 'deepseek-r1:8b', + targetLang: 'Traditional Chinese', + onChunk: (c) => chunks.push(c), + }); + + expect(chunks.join('')).toBe('你好世界'); + }); + it('strips a leading preamble emitted over the stream', async () => { vi.stubGlobal( 'fetch', vi .fn() .mockResolvedValue( - sseResponse([ - sseLine('Sure, here is the translation:\n'), - sseLine('你好世界'), - 'data: [DONE]\n', + ndjsonResponse([ + ndjsonLine('Sure, here is the translation:\n'), + ndjsonLine('你好世界'), + ndjsonLine('', true), ]), ), ); @@ -104,7 +178,7 @@ describe('translateStream', () => { vi .fn() .mockResolvedValue( - sseResponse([sseLine('鼠标和软件都很好用啊'), 'data: [DONE]\n']), + ndjsonResponse([ndjsonLine('鼠标和软件都很好用啊'), ndjsonLine('', true)]), ), ); @@ -120,16 +194,13 @@ describe('translateStream', () => { expect(chunks.join('')).toBe('滑鼠和軟體都很好用啊'); }); - it('throws on a non-OK response', async () => { + it('throws on a non-OK response with a native string error', async () => { vi.stubGlobal( 'fetch', vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ error: { message: 'model not found' } }), - { - status: 404, - }, - ), + new Response(JSON.stringify({ error: 'model not found' }), { + status: 404, + }), ), ); diff --git a/src/api/ollama.ts b/src/api/ollama.ts index c06559c..f946461 100644 --- a/src/api/ollama.ts +++ b/src/api/ollama.ts @@ -1,31 +1,43 @@ /** - * Ollama chat-completions client. Ollama exposes an OpenAI-compatible endpoint - * (`/v1/chat/completions`) with identical SSE streaming, so the streaming - * machinery here is standard — the only Ollama specifics are the local base URL - * and the absence of any auth header (inference runs on the user's machine). + * Ollama chat client, on the native `/api/chat` endpoint. * - * Two entry points: - * - translateStream: SSE streaming for the interactive path. - * - translateText: single non-streamed call, used for retry fallbacks. + * Native rather than the OpenAI-compat `/v1` endpoint for one hard reason, + * found by the benchmark harness: on reasoning models (qwen3 family, + * deepseek-r1) the compat endpoint routes chain-of-thought into a separate + * `reasoning` field and — with thinking enabled by default — can burn the + * entire generation thinking while `content` stays empty, so the extension + * rendered nothing. The native endpoint accepts `think: false` (honoured by + * hybrid thinkers, tolerated as a no-op by non-thinkers, and by models that + * cannot stop thinking — e.g. deepseek-r1 — it still keeps the thinking out + * of `content`). Requires Ollama ≥ 0.9. * - * All prompt building and output cleanup lives in the pure `core` modules; this - * file only owns the network I/O. Cancellation is per-request via an injected - * AbortSignal — no shared mutable controller, so concurrent callers never race. + * Streaming is NDJSON: one JSON object per line, `done: true` on the final + * chunk (which also carries token counts). `extractChunk` parses one line and + * is pure, so the stream parser is unit-testable without a socket. + * + * All prompt building and output cleanup lives in the pure `core` modules; + * this file only owns the network I/O. Cancellation is per-request via an + * injected AbortSignal — no shared mutable controller, so concurrent callers + * never race. */ import { generateSystemPrompt, getFewShotMessages } from '../core/prompt'; import { cleanTranslationOutput } from '../core/sanitize'; import { toTraditionalTW } from '../core/zh-convert'; import { StreamAssembler } from '../core/stream'; -import { buildEnrichMessages, parseEnrichResponse } from '../core/enrich'; +import { + buildEnrichMessages, + parseEnrichResponse, + ENRICH_SCHEMA, +} from '../core/enrich'; import type { ChatMessage, TranslationContext, EnrichResult, } from '../core/types'; -/** Build the OpenAI-compatible chat endpoint for an Ollama server base URL. */ +/** Build the native chat endpoint for an Ollama server base URL. */ function endpoint(baseUrl: string): string { - return `${baseUrl.replace(/\/+$/, '')}/v1/chat/completions`; + return `${baseUrl.replace(/\/+$/, '')}/api/chat`; } function wantsTraditional(targetLang: string): boolean { @@ -46,7 +58,8 @@ function buildUserContent(text: string, context?: TranslationContext): string { ); } -function buildMessages( +/** Exported so the benchmark harness scores the exact shipped prompt. */ +export function buildMessages( text: string, targetLang: string, context?: TranslationContext, @@ -61,27 +74,44 @@ function buildMessages( async function assertOk(response: Response): Promise { if (response.ok) return; const data: unknown = await response.json().catch(() => ({})); + const error = (data as { error?: string | { message?: string } })?.error; const message = - (data as { error?: { message?: string } })?.error?.message ?? - JSON.stringify(data); + typeof error === 'string' + ? error + : (error?.message ?? JSON.stringify(data)); throw new Error(`Ollama ${response.status}: ${message}`); } +export interface NativeChunk { + /** Assistant-content delta in this chunk; '' for thinking-only chunks. */ + content: string; + /** Chain-of-thought delta, kept separate so it never reaches the UI. */ + thinking: string; + done: boolean; + /** Generated-token count, present on the `done: true` chunk. */ + evalCount?: number; +} + /** - * Parse one SSE line into its delta content. Returns null for keep-alives, - * the `[DONE]` sentinel, non-`data:` lines, and unparseable payloads. Pure, so - * the stream parser is unit-testable without a socket. + * Parse one NDJSON stream line from `/api/chat`. Returns null for blank lines + * and unparseable payloads. Pure, so the stream parser is unit-testable. */ -export function extractDelta(line: string): string | null { +export function extractChunk(line: string): NativeChunk | null { const trimmed = line.trim(); - if (!trimmed.startsWith('data:')) return null; - const payload = trimmed.slice('data:'.length).trim(); - if (payload === '' || payload === '[DONE]') return null; + if (!trimmed) return null; try { - const json = JSON.parse(payload) as { - choices?: Array<{ delta?: { content?: string } }>; + const json = JSON.parse(trimmed) as { + message?: { content?: string; thinking?: string }; + done?: boolean; + eval_count?: number; + }; + if (typeof json !== 'object' || json === null) return null; + return { + content: json.message?.content ?? '', + thinking: json.message?.thinking ?? '', + done: json.done === true, + evalCount: json.eval_count, }; - return json.choices?.[0]?.delta?.content ?? null; } catch { return null; } @@ -113,8 +143,9 @@ export async function translateStream(params: StreamParams): Promise { body: JSON.stringify({ model, messages: buildMessages(text, targetLang, context), - temperature: retryCount > 0 ? 0.7 : 0.3, stream: true, + think: false, + options: { temperature: retryCount > 0 ? 0.7 : 0.3 }, }), }); await assertOk(response); @@ -135,9 +166,9 @@ export async function translateStream(params: StreamParams): Promise { const lines = lineBuffer.split('\n'); lineBuffer = lines.pop() ?? ''; for (const line of lines) { - const delta = extractDelta(line); - if (delta === null) continue; - const emit = assembler.push(delta); + const chunk = extractChunk(line); + if (chunk === null || chunk.content === '') continue; + const emit = assembler.push(chunk.content); if (emit) onChunk(emit); } } @@ -170,15 +201,17 @@ export async function translateText(params: TranslateParams): Promise { body: JSON.stringify({ model, messages: buildMessages(text, targetLang, context), - temperature: 0.3, + stream: false, + think: false, + options: { temperature: 0.3 }, }), }); await assertOk(response); const data = (await response.json()) as { - choices?: Array<{ message?: { content?: string } }>; + message?: { content?: string }; }; - const content = data.choices?.[0]?.message?.content; + const content = data.message?.content; if (!content) throw new Error('Ollama returned an empty message'); const cleaned = cleanTranslationOutput(text, content); @@ -212,15 +245,16 @@ export async function enrichText( body: JSON.stringify({ model, messages: buildEnrichMessages(text, targetLang), - temperature: 0, stream: false, + think: false, + format: ENRICH_SCHEMA, + options: { temperature: 0 }, }), }); if (!response.ok) return null; const data = (await response.json()) as { - choices?: Array<{ message?: { content?: string } }>; + message?: { content?: string }; }; - const content = data.choices?.[0]?.message?.content ?? ''; - return parseEnrichResponse(content); + return parseEnrichResponse(data.message?.content ?? ''); } diff --git a/src/core/enrich.ts b/src/core/enrich.ts index 835fb55..0082a6c 100644 --- a/src/core/enrich.ts +++ b/src/core/enrich.ts @@ -28,6 +28,24 @@ const MAX_TAG_LEN = 40; const MAX_TITLE_LEN = 120; const MAX_SUMMARY_LEN = 400; +/** + * JSON schema for Ollama's constrained decoding (`format` parameter). The + * structured-output study (`pnpm eval:structured`) measured this closing the + * last unreliable tail — deepseek-r1 went 93.3% → 100% usable — at no latency + * cost, so the client sends it on every enrichment request. The tolerant + * parser below still runs afterwards: constrained decoding guarantees shape, + * not content hygiene (length caps, tag normalisation, deduping). + */ +export const ENRICH_SCHEMA = { + type: 'object', + properties: { + title: { type: 'string' }, + summary: { type: 'string' }, + tags: { type: 'array', items: { type: 'string' } }, + }, + required: ['title', 'summary', 'tags'], +} as const; + /** * Build the chat messages for an enrichment request. Title and summary are * requested in `targetLang` so the downstream metadata matches the reader's diff --git a/src/entrypoints/popup/index.html b/src/entrypoints/popup/index.html index b19d022..0867af8 100644 --- a/src/entrypoints/popup/index.html +++ b/src/entrypoints/popup/index.html @@ -102,7 +102,7 @@

OpenRead

- + diff --git a/src/settings.ts b/src/settings.ts index 863b32d..c0573fc 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -21,7 +21,9 @@ export interface Settings { export const DEFAULT_SETTINGS: Settings = { baseUrl: 'http://localhost:11434', - modelId: 'qwen2.5', + // Benchmark-driven default: best quality/latency of the models measured + // in docs/BENCHMARK.md (chrF 46.3, TTFT-UI p50 451 ms on the test rig). + modelId: 'qwen3:latest', targetLang: 'Traditional Chinese', obsidianVault: '', obsidianFolder: 'OpenRead',