Judge calibration: 40 human labels, weighted kappa 0.53 adequacy / 0.27 fluency / 0.21 localization

CHANGELOG.md

@@ -19,7 +19,10 @@ reasoning-model bug.
   TTFT-net/TTFT-UI latency, tokens/s, and a JSON-schema-constrained LLM judge
   → `eval/BENCHMARK-RESULTS.md`
 - Judge calibration workflow (`pnpm bench:agreement`): seeded blind labeling
-  page + Cohen's κ (plain & quadratic-weighted) judge↔human → `eval/AGREEMENT.md`
+  page + Cohen's κ (plain & quadratic-weighted) judge↔human → `eval/AGREEMENT.md`.
+  Measured over 40 human labels: weighted κ 0.526 adequacy (moderate),
+  0.267 fluency / 0.213 localization (weak — judge over-lenient on Taiwan
+  terminology; report reads those axes as upper bounds)
 - Structured-output study (`pnpm eval:structured`): prompt-only vs
   schema-constrained decoding across 4 models × 16 realistic capture excerpts,
   with a failure-shape taxonomy → `eval/STRUCTURED-RESULTS.md`

README.md

@@ -77,8 +77,11 @@ asks the opposite question against live models: **which model should you run,
 and what does each design choice cost?** 27 curated EN→zh-TW fixtures with
 Taiwan-convention references × 4 models × 2 prompt conditions, streamed
 through the exact shipped pipeline and scored with sacrebleu-cross-validated
-chrF, artifact detectors, latency probes, and a schema-constrained LLM judge
-(itself calibrated against human labels — Cohen's κ workflow included).
+chrF, artifact detectors, latency probes, and a schema-constrained LLM judge —
+itself calibrated against 40 blind human labels: quadratic-weighted Cohen's κ
+0.53 on adequacy (moderate — usable), 0.21–0.27 on fluency/localization (weak
+— so quality claims lean on chrF + adequacy, and the judge's localization
+scores are treated as an upper bound; [`eval/AGREEMENT.md`](eval/AGREEMENT.md)).
 
 | Model | chrF ↑ | TTFT-UI p50 | Tokens/s | Verdict |
 | --- | --- | --- | --- | --- |

docs/BENCHMARK.md

@@ -98,6 +98,16 @@ quadratically-weighted Cohen's κ per axis
 ([`eval/bench/kappa.ts`](../eval/bench/kappa.ts), verified against textbook
 cases). Results land in [`eval/AGREEMENT.md`](../eval/AGREEMENT.md).
 
+**Measured (40 blind human labels, 2026-07-10):** quadratic-weighted κ =
+**0.526 adequacy** (moderate — the judge's adequacy rankings are usable),
+**0.267 fluency** and **0.213 localization** (weak). The judge is
+systematically more lenient than the human on Taiwan-localization — it hands
+out 4s and 5s where a Taiwanese reader gives 1s and 2s — so the localization
+column in the judge table should be read as an upper bound, and cross-model
+quality claims in this report lean on chrF + adequacy, not on the judge's
+localization axis. Calibrating the judge didn't just validate it; it told us
+*which parts* of it to trust.
+
 ## 5. Conditions
 
 | Condition | Prompt | Pipeline |
@@ -169,7 +179,9 @@ Run of 2026-07-10 — 216/216 generations, 0 errors, all cells judged:
   are judged on raw output and `engineered` on pipeline output, so the small,
   consistent adequacy dip under `engineered` partly reflects the pipeline's
   clipping, and qwen3.5 grading its own family invites bias — hence the human
-  calibration step (§4).
+  calibration step (§4), which confirmed the judge is only moderately
+  trustworthy on adequacy (weighted κ 0.526) and weak on fluency/localization
+  (0.267 / 0.213).
 <!-- BENCH-SUMMARY:END -->
 
 ## 8. Structured-output study

eval/AGREEMENT.md

@@ -0,0 +1,10 @@
+# OpenRead — Judge ↔ Human Agreement
+
+Cohen's kappa between the LLM judge and **40** human-rated items (1–5 scales). Quadratic weighting is the headline number for ordinal ratings; ≥0.4 = moderate, ≥0.6 = substantial agreement.
+
+| Axis | Raw agreement | Cohen κ | Weighted κ (quadratic) | n |
+| --- | --- | --- | --- | --- |
+| adequacy | 62.5% | 0.339 | 0.526 | 40 |
+| fluency | 60.0% | 0.191 | 0.267 | 40 |
+| localization | 47.5% | 0.104 | 0.213 | 40 |
+

eval/bench/agreement.ts

@@ -93,7 +93,9 @@ function makePage(sampleSize: number): void {
     .map(([key]) => key)
     .sort();
   if (judged.length === 0) {
-    console.error('No judged items in the checkpoint — run `pnpm bench` first.');
+    console.error(
+      'No judged items in the checkpoint — run `pnpm bench` first.',
+    );
     process.exitCode = 1;
     return;
   }
@@ -111,7 +113,11 @@ function makePage(sampleSize: number): void {
   }
   const sampled: string[] = [];
   const cells = [...byCell.values()];
-  for (let round = 0; sampled.length < Math.min(sampleSize, judged.length); round++) {
+  for (
+    let round = 0;
+    sampled.length < Math.min(sampleSize, judged.length);
+    round++
+  ) {
     let advanced = false;
     for (const keys of cells) {
       const key = keys[round];

eval/bench/chrf.test.ts

@@ -1,5 +1,11 @@
 import { describe, expect, it } from 'vitest';
-import { addStats, chrfFromStats, chrfScore, chrfStats, corpusChrf } from './chrf';
+import {
+  addStats,
+  chrfFromStats,
+  chrfScore,
+  chrfStats,
+  corpusChrf,
+} from './chrf';
 
 /**
  * Expected values cross-validated against Python sacrebleu 2.6.0

eval/bench/kappa.ts

@@ -17,7 +17,9 @@ function buildConfusion(
   categories: number[],
 ): number[][] {
   if (a.length !== b.length) {
-    throw new Error(`Rating lists differ in length: ${a.length} vs ${b.length}`);
+    throw new Error(
+      `Rating lists differ in length: ${a.length} vs ${b.length}`,
+    );
   }
   if (a.length === 0) throw new Error('Cannot compute kappa on zero items');
   const index = new Map(categories.map((c, i) => [c, i]));

eval/bench/run.ts

@@ -300,10 +300,7 @@ function judgePrompt(fixture: Fixture, candidate: string): string {
 }
 
 /** Judge one candidate via native /api/chat with a constrained JSON schema. */
-async function judge(
-  fixture: Fixture,
-  candidate: string,
-): Promise<JudgeScore> {
+async function judge(fixture: Fixture, candidate: string): Promise<JudgeScore> {
   const body: Record<string, unknown> = {
     model: judgeModel,
     messages: [{ role: 'user', content: judgePrompt(fixture, candidate) }],
@@ -367,7 +364,11 @@ interface CellAggregate {
   ttftNetP50: number | null;
   ttftUiP50: number | null;
   tokensPerSec: number | null;
-  judgeMeans: { adequacy: number; fluency: number; localization: number } | null;
+  judgeMeans: {
+    adequacy: number;
+    fluency: number;
+    localization: number;
+  } | null;
   judged: number;
 }
 
@@ -417,7 +418,10 @@ function aggregate(checkpoint: Checkpoint): CellAggregate[] {
         const fixture = byFixture.get(r.fixtureId);
         if (!fixture) continue;
         statsRaw = addStats(statsRaw, chrfStats(r.raw, fixture.reference));
-        statsPiped = addStats(statsPiped, chrfStats(r.piped, fixture.reference));
+        statsPiped = addStats(
+          statsPiped,
+          chrfStats(r.piped, fixture.reference),
+        );
         if (hasPreamble(r.raw)) preambleRaw++;
         if (hasPreamble(r.piped)) preamblePiped++;
         if (hasEcho(fixture.source, r.raw)) echoRaw++;
@@ -508,7 +512,9 @@ function writeReport(checkpoint: Checkpoint): void {
 
   lines.push('## Quality — corpus chrF against references');
   lines.push('');
-  lines.push('| Model | Prompt | chrF raw | chrF shipped pipeline | Δ pipeline |');
+  lines.push(
+    '| Model | Prompt | chrF raw | chrF shipped pipeline | Δ pipeline |',
+  );
   lines.push('| --- | --- | --- | --- | --- |');
   for (const c of cells) {
     lines.push(
@@ -563,7 +569,9 @@ function writeReport(checkpoint: Checkpoint): void {
       'for judge calibration against human labels._',
   );
   lines.push('');
-  lines.push('| Model | Prompt | Adequacy | Fluency | TW localization | Judged |');
+  lines.push(
+    '| Model | Prompt | Adequacy | Fluency | TW localization | Judged |',
+  );
   lines.push('| --- | --- | --- | --- | --- | --- |');
   for (const c of cells) {
     lines.push(
@@ -626,9 +634,7 @@ async function main(): Promise<void> {
           done++;
           const existing = checkpoint.generations[key];
           if (existing && !existing.error) continue;
-          process.stdout.write(
-            `[${done}/${totalCells}] ${key} ... `,
-          );
+          process.stdout.write(`[${done}/${totalCells}] ${key} ... `);
           const record = await generate(model, condition, fixture);
           checkpoint.generations[key] = record;
           saveCheckpoint(checkpoint);
@@ -653,7 +659,9 @@ async function main(): Promise<void> {
       for (const record of pending) {
         const key = cellKey(record.model, record.condition, record.fixtureId);
         judgedCount++;
-        process.stdout.write(`[judge ${judgedCount}/${pending.length}] ${key} ... `);
+        process.stdout.write(
+          `[judge ${judgedCount}/${pending.length}] ${key} ... `,
+        );
         try {
           const score = await judge(
             fixtures.find((f) => f.id === record.fixtureId)!,