From 42e83d32e319e1c01506e338188466669eb71e44 Mon Sep 17 00:00:00 2001 From: Farhan Helmy <59960562+farhan-helmy@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:05:11 +0800 Subject: [PATCH] feat: add Malay BFCL benchmark dashboard --- .github/workflows/pages.yml | 55 +++ .gitignore | 1 + README.md | 3 + site/app.js | 208 +++++++++ site/build.py | 143 ++++++ site/favicon.svg | 9 + site/index.html | 175 ++++++++ site/styles.css | 874 ++++++++++++++++++++++++++++++++++++ site/tests/test_browser.py | 92 ++++ site/tests/test_build.py | 137 ++++++ 10 files changed, 1697 insertions(+) create mode 100644 .github/workflows/pages.yml create mode 100644 site/app.js create mode 100644 site/build.py create mode 100644 site/favicon.svg create mode 100644 site/index.html create mode 100644 site/styles.css create mode 100644 site/tests/test_browser.py create mode 100644 site/tests/test_build.py diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..7232176 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,55 @@ +name: Deploy benchmark dashboard + +on: + push: + branches: [main] + paths: + - "site/**" + - "bfcl-malay-bench/results/comparison.json" + - ".github/workflows/pages.yml" + pull_request: + paths: + - "site/**" + - "bfcl-malay-bench/results/comparison.json" + - ".github/workflows/pages.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Test site build + run: python -m unittest discover -s site/tests -v + - name: Validate JavaScript + run: node --check site/app.js + - name: Build site + run: python site/build.py --output _site + - uses: actions/upload-pages-artifact@v3 + with: + path: _site + + deploy: + if: github.event_name != 'pull_request' + needs: build + concurrency: + group: pages + cancel-in-progress: false + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - uses: actions/configure-pages@v5 + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 0a040db..5e81d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ build/ data/ runs/ private/ +_site/ diff --git a/README.md b/README.md index 36797e9..385356e 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ Public evaluation harnesses and reproducible result summaries from [PixelSpaceAI](https://github.com/PixelSpaceAI). +Explore the results on the +[public benchmark dashboard](https://pixelspaceai.github.io/benchmark/). + ## Benchmarks - [Malay BFCL v3 tool-calling benchmark](bfcl-malay-bench/) — evaluate an diff --git a/site/app.js b/site/app.js new file mode 100644 index 0000000..db2408a --- /dev/null +++ b/site/app.js @@ -0,0 +1,208 @@ +const DATA_URL = document.body.dataset.resultsUrl || "./data/comparison.json"; + +const MODEL_COLORS = ["#c8ff61", "#6ee7b7", "#66d8e5", "#ffcb66"]; +const CATEGORY_COPY = { + simple: { + label: "Simple", + title: "Simple tool calls", + description: "One function, one precise set of arguments.", + }, + multiple: { + label: "Multiple", + title: "Multiple choices", + description: "Choose the correct function from several available tools.", + }, + irrelevance: { + label: "Irrelevance", + title: "Tool restraint", + description: "Recognize when no available function should be called.", + }, + chatable: { + label: "Chatable", + title: "Conversation", + description: "Reply naturally when the prompt needs text, not a tool call.", + }, +}; + +const percent = (value) => `${(value * 100).toFixed(2)}%`; +const wholePercent = (value) => `${(value * 100).toFixed(2).replace(/\.00$/, "")}%`; +let categoryAnimationFrame; + +function element(tag, className, text) { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} + +function modelLabel(result) { + const label = element("div", "model-label"); + label.append(element("strong", "", result.model)); + label.append(element("span", "", `Rank ${result.rank}`)); + return label; +} + +function barTrack(value, color, label) { + const track = element("div", "bar-track"); + const fill = element("div", "bar-fill"); + fill.style.setProperty("--bar-color", color); + fill.setAttribute("role", "img"); + fill.setAttribute("aria-label", label); + fill.dataset.width = `${value * 100}%`; + track.append(fill); + return track; +} + +function animateBars(container) { + container.querySelectorAll(".bar-fill").forEach((fill) => { + fill.style.width = fill.dataset.width; + }); +} + +function renderOverall(results) { + const chart = document.querySelector("#overall-chart"); + chart.replaceChildren(); + + results.forEach((result, index) => { + const row = element("div", "leaderboard-row"); + row.append(modelLabel(result)); + row.append( + barTrack( + result.accuracy, + MODEL_COLORS[index % MODEL_COLORS.length], + `${result.model}: ${percent(result.accuracy)} overall accuracy`, + ), + ); + row.append(element("div", "bar-value", percent(result.accuracy))); + chart.append(row); + }); + requestAnimationFrame(() => animateBars(chart)); +} + +function renderCategory(results, colorIndexByModelId, category) { + const copy = CATEGORY_COPY[category]; + document.querySelector("#category-label").textContent = copy.label; + document.querySelector("#category-title").textContent = copy.title; + document.querySelector("#category-description").textContent = copy.description; + + document.querySelectorAll(".category-button").forEach((button) => { + button.setAttribute("aria-pressed", String(button.dataset.category === category)); + }); + + const chart = document.querySelector("#category-chart"); + chart.replaceChildren(); + const categoryResults = [...results].sort( + (first, second) => second.by_category[category].accuracy - first.by_category[category].accuracy, + ); + + categoryResults.forEach((result) => { + const detail = result.by_category[category]; + const colorIndex = colorIndexByModelId.get(result.model_id); + const row = element("div", "category-row"); + row.append(modelLabel(result)); + row.append( + barTrack( + detail.accuracy, + MODEL_COLORS[colorIndex % MODEL_COLORS.length], + `${result.model}: ${percent(detail.accuracy)} in ${copy.label}`, + ), + ); + const score = element("div", "category-detail"); + score.append(element("strong", "", wholePercent(detail.accuracy))); + score.append(element("span", "", `${detail.passed} / ${detail.scored}`)); + row.append(score); + chart.append(row); + }); + cancelAnimationFrame(categoryAnimationFrame); + categoryAnimationFrame = requestAnimationFrame(() => animateBars(chart)); +} + +function renderControls(results, colorIndexByModelId, categories, selectedCategory) { + const controls = document.querySelector("#category-controls"); + controls.replaceChildren(); + categories.forEach((category, index) => { + const button = element("button", "category-button", CATEGORY_COPY[category]?.label ?? category); + button.type = "button"; + button.dataset.category = category; + button.setAttribute("aria-pressed", String(category === selectedCategory)); + button.addEventListener("click", () => { + const url = new URL(window.location); + url.searchParams.set("category", category); + window.history.replaceState({}, "", url); + renderCategory(results, colorIndexByModelId, category); + }); + controls.append(button); + }); +} + +function scoreCell(detail) { + return element("td", "", `${detail.passed}/${detail.scored} · ${wholePercent(detail.accuracy)}`); +} + +function renderTable(results) { + const body = document.querySelector("#results-table-body"); + body.replaceChildren(); + + results.forEach((result) => { + const row = document.createElement("tr"); + const rank = document.createElement("td"); + rank.append(element("span", "rank-badge", String(result.rank))); + row.append(rank); + row.append(element("td", "", result.model)); + row.append(element("td", "", `${result.passed}/${result.scored} · ${wholePercent(result.accuracy)}`)); + row.append(scoreCell(result.by_category.simple)); + row.append(scoreCell(result.by_category.multiple)); + row.append(scoreCell(result.by_category.irrelevance)); + row.append(scoreCell(result.by_category.chatable)); + body.append(row); + }); +} + +function renderSummary(data) { + const [leader, runnerUp] = data.results; + const multipleLeader = [...data.results].sort( + (first, second) => second.by_category.multiple.accuracy - first.by_category.multiple.accuracy, + )[0]; + const margin = (leader.accuracy - runnerUp.accuracy) * 100; + document.querySelector("#leading-score").textContent = percent(leader.accuracy); + document.querySelector("#leading-model").textContent = leader.model; + document.querySelector("#lead-margin").textContent = `${margin.toFixed(2)} points ahead of ${runnerUp.model}`; + document.querySelector("#leader-insight").textContent = + `${leader.model} leads overall by ${margin.toFixed(2)} points, while ${multipleLeader.model} wins the multiple-tool category.`; + document.querySelector("#case-count").textContent = data.total_cases.toLocaleString("en-US"); + document.querySelector("#model-count").textContent = String(data.results.length); + document.querySelector("#category-count").textContent = String(data.categories.length); + document.querySelector("#dataset-revision").textContent = `Pinned at ${data.dataset_revision.slice(0, 10)}`; +} + +function showLoadError(error) { + const chart = document.querySelector("#overall-chart"); + chart.replaceChildren( + element("p", "error-state", "The benchmark data could not be loaded. Refresh or view the source results on GitHub."), + ); + console.error("Benchmark data load failed", error); +} + +async function initialise() { + try { + const response = await fetch(DATA_URL); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + const colorIndexByModelId = new Map( + data.results.map((result, index) => [result.model_id, index]), + ); + const requestedCategory = new URLSearchParams(window.location.search).get("category"); + const selectedCategory = data.categories.includes(requestedCategory) + ? requestedCategory + : data.categories[0]; + renderSummary(data); + renderOverall(data.results); + renderControls(data.results, colorIndexByModelId, data.categories, selectedCategory); + renderCategory(data.results, colorIndexByModelId, selectedCategory); + renderTable(data.results); + } catch (error) { + showLoadError(error); + } +} + +initialise(); diff --git a/site/build.py b/site/build.py new file mode 100644 index 0000000..6054c04 --- /dev/null +++ b/site/build.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Build the dependency-free benchmark dashboard for GitHub Pages.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +import shutil + + +ASSET_NAMES = ("index.html", "styles.css", "app.js", "favicon.svg") +RESULTS_PATH = Path("bfcl-malay-bench/results/comparison.json") +BUILD_MARKER = ".pixelspace-benchmark-site" +SUPPORTED_CATEGORIES = ("simple", "multiple", "irrelevance", "chatable") + + +def _require_non_empty_string(value: object, field: str) -> None: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"comparison {field} must be a non-empty string") + + +def _require_count(value: object, field: str) -> None: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"comparison {field} must be a non-negative integer") + + +def _validate_score(score: dict, field: str) -> None: + if not isinstance(score, dict): + raise ValueError(f"comparison {field} must be an object") + for key in ("passed", "scored"): + if key not in score: + raise ValueError(f"comparison {field} is missing {key}") + _require_count(score[key], f"{field}.{key}") + if score["scored"] == 0 or score["passed"] > score["scored"]: + raise ValueError(f"comparison {field} has invalid counts") + accuracy = score.get("accuracy") + if not isinstance(accuracy, (int, float)) or isinstance(accuracy, bool): + raise ValueError(f"comparison {field}.accuracy must be numeric") + if not math.isfinite(accuracy) or not 0 <= accuracy <= 1: + raise ValueError(f"comparison {field}.accuracy must be between zero and one") + expected_accuracy = score["passed"] / score["scored"] + if not math.isclose(accuracy, expected_accuracy, rel_tol=0, abs_tol=1e-12): + raise ValueError(f"comparison {field}.accuracy does not match its counts") + + +def _validate_results(data: dict) -> None: + required = { + "benchmark", + "categories", + "dataset", + "dataset_revision", + "results", + "total_cases", + } + missing = sorted(required.difference(data)) + if missing: + raise ValueError(f"comparison data is missing: {', '.join(missing)}") + if len(data["results"]) < 2: + raise ValueError("comparison data must contain at least two models") + if tuple(data["categories"]) != SUPPORTED_CATEGORIES: + raise ValueError("comparison categories do not match the dashboard contract") + _require_count(data["total_cases"], "total_cases") + if data["total_cases"] == 0: + raise ValueError("comparison total_cases must be positive") + + expected_ranks = list(range(1, len(data["results"]) + 1)) + ranks = [result.get("rank") for result in data["results"]] + if ranks != expected_ranks: + raise ValueError("comparison results must be ordered by consecutive rank") + model_ids = set() + previous_accuracy = math.inf + for index, result in enumerate(data["results"]): + if not isinstance(result, dict): + raise ValueError("comparison results must contain objects") + for key in ("model", "model_id"): + _require_non_empty_string(result.get(key), f"results[{index}].{key}") + if result["model_id"] in model_ids: + raise ValueError("comparison model_id values must be unique") + model_ids.add(result["model_id"]) + + _validate_score(result, f"results[{index}]") + _require_count(result.get("errors"), f"results[{index}].errors") + if result["scored"] != data["total_cases"]: + raise ValueError(f"{result['model']} scored count does not match total_cases") + if result["accuracy"] > previous_accuracy: + raise ValueError("comparison results must be ordered by descending accuracy") + previous_accuracy = result["accuracy"] + + by_category = result.get("by_category") + if not isinstance(by_category, dict) or not all( + category in by_category for category in SUPPORTED_CATEGORIES + ): + raise ValueError(f"{result.get('model', 'model')} is missing category results") + for category in SUPPORTED_CATEGORIES: + _validate_score(by_category[category], f"results[{index}].by_category.{category}") + if sum(by_category[category]["passed"] for category in SUPPORTED_CATEGORIES) != result["passed"]: + raise ValueError(f"{result['model']} category passed counts do not match overall") + if sum(by_category[category]["scored"] for category in SUPPORTED_CATEGORIES) != result["scored"]: + raise ValueError(f"{result['model']} category scored counts do not match overall") + + +def build_site(repository_root: Path, output: Path) -> None: + repository_root = repository_root.resolve() + source = repository_root / "site" + if output.is_symlink(): + raise ValueError("site output must not be a symbolic link") + output = output.absolute() + + if output in {repository_root, source}: + raise ValueError("site output must not replace the repository or source directory") + + data = json.loads((repository_root / RESULTS_PATH).read_text(encoding="utf-8")) + _validate_results(data) + + if output.exists(): + if not output.is_dir() or not (output / BUILD_MARKER).is_file(): + raise ValueError("refusing to replace an output directory not created by this builder") + shutil.rmtree(output) + (output / "data").mkdir(parents=True) + + for asset_name in ASSET_NAMES: + shutil.copy2(source / asset_name, output / asset_name) + + (output / "data" / "comparison.json").write_text( + json.dumps(data, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + (output / ".nojekyll").touch() + (output / BUILD_MARKER).touch() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=Path("_site")) + args = parser.parse_args() + repository_root = Path(__file__).resolve().parent.parent + build_site(repository_root, args.output) + + +if __name__ == "__main__": + main() diff --git a/site/favicon.svg b/site/favicon.svg new file mode 100644 index 0000000..0df2df1 --- /dev/null +++ b/site/favicon.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/site/index.html b/site/index.html new file mode 100644 index 0000000..038e104 --- /dev/null +++ b/site/index.html @@ -0,0 +1,175 @@ + + + + + + + + Malay BFCL Benchmark | PixelSpaceAI + + + + + + + + +
+ + + PixelSpaceAI + + + View source + + +
+ +
+
+
+

Open benchmark · August 2026

+

Tool calling,
measured in Malay.

+

+ A focused look at how four language models understand Malay prompts, + function descriptions, and parameter descriptions across BFCL v3. +

+
+ Explore results + Run it yourself +
+
+ + +
+ +
+
+
1,040test cases
+
4models compared
+
4Malay-focused categories
+
0tools executed
+
+
+ +
+
+
+

01 / Overall

+

The leaderboard

+
+

Exact local scoring over the same 1,040 cases. Higher is better.

+
+ +
+ +
+

Loading benchmark results…

+
+
+ +
+
+

Top-line takeaway

+

Loading the closest race…

+
+
+

What this measures

+

Tool choice, arguments, restraint, and conversational replies.

+
+
+
+ +
+
+
+

02 / Categories

+

Where models differ

+
+

Choose a category to compare the same models on a narrower skill.

+
+ +
+ +
+
+
+

Category

+

Simple tool calls

+
+

One function, one precise set of arguments.

+
+
+
+
+ +
+
+
+

03 / Exact results

+

Every score

+
+

Passed cases over scored cases. No hidden weighting.

+
+ +
+ + + + + + + + + + + + + +
RankModelOverallSimpleMultipleIrrelevanceChatable
+
+
+ +
+
+

Method note

+

A practical harness score,
not an official BFCL submission.

+
+
+

+ The focused suite covers simple, multiple, + irrelevance, and chatable. It scores the + first assistant response and never executes benchmark tools. +

+

+ Results compare full serving paths. ILMU used a direct OpenAI-compatible + endpoint; the other deployments used the Pixel Harness command adapter. +

+ Read the full methodology +
+
+
+ + + + diff --git a/site/styles.css b/site/styles.css new file mode 100644 index 0000000..47d48c2 --- /dev/null +++ b/site/styles.css @@ -0,0 +1,874 @@ +:root { + color-scheme: dark; + --ink: #f4f2e9; + --muted: #9ca79f; + --faint: #647068; + --line: rgba(255, 255, 255, 0.1); + --panel: rgba(14, 29, 23, 0.72); + --base: #07130f; + --acid: #c8ff61; + --acid-soft: #8fcf65; + --mint: #6ee7b7; + --cyan: #66d8e5; + --amber: #ffcb66; + --shell: min(1180px, calc(100vw - 48px)); + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + font-synthesis: none; +} + +* { + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +body { + margin: 0; + color: var(--ink); + background: + linear-gradient(rgba(255, 255, 255, 0.018) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.018) 1px, transparent 1px), + var(--base); + background-size: 72px 72px; + min-width: 320px; + overflow-x: hidden; +} + +a { + color: inherit; +} + +button, +a { + -webkit-tap-highlight-color: transparent; +} + +.shell { + width: var(--shell); + margin-inline: auto; +} + +.ambient { + position: fixed; + z-index: -1; + width: 520px; + height: 520px; + border-radius: 50%; + filter: blur(120px); + opacity: 0.11; + pointer-events: none; +} + +.ambient-one { + top: -260px; + right: -120px; + background: var(--acid); +} + +.ambient-two { + top: 50vh; + left: -340px; + background: #20c997; +} + +.site-header { + height: 92px; + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid var(--line); +} + +.brand { + display: inline-flex; + align-items: center; + gap: 12px; + color: var(--ink); + text-decoration: none; + font-size: 17px; + font-weight: 720; + letter-spacing: -0.03em; +} + +.brand > span:last-child > span { + color: var(--acid); +} + +.brand-mark { + width: 27px; + height: 27px; + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 3px; + transform: rotate(7deg); +} + +.brand-mark i { + display: block; + background: var(--acid); +} + +.brand-mark i:nth-child(2), +.brand-mark i:nth-child(3) { + border-radius: 50%; +} + +.header-link { + color: var(--muted); + text-decoration: none; + font-size: 13px; + font-weight: 650; + letter-spacing: 0.04em; + text-transform: uppercase; + transition: color 180ms ease; +} + +.header-link:hover, +.header-link:focus-visible { + color: var(--acid); +} + +.hero { + min-height: 680px; + display: grid; + grid-template-columns: minmax(0, 1fr) 350px; + gap: 10vw; + align-items: center; + padding-block: 88px 100px; +} + +.eyebrow, +.section-number, +.score-kicker, +.category-panel-header p { + margin: 0; + color: var(--acid); + font-size: 11px; + font-weight: 750; + letter-spacing: 0.16em; + text-transform: uppercase; +} + +.eyebrow { + display: flex; + align-items: center; + gap: 10px; +} + +.eyebrow span { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--acid); + box-shadow: 0 0 18px var(--acid); +} + +h1, +h2, +h3, +p { + text-wrap: pretty; +} + +h1 { + margin: 24px 0 28px; + max-width: 800px; + font-family: Georgia, "Times New Roman", serif; + font-size: clamp(62px, 8.2vw, 116px); + font-weight: 400; + line-height: 0.86; + letter-spacing: -0.07em; +} + +h1 em { + color: var(--acid); + font-weight: 400; +} + +.hero-lede { + max-width: 640px; + margin: 0; + color: var(--muted); + font-size: clamp(17px, 1.8vw, 21px); + line-height: 1.65; +} + +.hero-actions { + display: flex; + gap: 12px; + flex-wrap: wrap; + margin-top: 38px; +} + +.button { + min-height: 50px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0 22px; + border: 1px solid var(--line); + border-radius: 999px; + text-decoration: none; + font-size: 13px; + font-weight: 750; + letter-spacing: 0.02em; + transition: transform 180ms ease, border-color 180ms ease, background 180ms ease; +} + +.button:hover, +.button:focus-visible { + transform: translateY(-2px); +} + +.button-primary { + color: #0a170f; + background: var(--acid); + border-color: var(--acid); +} + +.button-secondary:hover, +.button-secondary:focus-visible { + border-color: rgba(200, 255, 97, 0.6); + background: rgba(200, 255, 97, 0.07); +} + +.hero-score { + position: relative; + aspect-ratio: 1; + display: flex; + flex-direction: column; + justify-content: center; + padding: 48px; + border: 1px solid rgba(200, 255, 97, 0.28); + border-radius: 50%; + background: radial-gradient(circle at 50% 38%, rgba(200, 255, 97, 0.1), transparent 58%); + text-align: center; +} + +.score-orbit { + position: absolute; + inset: -15px; + border: 1px dashed rgba(200, 255, 97, 0.2); + border-radius: 50%; + animation: orbit 36s linear infinite; +} + +.hero-score strong { + margin: 10px 0 4px; + color: var(--acid); + font-family: Georgia, "Times New Roman", serif; + font-size: clamp(58px, 6vw, 84px); + font-weight: 400; + letter-spacing: -0.06em; +} + +.hero-score > p:not(.score-kicker) { + margin: 0; + font-size: 15px; + font-weight: 700; +} + +.score-rule { + width: 44px; + height: 1px; + margin: 20px auto 14px; + background: rgba(255, 255, 255, 0.25); +} + +.hero-score > span { + color: var(--muted); + font-size: 12px; + line-height: 1.5; +} + +@keyframes orbit { + to { transform: rotate(360deg); } +} + +.stat-strip { + border-block: 1px solid var(--line); + background: rgba(255, 255, 255, 0.018); +} + +.stat-grid { + display: grid; + grid-template-columns: repeat(4, 1fr); +} + +.stat-grid > div { + min-height: 144px; + display: flex; + flex-direction: column; + justify-content: center; + padding: 28px; + border-left: 1px solid var(--line); +} + +.stat-grid > div:last-child { + border-right: 1px solid var(--line); +} + +.stat-grid strong { + font-family: Georgia, "Times New Roman", serif; + font-size: 38px; + font-weight: 400; + letter-spacing: -0.04em; +} + +.stat-grid span { + margin-top: 7px; + color: var(--muted); + font-size: 12px; +} + +.section, +.method { + padding-block: 130px; +} + +.section { + border-bottom: 1px solid var(--line); +} + +.section-heading { + display: flex; + justify-content: space-between; + gap: 40px; + align-items: end; + margin-bottom: 56px; +} + +.section-heading h2, +.method h2 { + margin: 12px 0 0; + font-family: Georgia, "Times New Roman", serif; + font-size: clamp(44px, 5.5vw, 72px); + font-weight: 400; + line-height: 1; + letter-spacing: -0.055em; +} + +.section-heading > p { + max-width: 360px; + margin: 0 0 4px; + color: var(--muted); + font-size: 14px; + line-height: 1.65; +} + +.chart-panel, +.category-panel, +.table-wrap { + border: 1px solid var(--line); + border-radius: 20px; + background: linear-gradient(145deg, rgba(255, 255, 255, 0.035), rgba(255, 255, 255, 0.012)); + box-shadow: 0 28px 90px rgba(0, 0, 0, 0.2); +} + +.chart-panel { + padding: 28px 34px 34px; +} + +.chart-scale { + display: grid; + grid-template-columns: repeat(5, 1fr); + margin: 0 0 24px 250px; + color: var(--faint); + font-size: 10px; +} + +.chart-scale span:not(:first-child) { + text-align: right; +} + +.leaderboard-chart { + display: grid; + gap: 22px; +} + +.leaderboard-row { + display: grid; + grid-template-columns: 220px 1fr 72px; + gap: 28px; + align-items: center; +} + +.model-label { + min-width: 0; +} + +.model-label strong { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 14px; +} + +.model-label span { + display: block; + margin-top: 5px; + color: var(--faint); + font-size: 10px; + font-weight: 750; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +.bar-track { + position: relative; + height: 34px; + border-radius: 4px; + overflow: hidden; + background: + linear-gradient(90deg, transparent calc(25% - 1px), rgba(255,255,255,.08) 25%, transparent calc(25% + 1px)), + linear-gradient(90deg, transparent calc(50% - 1px), rgba(255,255,255,.08) 50%, transparent calc(50% + 1px)), + linear-gradient(90deg, transparent calc(75% - 1px), rgba(255,255,255,.08) 75%, transparent calc(75% + 1px)), + rgba(255, 255, 255, 0.03); +} + +.bar-fill { + width: 0; + height: 100%; + border-radius: 4px; + background: var(--bar-color, var(--acid)); + box-shadow: 0 0 28px color-mix(in srgb, var(--bar-color, var(--acid)) 28%, transparent); + transition: width 720ms cubic-bezier(0.22, 1, 0.36, 1); +} + +.bar-value { + text-align: right; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 15px; + font-weight: 750; +} + +.insight-grid { + display: grid; + grid-template-columns: 1.2fr 0.8fr; + gap: 16px; + margin-top: 16px; +} + +.insight-card { + min-height: 190px; + padding: 30px; + border: 1px solid var(--line); + border-radius: 20px; +} + +.insight-featured { + color: #0a170f; + background: var(--acid); + border-color: var(--acid); +} + +.insight-card p { + margin: 0 0 24px; + color: var(--muted); + font-size: 11px; + font-weight: 750; + letter-spacing: 0.13em; + text-transform: uppercase; +} + +.insight-featured p { + color: rgba(10, 23, 15, 0.58); +} + +.insight-card h3 { + max-width: 650px; + margin: 0; + font-family: Georgia, "Times New Roman", serif; + font-size: clamp(25px, 3vw, 36px); + font-weight: 400; + line-height: 1.15; + letter-spacing: -0.04em; +} + +.section-categories { + border-bottom: 0; +} + +.category-controls { + display: flex; + flex-wrap: wrap; + gap: 9px; + margin-bottom: 18px; +} + +.category-button { + appearance: none; + padding: 10px 16px; + color: var(--muted); + border: 1px solid var(--line); + border-radius: 999px; + background: transparent; + font: inherit; + font-size: 12px; + font-weight: 700; + cursor: pointer; + transition: color 160ms ease, background 160ms ease, border-color 160ms ease; +} + +.category-button:hover, +.category-button:focus-visible { + color: var(--ink); + border-color: rgba(255, 255, 255, 0.25); +} + +.category-button[aria-pressed="true"] { + color: #0a170f; + background: var(--acid); + border-color: var(--acid); +} + +.category-panel { + display: grid; + grid-template-columns: 300px 1fr; + min-height: 480px; + overflow: hidden; +} + +.category-panel-header { + display: flex; + flex-direction: column; + justify-content: space-between; + padding: 38px; + border-right: 1px solid var(--line); + background: rgba(255, 255, 255, 0.02); +} + +.category-panel-header h3 { + margin: 12px 0 0; + font-family: Georgia, "Times New Roman", serif; + font-size: 40px; + font-weight: 400; + line-height: 1.04; + letter-spacing: -0.05em; +} + +.category-panel-header > p:last-child { + color: var(--muted); + font-size: 13px; + font-weight: 400; + line-height: 1.65; + letter-spacing: 0; + text-transform: none; +} + +.category-chart { + display: grid; + align-content: center; + gap: 26px; + padding: 44px; +} + +.category-row { + display: grid; + grid-template-columns: 170px 1fr 110px; + gap: 22px; + align-items: center; +} + +.category-row .bar-track { + height: 18px; +} + +.category-detail { + text-align: right; +} + +.category-detail strong { + display: block; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 14px; +} + +.category-detail span { + color: var(--faint); + font-size: 10px; +} + +.table-wrap { + overflow-x: auto; +} + +table { + width: 100%; + border-collapse: collapse; + min-width: 850px; +} + +th, +td { + padding: 22px 20px; + border-bottom: 1px solid var(--line); + text-align: right; +} + +th { + color: var(--faint); + font-size: 10px; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +th:nth-child(2), +td:nth-child(2) { + text-align: left; +} + +th:first-child, +td:first-child { + width: 70px; + text-align: center; +} + +td { + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 12px; +} + +td:nth-child(2) { + font-family: inherit; + font-size: 14px; + font-weight: 700; +} + +tbody tr:last-child td { + border-bottom: 0; +} + +tbody tr:first-child td { + background: rgba(200, 255, 97, 0.035); +} + +.rank-badge { + width: 26px; + height: 26px; + display: inline-grid; + place-items: center; + border: 1px solid var(--line); + border-radius: 50%; + color: var(--muted); +} + +tbody tr:first-child .rank-badge { + color: #0a170f; + background: var(--acid); + border-color: var(--acid); +} + +.method { + display: grid; + grid-template-columns: 1.1fr 0.9fr; + gap: 12vw; +} + +.method-copy { + padding-top: 22px; +} + +.method-copy p { + margin: 0 0 22px; + color: var(--muted); + font-size: 15px; + line-height: 1.75; +} + +.method-copy strong { + color: var(--ink); +} + +.method-copy a { + display: inline-block; + margin-top: 14px; + color: var(--acid); + text-decoration: none; + font-size: 13px; + font-weight: 750; +} + +footer { + padding-block: 55px; + border-top: 1px solid var(--line); + background: rgba(0, 0, 0, 0.18); +} + +.footer-grid { + display: flex; + align-items: end; + justify-content: space-between; + gap: 40px; +} + +.footer-grid p { + margin: 14px 0 0; + color: var(--faint); + font-size: 12px; + line-height: 1.7; +} + +.footer-grid > p { + text-align: right; +} + +.footer-grid a:not(.brand) { + color: var(--muted); + text-decoration: none; +} + +.loading-state, +.error-state { + margin: 0; + padding: 40px; + color: var(--muted); + text-align: center; +} + +.error-state { + color: #ff9f89; +} + +:focus-visible { + outline: 2px solid var(--acid); + outline-offset: 4px; +} + +@media (max-width: 900px) { + :root { --shell: min(100% - 32px, 720px); } + + .hero { + grid-template-columns: 1fr; + min-height: auto; + gap: 72px; + padding-block: 72px 88px; + } + + .hero-score { + width: min(350px, 86vw); + justify-self: center; + } + + .stat-grid { + grid-template-columns: repeat(2, 1fr); + } + + .stat-grid > div:nth-child(3), + .stat-grid > div:nth-child(4) { + border-top: 1px solid var(--line); + } + + .stat-grid > div:nth-child(4) { + border-right: 1px solid var(--line); + } + + .section, + .method { + padding-block: 96px; + } + + .leaderboard-row { + grid-template-columns: 160px 1fr 64px; + gap: 18px; + } + + .chart-scale { + margin-left: 178px; + } + + .category-panel, + .method { + grid-template-columns: 1fr; + } + + .category-panel-header { + min-height: 240px; + border-right: 0; + border-bottom: 1px solid var(--line); + } +} + +@media (max-width: 620px) { + .site-header { height: 76px; } + .header-link { font-size: 0; } + .header-link span { font-size: 18px; } + + h1 { + font-size: clamp(52px, 16vw, 76px); + line-height: 0.9; + } + + .hero-lede { font-size: 16px; } + + .stat-grid strong { font-size: 30px; } + .stat-grid > div { min-height: 118px; padding: 20px 16px; } + + .section-heading { + display: block; + margin-bottom: 40px; + } + + .section-heading > p { + margin-top: 24px; + } + + .chart-panel { padding: 22px 18px 26px; } + .chart-scale { display: none; } + + .leaderboard-row { + grid-template-columns: 1fr 58px; + gap: 10px; + } + + .leaderboard-row .model-label { grid-column: 1 / -1; } + .bar-track { height: 26px; } + + .insight-grid { grid-template-columns: 1fr; } + .insight-card { min-height: 170px; } + + .category-panel-header, + .category-chart { padding: 28px 22px; } + + .category-row { + grid-template-columns: 1fr 76px; + gap: 9px 14px; + } + + .category-row .model-label { grid-column: 1 / -1; } + .category-detail { grid-column: 2; grid-row: 2; } + .category-row .bar-track { grid-column: 1; grid-row: 2; } + + .method h2 { font-size: 42px; } + + .footer-grid { + display: block; + } + + .footer-grid > p { + margin-top: 32px; + text-align: left; + } +} + +@media (prefers-reduced-motion: reduce) { + .score-orbit { + animation: none; + } + + .bar-fill { + transition: none; + } + + *, + *::before, + *::after { + scroll-behavior: auto !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + } +} diff --git a/site/tests/test_browser.py b/site/tests/test_browser.py new file mode 100644 index 0000000..e00177b --- /dev/null +++ b/site/tests/test_browser.py @@ -0,0 +1,92 @@ +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +import shutil +import subprocess +import tempfile +import threading +import unittest + +from test_build import REPOSITORY_ROOT, load_build_module + + +class QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, format, *args): + pass + + +class BrowserSmokeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.chrome = next( + ( + executable + for name in ("google-chrome", "chromium", "chromium-browser") + if (executable := shutil.which(name)) + ), + None, + ) + if cls.chrome is None: + raise RuntimeError("Chrome or Chromium is required for the dashboard smoke test") + + def dump_page(self, url, user_data_directory): + completed = subprocess.run( + [ + self.chrome, + "--headless=new", + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + "--dump-dom", + "--virtual-time-budget=3000", + f"--user-data-dir={user_data_directory}", + url, + ], + capture_output=True, + check=True, + text=True, + timeout=20, + ) + return completed.stdout + + def test_built_dashboard_renders_data_category_and_error_state(self): + build = load_build_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = Path(temporary_directory) + output = temporary_path / "public" + build.build_site(REPOSITORY_ROOT, output) + handler = lambda *args, **kwargs: QuietHandler( + *args, directory=str(output), **kwargs + ) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + base_url = f"http://127.0.0.1:{server.server_port}" + rendered = self.dump_page( + f"{base_url}/?category=multiple", temporary_path / "chrome-success" + ) + self.assertIn("76.54%", rendered) + self.assertIn("ILMU Mini v3.3", rendered) + self.assertIn("Multiple choices", rendered) + self.assertIn("130 / 200", rendered) + self.assertNotIn("Loading benchmark results", rendered) + + index = output / "index.html" + index.write_text( + index.read_text(encoding="utf-8").replace( + './data/comparison.json', './data/missing.json', 1 + ), + encoding="utf-8", + ) + failed = self.dump_page(base_url, temporary_path / "chrome-failure") + self.assertIn("The benchmark data could not be loaded", failed) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +if __name__ == "__main__": + unittest.main() diff --git a/site/tests/test_build.py b/site/tests/test_build.py new file mode 100644 index 0000000..fc3bc8d --- /dev/null +++ b/site/tests/test_build.py @@ -0,0 +1,137 @@ +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +REPOSITORY_ROOT = Path(__file__).parents[2] +BUILD_MODULE_PATH = REPOSITORY_ROOT / "site" / "build.py" + + +def load_build_module(): + spec = importlib.util.spec_from_file_location("benchmark_site_build", BUILD_MODULE_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class SiteBuildTests(unittest.TestCase): + def test_build_copies_assets_and_canonical_results(self): + build = load_build_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + output = Path(temporary_directory) / "public" + build.build_site(REPOSITORY_ROOT, output) + + self.assertTrue((output / ".nojekyll").exists()) + self.assertTrue((output / "index.html").exists()) + self.assertTrue((output / "styles.css").exists()) + self.assertTrue((output / "app.js").exists()) + self.assertTrue((output / "favicon.svg").exists()) + + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + published_results = json.loads( + (output / "data" / "comparison.json").read_text(encoding="utf-8") + ) + self.assertEqual(published_results, canonical_results) + + page = (output / "index.html").read_text(encoding="utf-8") + self.assertIn('id="overall-chart"', page) + self.assertIn('id="category-controls"', page) + self.assertIn('id="results-table-body"', page) + + def test_build_replaces_stale_output(self): + build = load_build_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + output = Path(temporary_directory) / "public" + output.mkdir() + (output / build.BUILD_MARKER).touch() + stale_file = output / "stale.txt" + stale_file.write_text("old", encoding="utf-8") + + build.build_site(REPOSITORY_ROOT, output) + + self.assertFalse(stale_file.exists()) + + def test_build_refuses_to_replace_unmarked_output(self): + build = load_build_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + output = Path(temporary_directory) / "public" + output.mkdir() + protected_file = output / "keep.txt" + protected_file.write_text("keep", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "refusing to replace"): + build.build_site(REPOSITORY_ROOT, output) + + self.assertEqual(protected_file.read_text(encoding="utf-8"), "keep") + + def test_validation_rejects_dashboard_schema_drift(self): + build = load_build_module() + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + canonical_results["categories"] = ["simple"] + + with self.assertRaisesRegex(ValueError, "dashboard contract"): + build._validate_results(canonical_results) + + def test_validation_requires_comparison_between_models(self): + build = load_build_module() + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + canonical_results["results"] = canonical_results["results"][:1] + + with self.assertRaisesRegex(ValueError, "at least two models"): + build._validate_results(canonical_results) + + def test_validation_rejects_missing_required_key(self): + build = load_build_module() + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + del canonical_results["dataset"] + + with self.assertRaisesRegex(ValueError, "missing: dataset"): + build._validate_results(canonical_results) + + def test_validation_rejects_nonconsecutive_ranks(self): + build = load_build_module() + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + canonical_results["results"][1]["rank"] = 3 + + with self.assertRaisesRegex(ValueError, "consecutive rank"): + build._validate_results(canonical_results) + + def test_validation_rejects_missing_result_category(self): + build = load_build_module() + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + del canonical_results["results"][0]["by_category"]["simple"] + + with self.assertRaisesRegex(ValueError, "missing category results"): + build._validate_results(canonical_results) + + def test_validation_rejects_inconsistent_accuracy(self): + build = load_build_module() + canonical_results = json.loads( + (REPOSITORY_ROOT / build.RESULTS_PATH).read_text(encoding="utf-8") + ) + canonical_results["results"][0]["accuracy"] = 0.99 + + with self.assertRaisesRegex(ValueError, "does not match its counts"): + build._validate_results(canonical_results) + + +if __name__ == "__main__": + unittest.main()