From f5d05f827461cdacd2d150f6ab2c346723c67daa Mon Sep 17 00:00:00 2001 From: leo-gan Date: Tue, 1 Sep 2026 14:14:55 -0700 Subject: [PATCH 1/4] Stop ranking queues on payload size; publish CPU instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Size is a leftover serializer score. Every library in a cell moves the same already-built payload, so “most compact” and speed-vs-size Pareto were a horizontal line. Keep Size on the CSV as fixture metadata, treat fidelity as a validity gate, and rank on handoff plus messages per CPU-second (spin vs block). Dashboard, analysis, and experiment tables follow that contract. --- README.md | 5 +- .../src/benchmark_analysis/metrics_catalog.py | 9 +- analysis/src/benchmark_analysis/reports.py | 4 +- analysis/src/benchmark_analysis/stats.py | 26 +++- config/benchmark_config.yaml | 6 +- dashboard/charts.js | 65 +++++---- dashboard/exp-export.js | 11 +- dashboard/format.js | 3 +- dashboard/index.html | 30 ++-- dashboard/main.js | 134 +++++++----------- dashboard/test/exp-graphs.test.js | 37 +++++ docs/analysis/BENCHMARK_DESIGN.md | 7 +- docs/analysis/METRICS.md | 101 +++++++++---- docs/analysis/architecture.md | 3 +- docs/internal/DESIGN.md | 2 +- experiments/lib/summarize_handoff.py | 7 +- python/src/benchmark/report.py | 1 + 17 files changed, 259 insertions(+), 192 deletions(-) diff --git a/README.md b/README.md index ff9403c..9a1ec33 100644 --- a/README.md +++ b/README.md @@ -131,9 +131,10 @@ Every language writes the same columns (nanoseconds). Domain mapping: | `TimeDeq` | Dequeue ns | | `TimeHandoff` | Handoff ns | | `Pattern` | `bytes` = **SPSC**, `stream` = **MPMC** (not I/O) | -| `Size` | Payload bytes | +| `CpuTimeNs` | Process CPU time (spin vs block) | +| `Size` | Payload bytes in this cell — **fixture**, not a library score | -See [architecture](https://leo-gan.github.io/queue-benchmark/analysis/architecture/). +See [Metrics](https://leo-gan.github.io/queue-benchmark/analysis/METRICS/) and [architecture](https://leo-gan.github.io/queue-benchmark/analysis/architecture/). --- diff --git a/analysis/src/benchmark_analysis/metrics_catalog.py b/analysis/src/benchmark_analysis/metrics_catalog.py index 6b349e6..08920d7 100644 --- a/analysis/src/benchmark_analysis/metrics_catalog.py +++ b/analysis/src/benchmark_analysis/metrics_catalog.py @@ -17,8 +17,8 @@ "avg_time_deq_ns": "medium", "handoff_mean_ns": "medium", "avg_ops_per_sec": "high", - "median_size_bytes": "high", - "mean_fidelity": "high", + "median_size_bytes": "low", + "mean_fidelity": "medium", "library_version": "high", "runs": "high", "handoff_ci_low_ns": "medium", @@ -110,8 +110,9 @@ def rank_by_field(metrics_cfg: Optional[Dict[str, Any]] = None) -> str: ("enq_median_ns", "Median enqueue (µs)", True, False), ("deq_median_ns", "Median dequeue (µs)", True, False), ("avg_ops_per_sec", "Ops/s (from mean)", False, True), - ("median_size_bytes", "Median size (B)", False, False), - # runs / mean_fidelity intentionally omitted from multi-way Summary. + ("handoff_p99_ns", "Handoff p99 (µs)", True, False), + ("msgs_per_cpu_sec", "Msgs / CPU-s", False, True), + # Size is a fixture (same payload for every library). Fidelity is a gate. ("library_version", "Version", False, None), ("effect_vs_fastest_cliffs_label", "δ vs fastest", False, None), ) diff --git a/analysis/src/benchmark_analysis/reports.py b/analysis/src/benchmark_analysis/reports.py index 3bb54d1..195045e 100644 --- a/analysis/src/benchmark_analysis/reports.py +++ b/analysis/src/benchmark_analysis/reports.py @@ -1133,7 +1133,7 @@ def _scientific_summary_md(stats: Dict, profile: str = "multi_way") -> str: # Shared K/M scale per column (same rule as ops/s and pivot tables) col_units: Dict[str, tuple] = {} - for field_id in ("avg_ops_per_sec", "median_size_bytes"): + for field_id in ("avg_ops_per_sec", "msgs_per_cpu_sec"): vals_in_col = [cell_vals[(ser, field_id)] for ser in serializers if (ser, field_id) in cell_vals] if vals_in_col: col_units[field_id] = _pick_column_unit(vals_in_col) @@ -1180,7 +1180,7 @@ def _scientific_summary_md(stats: Dict, profile: str = "multi_way") -> str: elif is_time: # ns → µs; fixed-point only (never 1.17e+03) text = _format_sig(num / 1000.0, sig=3) - elif field_id in ("avg_ops_per_sec", "median_size_bytes"): + elif field_id in ("avg_ops_per_sec", "msgs_per_cpu_sec"): # 3 significant digits + shared column K/M (thousands / millions) div, unit = col_units.get(field_id) or _pick_column_unit([num]) text = _format_in_unit(num, div, unit, sig=3) diff --git a/analysis/src/benchmark_analysis/stats.py b/analysis/src/benchmark_analysis/stats.py index e39108d..e6784af 100644 --- a/analysis/src/benchmark_analysis/stats.py +++ b/analysis/src/benchmark_analysis/stats.py @@ -1201,7 +1201,11 @@ def public_stats_entry(entry: Dict[str, Any]) -> Dict[str, Any]: def compute_pareto_front(groups: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Classical 2D Pareto: minimize avg_time_handoff_ns and median_size_bytes.""" + """2D Pareto: minimize handoff time, maximize msgs_per_cpu_sec. + + Size is a fixture (same payload for every library in the cell) and is not + an axis. When CPU is missing, the front is the fastest handoff only. + """ front: List[Dict[str, Any]] = [] workloads: Dict[Tuple[Any, Any], List[Dict[str, Any]]] = {} for g in groups: @@ -1210,18 +1214,26 @@ def compute_pareto_front(groups: List[Dict[str, Any]]) -> List[Dict[str, Any]]: for items in workloads.values(): for item in items: t = pick_stats(item, "avg_time_handoff_ns") - s = item.get("median_size_bytes") - if t is None or s is None: + cpu = item.get("msgs_per_cpu_sec") + if t is None: continue dominated = False for other in items: if other is item: continue ot = pick_stats(other, "avg_time_handoff_ns") - os_ = other.get("median_size_bytes") - if ot is None or os_ is None: + ocpu = other.get("msgs_per_cpu_sec") + if ot is None: + continue + if cpu is None or ocpu is None: + if ot < t: + dominated = True + break continue - if (ot <= t and os_ < s) or (ot < t and os_ <= s): + better_or_eq_time = ot <= t + better_or_eq_cpu = ocpu >= cpu + strictly = ot < t or ocpu > cpu + if better_or_eq_time and better_or_eq_cpu and strictly: dominated = True break if not dominated: @@ -1231,7 +1243,7 @@ def compute_pareto_front(groups: List[Dict[str, Any]]) -> List[Dict[str, Any]]: "test_data": item.get("test_data"), "mode": item.get("mode"), "time": t, - "size": s, + "msgs_per_cpu_sec": cpu, "filter_policy": (item.get("filter") or {}).get("policy"), } ) diff --git a/config/benchmark_config.yaml b/config/benchmark_config.yaml index b9906ef..b757561 100644 --- a/config/benchmark_config.yaml +++ b/config/benchmark_config.yaml @@ -190,7 +190,7 @@ metrics: multi_way: include_importance: [high] rank_by: handoff_median_ns - secondary_rank_by: median_size_bytes + secondary_rank_by: msgs_per_cpu_sec pairwise: include_importance: [high, medium, low] importance: @@ -204,8 +204,8 @@ metrics: enq_mean_ns: medium deq_mean_ns: medium avg_ops_per_sec: high - median_size_bytes: high - mean_fidelity: high + median_size_bytes: low + mean_fidelity: medium library_version: high runs: high runs_raw: medium diff --git a/dashboard/charts.js b/dashboard/charts.js index 815f555..ab99fbf 100644 --- a/dashboard/charts.js +++ b/dashboard/charts.js @@ -17,7 +17,7 @@ const fontStyle = { const gridColor = 'rgba(0, 0, 0, 0.06)'; const tickColor = '#5f6368'; -/** @type {{ logScale: boolean, rankSort: 'speed' | 'size' }} */ +/** @type {{ logScale: boolean, rankSort: 'speed' | 'cpu' }} */ let chartOptions = { logScale: false, rankSort: 'speed' }; export function initCharts() { @@ -32,9 +32,9 @@ export function getChartLogScale() { return chartOptions.logScale; } -/** Ranking chart row order: 'speed' (ops/latency) or 'size' (median bytes, compact first). */ +/** Ranking chart row order: 'speed' (ops/latency) or 'cpu' (msgs per CPU-s). */ export function setRankSort(sort) { - chartOptions.rankSort = sort === 'size' ? 'size' : 'speed'; + chartOptions.rankSort = sort === 'cpu' || sort === 'size' ? 'cpu' : 'speed'; } export function getRankSort() { @@ -47,20 +47,20 @@ export function updateCharts(groups, paretoNames, metric) { const title = document.getElementById('bar-chart-title'); if (title) { title.textContent = - metric === 'ops' ? 'Throughput & Size Ranking' : 'Latency & Size Ranking'; + metric === 'ops' ? 'Throughput & CPU ranking' : 'Latency & CPU ranking'; } const help = document.getElementById('ranking-help'); if (help) { const primaryRight = metric === 'ops' ? 'ops/s' : 'latency'; const sortLabel = - chartOptions.rankSort === 'size' - ? 'sorted by size (most compact first)' + chartOptions.rankSort === 'cpu' + ? 'sorted by messages per CPU-second (highest first)' : metric === 'ops' ? 'sorted by ops/s (highest first)' : 'sorted by latency (lowest first)'; help.innerHTML = `Single diverging chart, ${sortLabel}: ` + - `◀ size left · ` + + `◀ CPU left · ` + `${primaryRight} ▶ right. ` + `Each side is normalized to the chart max (100); hover for absolute values.`; } @@ -89,7 +89,7 @@ function updateScatterChart(groups, paretoNames, metric) { groups.forEach((g) => { const xVal = isOps ? g.avg_ops_per_sec : g.avg_time_handoff_ns; - const yVal = g.median_size_bytes; + const yVal = g.msgs_per_cpu_sec; if (xVal == null || yVal == null || !Number.isFinite(xVal) || !Number.isFinite(yVal)) return; if (chartOptions.logScale && xVal <= 0) return; const point = { @@ -118,7 +118,7 @@ function updateScatterChart(groups, paretoNames, metric) { return formatSig(value / latencyScale.divisor); }; - const sortedPareto = [...paretoPoints].sort((a, b) => a.y - b.y); + const sortedPareto = [...paretoPoints].sort((a, b) => b.y - a.y); const frontierLineData = []; for (let i = 0; i < sortedPareto.length; i++) { frontierLineData.push({ x: sortedPareto[i].x, y: sortedPareto[i].y }); @@ -197,8 +197,8 @@ function updateScatterChart(groups, paretoNames, metric) { `Queue: ${p.label}`, `Throughput: ${formatOpsCompact(p.ops)}`, `Latency: ${formatTimeCompact(p.time)}`, - `Size: ${formatIntGrouped(p.y)} bytes`, - p.onFrontier ? 'On Pareto frontier' : 'Dominated on speed/size', + `Msgs / CPU-s: ${formatSig(p.y)}`, + p.onFrontier ? 'On Pareto frontier' : 'Dominated on speed/CPU', ]; }, }, @@ -223,7 +223,7 @@ function updateScatterChart(groups, paretoNames, metric) { y: { title: { display: true, - text: 'Payload size (bytes)', + text: 'Messages per CPU-second', color: tickColor, font: { ...fontStyle, weight: 'bold' }, }, @@ -231,7 +231,7 @@ function updateScatterChart(groups, paretoNames, metric) { ticks: { color: tickColor, font: fontStyle, - callback: (v) => formatIntGrouped(v), + callback: (v) => formatSig(v), }, }, }, @@ -267,7 +267,7 @@ function updateScatterChart(groups, paretoNames, metric) { /** * Single diverging (butterfly) horizontal bar chart: * - Right (blue): throughput or latency, normalized 0..100 vs chart max - * - Left (green): median size, normalized 0..-100 vs chart max + * - Left (green): messages per CPU-second, normalized 0..-100 vs chart max * Absolute values only in tooltips — avoids dual-axis collisions. */ function updateBarChart(groups, paretoNames, metric) { @@ -276,23 +276,22 @@ function updateBarChart(groups, paretoNames, metric) { if (barChartInstance) barChartInstance.destroy(); const isOps = metric === 'ops'; - const sortBySize = chartOptions.rankSort === 'size'; + const sortByCpu = chartOptions.rankSort === 'cpu'; const sortedGroups = [...groups] .filter((g) => { if (!g) return false; - if (g.median_size_bytes == null && sortBySize) return false; + if (g.msgs_per_cpu_sec == null && sortByCpu) return false; return isOps ? g.avg_ops_per_sec != null : g.avg_time_handoff_ns != null; }) .sort((a, b) => { - if (sortBySize) { - // Compact first (ascending size); tie-break by speed - const ds = (a.median_size_bytes ?? 0) - (b.median_size_bytes ?? 0); - if (ds !== 0) return ds; + if (sortByCpu) { + const dc = (b.msgs_per_cpu_sec ?? 0) - (a.msgs_per_cpu_sec ?? 0); + if (dc !== 0) return dc; } return isOps ? b.avg_ops_per_sec - a.avg_ops_per_sec - : a.avg_time_total_ns - b.avg_time_total_ns; + : (a.avg_time_handoff_ns ?? 0) - (b.avg_time_handoff_ns ?? 0); }) .slice(0, 15); @@ -300,17 +299,17 @@ function updateBarChart(groups, paretoNames, metric) { const primaryRaw = sortedGroups.map((g) => isOps ? g.avg_ops_per_sec : g.avg_time_handoff_ns ); - const sizeRaw = sortedGroups.map((g) => Number(g.median_size_bytes) || 0); + const cpuRaw = sortedGroups.map((g) => Number(g.msgs_per_cpu_sec) || 0); const maxPrimary = Math.max(...primaryRaw.filter((v) => Number.isFinite(v) && v > 0), 1); - const maxSize = Math.max(...sizeRaw.filter((v) => Number.isFinite(v) && v > 0), 1); + const maxCpu = Math.max(...cpuRaw.filter((v) => Number.isFinite(v) && v > 0), 1); - // Normalized: speed → +0..100, size → -0..-100 (diverging from center) + // Normalized: speed → +0..100, CPU → -0..-100 (diverging from center) const speedNorm = primaryRaw.map((v) => Number.isFinite(v) && maxPrimary > 0 ? (v / maxPrimary) * 100 : 0 ); - const sizeNorm = sizeRaw.map((v) => - Number.isFinite(v) && maxSize > 0 ? -(v / maxSize) * 100 : 0 + const sizeNorm = cpuRaw.map((v) => + Number.isFinite(v) && maxCpu > 0 ? -(v / maxCpu) * 100 : 0 ); const speedColors = sortedGroups.map((g) => @@ -324,10 +323,10 @@ function updateBarChart(groups, paretoNames, metric) { type: 'bar', data: { labels, - // Legend order matches plot: size left (←), then speed/latency right (→). + // Legend order matches plot: CPU left (←), then speed/latency right (→). datasets: [ { - label: 'Size (←)', + label: 'CPU (←)', data: sizeNorm, backgroundColor: sizeColors, borderColor: sortedGroups.map((g) => @@ -383,9 +382,9 @@ function updateBarChart(groups, paretoNames, metric) { const g = sortedGroups[context.dataIndex]; if (!g) return ''; const pct = Math.abs(context.raw).toFixed(0); - // dataset 0 = size (left), 1 = speed/latency (right) + // dataset 0 = CPU (left), 1 = speed/latency (right) if (context.datasetIndex === 0) { - return `Size: ${formatIntGrouped(g.median_size_bytes)} bytes (${pct}% of max in chart)`; + return `Msgs / CPU-s: ${formatSig(g.msgs_per_cpu_sec)} (${pct}% of max in chart)`; } const abs = isOps ? formatOpsCompact(g.avg_ops_per_sec) @@ -398,7 +397,7 @@ function updateBarChart(groups, paretoNames, metric) { const g = sortedGroups[items[0]?.dataIndex]; if (!g) return []; return [ - paretoNames.includes(g.library) ? 'Pareto optimal' : 'Dominated on speed/size', + paretoNames.includes(g.library) ? 'Pareto optimal' : 'Dominated on speed/CPU', ]; }, }, @@ -417,8 +416,8 @@ function updateBarChart(groups, paretoNames, metric) { title: { display: true, text: isOps - ? '◀ larger size normalized % of chart max higher ops/s ▶' - : '◀ larger size normalized % of chart max higher latency ▶', + ? '◀ more msgs / CPU-s normalized % of chart max higher ops/s ▶' + : '◀ more msgs / CPU-s normalized % of chart max higher latency ▶', color: tickColor, font: { ...fontStyle, size: 10 }, }, diff --git a/dashboard/exp-export.js b/dashboard/exp-export.js index 6d4ff90..01516b5 100644 --- a/dashboard/exp-export.js +++ b/dashboard/exp-export.js @@ -36,13 +36,20 @@ function hasFiniteMetric(rows, key) { return (rows || []).some((r) => Number.isFinite(Number(r?.[key]))); } -/** Which optional experiment-table columns have at least one real value. */ +function valuesVary(rows, key) { + const nums = (rows || []).map((r) => Number(r?.[key])).filter(Number.isFinite); + if (nums.length < 2) return false; + const first = nums[0]; + return nums.some((v) => Math.abs(v - first) > 1e-9); +} + +/** Optional experiment-table columns. Size is a data-type property unless values actually differ. */ export function experimentTableFlags(rows) { const list = rows || []; return { write: hasFiniteMetric(list, 'write_median_ns'), read: hasFiniteMetric(list, 'read_median_ns'), - size: hasFiniteMetric(list, 'size_bytes'), + size: hasFiniteMetric(list, 'size_bytes') && valuesVary(list, 'size_bytes'), spread: list.some((r) => totalStdUs(r) != null), }; } diff --git a/dashboard/format.js b/dashboard/format.js index 5d10081..e0f4f78 100644 --- a/dashboard/format.js +++ b/dashboard/format.js @@ -158,6 +158,7 @@ export function metricKind(key) { ) { return 'ops'; } + if (key === 'msgs_per_cpu_sec') return 'rate'; if (key.endsWith('_bytes') || key.startsWith('size_') || key === 'median_size_bytes') return 'bytes'; if ( key === 'runs' || @@ -187,7 +188,7 @@ export function formatMetricCell(key, value, scales = {}) { if (kind === 'latency') return formatLatencyCell(value, scales.latency); if (kind === 'ops') return formatOpsCell(value, scales.ops); if (kind === 'bytes' || kind === 'count') return formatIntGrouped(value); - if (kind === 'ratio') return formatSig(value); + if (kind === 'ratio' || kind === 'rate') return formatSig(value); if (Number.isInteger(value) && Math.abs(value) < 1e9) return formatIntGrouped(value); return formatSig(value); } diff --git a/dashboard/index.html b/dashboard/index.html index 4187459..784871e 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -151,11 +151,11 @@