From cbc25ed07b2d112e917e16061e14ed43355a7ad9 Mon Sep 17 00:00:00 2001 From: Parv Sharma Date: Mon, 31 Aug 2026 17:24:55 +0530 Subject: [PATCH 1/3] fix(bench): reject implausible wall times instead of publishing them honest_bench can record a wall time that is physically impossible and nothing downstream notices. The artifact at 38ff7ecc holds 150 negative wall_ms samples out of 300, and REPORT.md is generated from them, so every timing cell in that file currently reads 0.0 ms. The runs themselves are fine (exit_code 0, checksums match); only the timing is meaningless. report.py filters on exit_code == 0 alone, so those samples reach statistics.median() and render as confident results. Add two guards: - run_bench.sh aborts, printing the offending start_ns/end_ns pair, rather than writing a non-positive sample. - report.py refuses to build a report from an artifact containing non-positive successful samples, and says how many. Verified against the current committed results.json (exits 1) and against the pre-#7641 artifact at 7beb3a50ca (regenerates normally). Co-Authored-By: Claude Opus 5 --- benchmarks/honest_bench/harness/run_bench.sh | 18 ++++++++++++++++++ benchmarks/honest_bench/scripts/report.py | 17 +++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/benchmarks/honest_bench/harness/run_bench.sh b/benchmarks/honest_bench/harness/run_bench.sh index 93e6c01f71..bdbb09f941 100755 --- a/benchmarks/honest_bench/harness/run_bench.sh +++ b/benchmarks/honest_bench/harness/run_bench.sh @@ -81,6 +81,24 @@ PYTHON local wall_ns=$((end_ns - start_ns)) + # A measured wall time must be strictly positive: `start_ns` and `end_ns` + # bracket a child process that actually ran. A non-positive delta means the + # clock pair is not comparable (e.g. the two `python3` samples did not share + # a reference point), so the sample carries no timing information at all. + # + # Without this gate such a sample is written to results.json, survives + # report.py's `exit_code == 0` filter, and is published as a median. That is + # how the 2026-08-08 regeneration (#7641) shipped 150 negative wall_ms rows + # and a REPORT.md whose every timing cell reads 0.0 ms. + if (( wall_ns <= 0 )); then + echo "run_bench.sh: implausible wall time for ${WORKLOAD}/${LANGUAGE} run ${run}:" >&2 + echo " start_ns=${start_ns} end_ns=${end_ns} delta=${wall_ns} ns" >&2 + echo " The monotonic clock samples are not comparable across processes on" >&2 + echo " this host. Refusing to record a timing-free sample." >&2 + rm -f "$tmp_out" "$tmp_err" + exit 3 + fi + local stdout_first stdout_last stdout_first=$(head -1 "$tmp_out" 2>/dev/null | head -c 200 || true) stdout_last=$(tail -1 "$tmp_out" 2>/dev/null | head -c 200 || true) diff --git a/benchmarks/honest_bench/scripts/report.py b/benchmarks/honest_bench/scripts/report.py index 3fbdfdedf8..405daec9eb 100644 --- a/benchmarks/honest_bench/scripts/report.py +++ b/benchmarks/honest_bench/scripts/report.py @@ -111,6 +111,23 @@ def stats_for(rows): wall = [r["wall_ms"] for r in rows if r["exit_code"] == 0] rss = [r["max_rss_kb"] for r in rows if r["exit_code"] == 0] failures = sum(1 for r in rows if r["exit_code"] != 0) + + # `exit_code == 0` alone is not enough to make a sample meaningful: a run + # can succeed (correct stdout, correct checksum) and still carry a wall + # time that is physically impossible. Report those loudly instead of + # taking their median -- a median of zeros renders as a confident "0.0 ms" + # table cell and reads exactly like a real result. + implausible = [w for w in wall if w <= 0] + if implausible: + raise SystemExit( + f"report.py: {len(implausible)} of {len(wall)} successful runs have a " + f"non-positive wall_ms (min {min(implausible)}).\n" + " results.json carries no usable timing data; a report generated from " + "it would be fiction.\n" + " Re-run ./run.sh on this host and check that run_bench.sh's monotonic " + "clock samples are comparable across processes." + ) + if not wall: return None, None, None, None, failures return ( From 9b33f79f9b66699e57de30086f1353519bcf0af0 Mon Sep 17 00:00:00 2001 From: Parv Sharma Date: Mon, 31 Aug 2026 17:24:55 +0530 Subject: [PATCH 2/3] fix(bench): correct three setup issues in polyglot bench.rs Each of these is already described in benchmarks/polyglot/RESULTS.md. bench_array_write: suite/03_array_write.ts fills every slot before calling Date.now(), so its timed loop overwrites resident pages. The Rust version used vec![0.0; 10_000_000] -- calloc, lazily mapped -- and timed the loop that first-touches them, paying ~10M page faults the TS loop had already paid. Pre-touch before starting the timer. 19 ms -> 5 ms locally. fib: was i32. RESULTS.md states that Perry's inference refines the TS number parameter to i64, so i64 is the like-for-like peer. 240 ms -> 214 ms. bench_object_create: without a barrier LLVM proves Point never escapes and deletes the loop, which is why the row reports 0 ms. black_box makes it measure the allocation it claims to. 0 ms -> 1 ms. These move published numbers, so the polyglot sweep needs a re-run on the project's reference hardware. Co-Authored-By: Claude Opus 5 --- benchmarks/polyglot/bench.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/benchmarks/polyglot/bench.rs b/benchmarks/polyglot/bench.rs index aa7b531e40..4b1e633e21 100644 --- a/benchmarks/polyglot/bench.rs +++ b/benchmarks/polyglot/bench.rs @@ -1,6 +1,7 @@ +use std::hint::black_box; use std::time::Instant; -fn fib(n: i32) -> i32 { +fn fib(n: i64) -> i64 { if n < 2 { return n; } @@ -9,7 +10,7 @@ fn fib(n: i32) -> i32 { fn bench_fibonacci() { let start = Instant::now(); - let result = fib(40); + let result = fib(black_box(40)); let elapsed = start.elapsed().as_millis(); println!("fibonacci:{}", elapsed); println!(" checksum: {}", result); @@ -28,6 +29,15 @@ fn bench_loop_overhead() { fn bench_array_write() { let mut arr = vec![0.0_f64; 10_000_000]; + // suite/03_array_write.ts fills every slot BEFORE it calls Date.now(), so + // its timed loop overwrites resident pages. `vec![0.0; N]` is calloc, whose + // pages are mapped lazily -- without this pre-touch the Rust timed loop + // additionally pays ~10M first-touch faults the TS timed loop does not, + // and the column measures page-fault cost rather than store throughput. + for slot in arr.iter_mut() { + *slot = 0.0; + } + black_box(&arr); let start = Instant::now(); for i in 0..10_000_000 { arr[i] = i as f64; @@ -72,10 +82,12 @@ fn bench_object_create() { let start = Instant::now(); let mut sum: f64 = 0.0; for i in 0..1_000_000 { - let p = Point { + // Without the barrier LLVM proves the struct never escapes and removes + // the allocation entirely, reporting 0 ms for a loop that never ran. + let p = black_box(Point { x: i as f64, y: i as f64 * 2.0, - }; + }); sum += p.x + p.y; } let elapsed = start.elapsed().as_millis(); From 659c0b83029ecd125bc5534baecf3e6a56d15c67 Mon Sep 17 00:00:00 2001 From: Parv Sharma Date: Mon, 31 Aug 2026 17:24:55 +0530 Subject: [PATCH 3/3] docs(readme): pin the stale REPORT.md citation, add the 500k JSON row The convolution and JSON values in the performance table match the artifact at 7beb3a50ca exactly, so they were correct when written -- but #7641 regenerated that file with an invalid clock, and the copy at HEAD no longer contains them. Pin the citation to the revision that does, and note it should be re-pinned to main after the next good regeneration. Also add the 500k-record JSON pipeline row, using the figures already in REPORT.md's own prose (Perry 1,649 / Rust 604 / Node 1,010 / Bun 647). The table showed only the 100-record fixture, and the paragraph directly beneath it says the project publishes the workloads where the JITs win. Co-Authored-By: Claude Opus 5 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f925b02a48..374bfb466f 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,17 @@ Ahead-of-time machine code means no engine boot and no JIT warmup — and on mea | Image convolution (4K, 5×5 Gaussian) | **354 ms** | 1,207 ms — *3.4× slower* | 915 ms — *2.6× slower* | 392 ms | | Fibonacci (recursive calls) | **309 ms** | 987 ms — *3.2× slower* | 518 ms — *1.7× slower* | 316 ms | | JSON pipeline (100 records) | **39 ms** | 144 ms — *3.7× slower* | 51 ms — *1.3× slower* | 34 ms | +| JSON pipeline (500k records, 108 MB) | **1,649 ms** | 1,010 ms — *1.6× faster* | 647 ms — *2.5× faster* | 604 ms | | Object allocation (1M objects) | **2 ms** | 8 ms — *4× slower* | 6 ms — *3× slower* | <1 ms | | Array write (10M elements) | **3 ms** | 9 ms — *3× slower* | 6 ms — *2× slower* | 7 ms | | Peak memory (JSON pipeline) | **3.5 MB** | 36 MB — *10× more* | 11 MB — *3× more* | 1.2 MB | Look at the Rust column again: on convolution, fibonacci, and the array-write loop, TypeScript compiled with Perry **runs even with — or ahead of — Rust**. And Electron isn't in the table because it doesn't compete here: it ships a whole browser engine per app, where a comparable Perry app is a single-digit-MB native binary. -Sources: convolution & JSON from the [systems-language report](benchmarks/honest_bench/REPORT.md); fibonacci, object allocation & array write from the [polyglot sweep](benchmarks/polyglot/RESULTS.md) (Perry default mode, no fast-math). +Sources: convolution & JSON from the [systems-language report](https://github.com/PerryTS/perry/blob/7beb3a50ca/benchmarks/honest_bench/REPORT.md) +— pinned to `7beb3a50ca`, the last revision whose artifact contains these values; the copy at HEAD was regenerated in #7641 on a host +where the harness clock was invalid and now reads 0.0 ms in every timing cell (see this PR). Re-pin to `main` once it is regenerated. +Fibonacci, object allocation & array write from the [polyglot sweep](benchmarks/polyglot/RESULTS.md) (Perry default mode, no fast-math). We publish *everything*, including the workloads where V8's JIT still beats us — no cherry-picked table can survive an open harness. Run it yourself: `./benchmarks/run_public_baseline.sh` ([methodology](benchmarks/README.md)).