diff --git a/README.md b/README.md index f925b02a48..374bfb466f 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,17 @@ Ahead-of-time machine code means no engine boot and no JIT warmup — and on mea | Image convolution (4K, 5×5 Gaussian) | **354 ms** | 1,207 ms — *3.4× slower* | 915 ms — *2.6× slower* | 392 ms | | Fibonacci (recursive calls) | **309 ms** | 987 ms — *3.2× slower* | 518 ms — *1.7× slower* | 316 ms | | JSON pipeline (100 records) | **39 ms** | 144 ms — *3.7× slower* | 51 ms — *1.3× slower* | 34 ms | +| JSON pipeline (500k records, 108 MB) | **1,649 ms** | 1,010 ms — *1.6× faster* | 647 ms — *2.5× faster* | 604 ms | | Object allocation (1M objects) | **2 ms** | 8 ms — *4× slower* | 6 ms — *3× slower* | <1 ms | | Array write (10M elements) | **3 ms** | 9 ms — *3× slower* | 6 ms — *2× slower* | 7 ms | | Peak memory (JSON pipeline) | **3.5 MB** | 36 MB — *10× more* | 11 MB — *3× more* | 1.2 MB | Look at the Rust column again: on convolution, fibonacci, and the array-write loop, TypeScript compiled with Perry **runs even with — or ahead of — Rust**. And Electron isn't in the table because it doesn't compete here: it ships a whole browser engine per app, where a comparable Perry app is a single-digit-MB native binary. -Sources: convolution & JSON from the [systems-language report](benchmarks/honest_bench/REPORT.md); fibonacci, object allocation & array write from the [polyglot sweep](benchmarks/polyglot/RESULTS.md) (Perry default mode, no fast-math). +Sources: convolution & JSON from the [systems-language report](https://github.com/PerryTS/perry/blob/7beb3a50ca/benchmarks/honest_bench/REPORT.md) +— pinned to `7beb3a50ca`, the last revision whose artifact contains these values; the copy at HEAD was regenerated in #7641 on a host +where the harness clock was invalid and now reads 0.0 ms in every timing cell (see this PR). Re-pin to `main` once it is regenerated. +Fibonacci, object allocation & array write from the [polyglot sweep](benchmarks/polyglot/RESULTS.md) (Perry default mode, no fast-math). We publish *everything*, including the workloads where V8's JIT still beats us — no cherry-picked table can survive an open harness. Run it yourself: `./benchmarks/run_public_baseline.sh` ([methodology](benchmarks/README.md)). diff --git a/benchmarks/honest_bench/harness/run_bench.sh b/benchmarks/honest_bench/harness/run_bench.sh index 93e6c01f71..bdbb09f941 100755 --- a/benchmarks/honest_bench/harness/run_bench.sh +++ b/benchmarks/honest_bench/harness/run_bench.sh @@ -81,6 +81,24 @@ PYTHON local wall_ns=$((end_ns - start_ns)) + # A measured wall time must be strictly positive: `start_ns` and `end_ns` + # bracket a child process that actually ran. A non-positive delta means the + # clock pair is not comparable (e.g. the two `python3` samples did not share + # a reference point), so the sample carries no timing information at all. + # + # Without this gate such a sample is written to results.json, survives + # report.py's `exit_code == 0` filter, and is published as a median. That is + # how the 2026-08-08 regeneration (#7641) shipped 150 negative wall_ms rows + # and a REPORT.md whose every timing cell reads 0.0 ms. + if (( wall_ns <= 0 )); then + echo "run_bench.sh: implausible wall time for ${WORKLOAD}/${LANGUAGE} run ${run}:" >&2 + echo " start_ns=${start_ns} end_ns=${end_ns} delta=${wall_ns} ns" >&2 + echo " The monotonic clock samples are not comparable across processes on" >&2 + echo " this host. Refusing to record a timing-free sample." >&2 + rm -f "$tmp_out" "$tmp_err" + exit 3 + fi + local stdout_first stdout_last stdout_first=$(head -1 "$tmp_out" 2>/dev/null | head -c 200 || true) stdout_last=$(tail -1 "$tmp_out" 2>/dev/null | head -c 200 || true) diff --git a/benchmarks/honest_bench/scripts/report.py b/benchmarks/honest_bench/scripts/report.py index 3fbdfdedf8..405daec9eb 100644 --- a/benchmarks/honest_bench/scripts/report.py +++ b/benchmarks/honest_bench/scripts/report.py @@ -111,6 +111,23 @@ def stats_for(rows): wall = [r["wall_ms"] for r in rows if r["exit_code"] == 0] rss = [r["max_rss_kb"] for r in rows if r["exit_code"] == 0] failures = sum(1 for r in rows if r["exit_code"] != 0) + + # `exit_code == 0` alone is not enough to make a sample meaningful: a run + # can succeed (correct stdout, correct checksum) and still carry a wall + # time that is physically impossible. Report those loudly instead of + # taking their median -- a median of zeros renders as a confident "0.0 ms" + # table cell and reads exactly like a real result. + implausible = [w for w in wall if w <= 0] + if implausible: + raise SystemExit( + f"report.py: {len(implausible)} of {len(wall)} successful runs have a " + f"non-positive wall_ms (min {min(implausible)}).\n" + " results.json carries no usable timing data; a report generated from " + "it would be fiction.\n" + " Re-run ./run.sh on this host and check that run_bench.sh's monotonic " + "clock samples are comparable across processes." + ) + if not wall: return None, None, None, None, failures return ( diff --git a/benchmarks/polyglot/bench.rs b/benchmarks/polyglot/bench.rs index aa7b531e40..4b1e633e21 100644 --- a/benchmarks/polyglot/bench.rs +++ b/benchmarks/polyglot/bench.rs @@ -1,6 +1,7 @@ +use std::hint::black_box; use std::time::Instant; -fn fib(n: i32) -> i32 { +fn fib(n: i64) -> i64 { if n < 2 { return n; } @@ -9,7 +10,7 @@ fn fib(n: i32) -> i32 { fn bench_fibonacci() { let start = Instant::now(); - let result = fib(40); + let result = fib(black_box(40)); let elapsed = start.elapsed().as_millis(); println!("fibonacci:{}", elapsed); println!(" checksum: {}", result); @@ -28,6 +29,15 @@ fn bench_loop_overhead() { fn bench_array_write() { let mut arr = vec![0.0_f64; 10_000_000]; + // suite/03_array_write.ts fills every slot BEFORE it calls Date.now(), so + // its timed loop overwrites resident pages. `vec![0.0; N]` is calloc, whose + // pages are mapped lazily -- without this pre-touch the Rust timed loop + // additionally pays ~10M first-touch faults the TS timed loop does not, + // and the column measures page-fault cost rather than store throughput. + for slot in arr.iter_mut() { + *slot = 0.0; + } + black_box(&arr); let start = Instant::now(); for i in 0..10_000_000 { arr[i] = i as f64; @@ -72,10 +82,12 @@ fn bench_object_create() { let start = Instant::now(); let mut sum: f64 = 0.0; for i in 0..1_000_000 { - let p = Point { + // Without the barrier LLVM proves the struct never escapes and removes + // the allocation entirely, reporting 0 ms for a loop that never ran. + let p = black_box(Point { x: i as f64, y: i as f64 * 2.0, - }; + }); sum += p.x + p.y; } let elapsed = start.elapsed().as_millis();