From d7d68dfcb5b232225ac6975bd918a08804a4bdd6 Mon Sep 17 00:00:00 2001 From: godofecht Date: Fri, 21 Aug 2026 11:32:00 +0100 Subject: [PATCH] benchmarks: disparity dimensions mean divergence, not existence Every one of the 19 canonical rows was flagged as having a tracked disparity, which made the flag meaningless. GaussianNB iris was the clearest casualty: fitted priors and classes exactly equal, every state diff at f32-ulp scale (theta frobenius 5e-8), score within 2.8e-8 of sklearn against a declared tolerance of 1e-3, and it still carried numerical, model-state and runtime dimensions. Two causes, both the existence-vs-divergence disease this repo has now hit three times (covered_rows > 0 in #429, the win/loss prose in #436): - "numerical" fired on any nonzero float difference. It now uses the disparity-gate precedent from #401: noise below max(1e-6, 1% of the row's declared tolerance) is not a finding. 18 rows flagged -> 3, and the three survivors are the genuine seed and tie-break gaps on the digits tree rows plus LogisticRegression. - "model-state" tested the diagnostics dict for truthiness, so the moment #402 took coverage to 19/19, every row became "disparate" by virtue of being measured. It now requires actual divergence: any first_divergent_index >= 0 (exact by construction), or a relative diff above 1e-5 whose paired absolute diff also clears 1e-7, so a near-zero denominator cannot promote pure noise. 19 rows -> 13. "runtime" is unchanged: a 37x speedup genuinely is a runtime difference, and tracking favourable differences is the report's stated purpose. Because runtime differs on every row, counts gain rows_with_substantive_disparity (dimensions beyond runtime, currently 14/19), and the benchmarks page card now shows that number with an honest label instead of the always-19/19 tracked count. The old count field is kept for compatibility; publish validation and the disparity.yml assertions were re-run against a regenerated report and pass. GaussianNB iris now reads ["runtime"] alone, which is the truth: the fitted models match to the last float digit and only the speed differs. Generated artifacts are not committed; freeze-results refreshes disparity_report.json on main. Co-Authored-By: Claude Fable 5 --- benchmarks/generate_disparity_report.py | 38 +++++++++++++++++++++++-- docs/benchmarks.html | 2 +- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/benchmarks/generate_disparity_report.py b/benchmarks/generate_disparity_report.py index 944915c..9a32d69 100644 --- a/benchmarks/generate_disparity_report.py +++ b/benchmarks/generate_disparity_report.py @@ -118,6 +118,31 @@ def enrich_state_from_raw_details( + +# Relative diffs are scale-free; 1e-5 sits two orders above f32 ulp noise in +# accumulated norms. A relative diff on a near-zero quantity can explode on +# pure noise, so it only counts when its paired absolute diff also clears a +# floor. A non-negative first_divergent_index is exact divergence by +# construction and needs no floor. +_STATE_REL_FLOOR = 1e-5 +_STATE_ABS_FLOOR = 1e-7 + + +def _model_state_diverges(state: dict) -> bool: + if not state: + return False + for key, value in state.items(): + if key.endswith("_first_divergent_index"): + if isinstance(value, (int, float)) and value >= 0: + return True + elif key.endswith("_relative_diff"): + if not isinstance(value, (int, float)) or value <= _STATE_REL_FLOOR: + continue + paired = state.get(key[: -len("_relative_diff")] + "_abs_diff") + if paired is None or (isinstance(paired, (int, float)) and paired > _STATE_ABS_FLOOR): + return True + return False + def _total_ms(row: dict, side: str): """Fit plus predict for one side of a headline row, or None if unresolved.""" fit = row.get(f"{side}_fit_ms") @@ -198,13 +223,21 @@ def main() -> int: tolerance_fraction = score_diff / effective_score_tol if effective_score_tol > 0 else None dimensions = [] - if score_diff > 0: + # Floors, so a dimension means divergence rather than existence. + # Without them every row was flagged: any nonzero f32 score diff set + # "numerical" (2.8e-8 against a declared tolerance of 1e-3), and + # "model-state" tested the diagnostics dict for truthiness, so the + # moment coverage reached 19/19 every row became "disparate". The + # numerical floor follows the disparity-gate precedent: noise below + # max(1e-6, 1% of the row's declared tolerance) is not a finding. + numerical_floor = max(1e-6, 0.01 * score_tol) + if score_diff > numerical_floor: dimensions.append("numerical") if config: dimensions.append("configuration") if semantic: dimensions.append("semantic") - if state: + if _model_state_diverges(state): dimensions.append("model-state") if diag.get("parity_status") != row.get("parity_status"): dimensions.append("strict-vs-final-parity-decision") @@ -249,6 +282,7 @@ def main() -> int: "counts": { "rows": len(rows), "rows_with_tracked_disparity": sum(r["has_tracked_disparity"] for r in rows), + "rows_with_substantive_disparity": sum(1 for r in rows if any(d != "runtime" for d in r["disparity_dimensions"])), "rows_with_configuration_difference": sum(bool(r["configuration_differences"]) for r in rows), "rows_with_semantic_difference": sum(bool(r["semantic_differences"]) for r in rows), "rows_with_model_state_diagnostics": sum(bool(r["model_state_diagnostics"]) for r in rows), diff --git a/docs/benchmarks.html b/docs/benchmarks.html index bc1913c..74136a9 100644 --- a/docs/benchmarks.html +++ b/docs/benchmarks.html @@ -1,4 +1,4 @@ -Benchmarks — flow-scikit
flow~scikit

canonical v2 / parity + disparity benchmark

Eligibility never means identity.

All 19 canonical rows are measured and currently eligible for comparison, but numerical, semantic and runtime disparities remain first-class evidence. This page renders the committed benchmark and disparity artifacts directly so differences cannot disappear merely because a row passes its contract.

Flow wins—

End-to-end fit + predict comparisons won by Flow.

sklearn wins—

End-to-end comparisons won by scikit-learn.

parity eligible—

Rows admitted to the competitive denominator.

tracked disparities—

Rows with non-zero numerical or explicit semantic/configuration differences.

TIMING_UNIT|msend-to-endseed=4280/20 persisted split2% practical tie thresholddisparity retained after eligibility
KMeans note: Digits KMeans is eligible under the same declared contract as every other clustering row. Its seeded k-means++ initialization now matches scikit-learn's, so the strict diagnostic and the final eligibility decision agree. The convergence statistic, the point at which inertia is reported, empty-cluster relocation and the n_init selection rule still differ and stay visible in the disparity artifact.

runtime overview

runtime overview

The plots are generated from the canonical JSON.

Each runtime plot shows end-to-end fit + predict time on a log scale. The plots use the same rows as the table below and therefore update whenever the frozen canonical result changes.

All 19 speed ratios

scikit-learn total time divided by Flow total time. The vertical 1× line separates Flow wins from scikit-learn wins.

Iris total runtime

scikit-learnFlow

Digits total runtime

scikit-learnFlow

Diabetes total runtime

scikit-learnFlow

persistent disparity

Passing parity does not erase the gap.

The disparity plot normalizes each row's principal numerical difference against its effective tolerance where a tolerance is available. A value near 1 means the row is close to the acceptance boundary. Semantic/configuration differences are tracked in the same artifact and remain visible in the table.

Numerical disparity relative to tolerance

The dashed line is the acceptance boundary. Values can remain non-zero even for eligible rows.

all canonical rows

No selected-win table.

Every row is shown below. Speedup is sklearn_ms / flow_ms; values above 1× favor Flow. Strict diagnostic status is kept separate from final eligibility.

AlgorithmDatasetFinal parityStrict diagnosticWinnerscore |Δ|sklearn msFlow msspeedup

methodology

Correctness, disparity and timing are separate dimensions.

The benchmark consumes the same persisted train/test indices in Python and Flow. Python uses high-resolution adaptive timing and the canonical runner aggregates repeated process measurements with medians and IQR. Flow timings are emitted in milliseconds and aggregated by the same runner.

Supervised rows compare predictive metrics under declared tolerances. PCA additionally checks explained variance, singular values, reconstruction error and sign-aligned components. KMeans uses permutation-invariant clustering quality and inertia. The persistent disparity artifact preserves raw numerical gaps and known semantic/configuration differences even after the estimator-specific eligibility contract succeeds.

historical deployment evidence

Footprint and startup remain separate experiments.

The repository also contains a historical deployment comparison recording a roughly 1.4 MB Flow native executable and a roughly 65× cold-start advantage (33 ms versus 2160 ms). Those figures come from a different deployment experiment and are intentionally not mixed into the canonical estimator timing denominator.

trajectory

Each implementation's own timings, across freezes.

Every other figure on this page is a ratio of Flow to scikit-learn. A ratio cannot distinguish Flow getting faster from scikit-learn getting slower, and it hides the case where both move together. These are the absolute per-row totals recorded at each freeze.

reproduce

Read the source artifacts.

Canonical result ↗ Disparity report ↗