diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index cf5fe2c..60a6708 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -49,6 +49,7 @@ jobs: python benchmarks/verify_fixtures.py python benchmarks/test_generate_headline_summary.py python benchmarks/test_disparity_regression_policy.py + python benchmarks/test_config_equivalence.py python benchmarks/publish_headline_v2.py --check python benchmarks/validate_execution_substrate.py python -m py_compile \ @@ -65,6 +66,8 @@ jobs: benchmarks/check_flow_regression.py \ benchmarks/check_disparity_regression.py \ benchmarks/generate_disparity_report.py \ + benchmarks/config_equivalence.py \ + benchmarks/test_config_equivalence.py \ benchmarks/test_disparity_regression_policy.py \ benchmarks/publish_headline_v2.py \ benchmarks/validate_execution_substrate.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 55c7b2a..ab1920d 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -28,6 +28,30 @@ Canonical v2 consumes persisted train/test fixtures shared by Python and Flow. C Digits KMeans is classified as `approximately equivalent` under the tolerances declared in `parity_contract.json`, with no estimator-specific exception. [`audit_kmeans_semantics.py`](audit_kmeans_semantics.py) re-derives Flow's initial centre indices from a Python mirror of Flow's own MT19937 and greedy k-means++ and checks them against scikit-learn's initializer for all ten `n_init` restarts. It also records the differences that survive that alignment: the convergence statistic, the point at which inertia is reported, empty-cluster relocation and the `n_init` selection rule. Each was substituted in turn and none moves a canonical row. +## Configuration comparison + +Each contract row records its configuration twice, once in Flow's vocabulary and +once in scikit-learn's. Comparing the two dictionaries key by key reported a +difference every time the two projects spelled the same setting differently, and +the differences that were real sat buried among them. + +[`config_equivalence.py`](config_equivalence.py) holds the equivalences the +comparator may apply, and `generate_disparity_report.py` records every applied +one in that row's `configuration_equivalences` so nothing is dropped silently. +Three rules: + +| Rule | Example | Condition | +| --- | --- | --- | +| Cross-vocabulary mapping | sklearn `C=1.0` against Flow `l2=0.008333333` | the recorded numbers satisfy `l2 = 1 / (C * n_train)` to 1e-5 relative, with `n_train` read from `split_indices.json` | +| Absent equals explicitly disabled | Flow `penalty=none` on LinearRegression | one side records the feature off and the other has no such parameter | +| Solver-private parameter | Flow `learning_rate` against sklearn lbfgs; sklearn `dual` against Flow's LinearSVC | the knob belongs to one side's solver and the counterpart's solver does not have it | + +All three apply only to a parameter present on exactly one side. When both sides +record a parameter, the values are compared and any difference is reported, so +`max_iter 200 vs 1000` and `optimizer lbfgs_no_line_search vs lbfgs` stay +visible. A cross-vocabulary mapping that does not hold numerically is reported +with the value the relation demanded attached; that is the mismatch #430 found. + ## Learned-state diagnostics A score tolerance says two implementations agree on the answer. It says nothing diff --git a/benchmarks/config_equivalence.py b/benchmarks/config_equivalence.py new file mode 100644 index 0000000..8ebf20e --- /dev/null +++ b/benchmarks/config_equivalence.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Declared equivalences between Flow and scikit-learn configuration vocabularies. + +The canonical benchmark records each row's configuration twice, once in Flow's +vocabulary and once in scikit-learn's, in `parity_contract.json`. Comparing the +two dictionaries key by key reports a difference whenever the two projects spell +the same setting differently, whenever one side declares a knob the other's +solver does not have, and whenever one side explicitly turns a feature off that +the other simply lacks. None of those is a configuration difference, and while +they sat in the report the differences that are real were buried in them. + +This module holds the equivalences the comparator is allowed to apply. Three +rules, in order: + +1. Cross-vocabulary mappings, each verified numerically. `C` on the sklearn side + and `l2` on the Flow side name the same setting under a conversion, so they + are equivalent only when the recorded numbers actually satisfy it. A mapping + that does not hold is still reported, with the expected value attached. +2. Absent against explicitly disabled. A parameter one side records as off and + the other does not have at all is the same configuration. +3. Solver-private parameters. A knob that belongs to one side's solver and has + no counterpart in the other's is not comparable. + +All three rules apply only to a parameter that appears on exactly one side, or +to a mapped pair each of whose names appears on exactly its own side. When both +sides record the same parameter, the recorded values are compared and any +difference is reported. That is what keeps `max_iter 200 vs 1000` and +`optimizer lbfgs_no_line_search vs lbfgs` visible. + +Nothing is discarded: every applied equivalence is returned alongside the +differences so the report can carry it as evidence. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable + +MISSING = "" + +# Keys under which a row's configuration may name its solver. Used to check that +# a solver-private parameter really is absent from the counterpart's solver +# rather than merely left out of its record. +SOLVER_PARAMETER_KEYS = ("optimizer", "solver") + +# String values that declare a feature switched off. `False` is deliberately not +# here: a boolean knob set to false still implies the knob exists, and a boolean +# on one side against nothing on the other is worth looking at. +DISABLED_STRINGS = ("none", "None") + +# Relative tolerance for a verified cross-vocabulary conversion. +# +# The contract records converted values rounded to nine significant digits, so +# the recorded 0.000695894 sits 3.2e-07 from the exact 1/1437 it stands for. +# 1e-05 clears that rounding by a decade and a half and still sits far below any +# real mismatch: the disagreement #430 found was 8.3x, not parts per million. +CONVERSION_RELATIVE_TOLERANCE = 1e-5 + + +@dataclass(frozen=True) +class CrossVocabularyMapping: + """One setting recorded under two names, plus the conversion between them.""" + + sklearn_parameter: str + flow_parameter: str + # Expected Flow value given the sklearn value and the row's training-set + # size. Returns None when the inputs cannot produce a conversion. + convert: Callable[[object, "int | None"], "float | None"] + relation: str + why: str + + +def _alpha_from_c(c_value, n_train): + """Flow's `l2` strength equivalent to scikit-learn's `C`. + + Established in #430, closing #408. `logistic_regression_fit` minimises + `(1/m) * sum_i logloss_i + 0.5 * alpha * ||w||^2` while scikit-learn + minimises `sum_i logloss_i + 0.5 * ||w||^2 / C`. Dividing the second by `m` + makes the two agree exactly when `alpha = 1 / (C * m)`, so the training-set + size is part of the conversion and no single constant serves two datasets. + """ + if n_train is None or n_train <= 0: + return None + try: + c = float(c_value) + except (TypeError, ValueError): + return None + if c <= 0.0: + return None + return 1.0 / (c * float(n_train)) + + +CROSS_VOCABULARY_MAPPINGS = ( + CrossVocabularyMapping( + sklearn_parameter="C", + flow_parameter="l2", + convert=_alpha_from_c, + relation="l2 = 1 / (C * n_train)", + why=( + "flow-scikit states regularization as an explicit penalty strength on a " + "mean-loss objective; scikit-learn states it as the inverse strength C on a " + "sum-loss objective. #430 derived the conversion and it carries n_train." + ), + ), +) + + +@dataclass(frozen=True) +class SolverPrivateParameter: + """A knob that exists only for the solver that declares it.""" + + parameter: str + # Solvers that do expose this knob. If the counterpart side names one of + # these as its solver and still omits the parameter, the omission is a real + # difference and stays reported. + owned_by: frozenset + why: str + + +SOLVER_PRIVATE_PARAMETERS = { + entry.parameter: entry + for entry in ( + SolverPrivateParameter( + parameter="learning_rate", + owned_by=frozenset({"gradient_descent", "sgd", "adam"}), + why=( + "a step size exists only for a first-order iterative solver. Every " + "scikit-learn counterpart in the canonical set solves without one: " + "LogisticRegression uses line-searched lbfgs, Ridge a direct " + "factorization, Lasso coordinate descent." + ), + ), + SolverPrivateParameter( + parameter="dual", + owned_by=frozenset({"liblinear"}), + why=( + "scikit-learn's LinearSVC picks between the primal and dual liblinear " + "formulations. flow-scikit implements one formulation and has no such " + "switch, so there is nothing on its side for the value to disagree with." + ), + ), + ) +} + +# `max_iter` is deliberately absent from the table above. scikit-learn's Ridge +# does take a max_iter, and an iteration budget is exactly the kind of setting +# #469 asks to keep visible, so a one-sided max_iter stays reported. + + +def _declared_solver(config): + for solver_key in SOLVER_PARAMETER_KEYS: + value = config.get(solver_key) + if isinstance(value, str): + return value + return None + + +def _is_disabled(value) -> bool: + if isinstance(value, bool): + return False + if value is None: + return True + return isinstance(value, str) and value in DISABLED_STRINGS + + +def _relative_difference(a: float, b: float) -> float: + return abs(a - b) / max(abs(a), abs(b), 1e-300) + + +def compare_configs(flow: dict, sklearn: dict, n_train=None): + """Compare two configuration records under the declared equivalences. + + Returns `(differences, equivalences)`. `differences` keeps the historical + `{"parameter", "flow", "sklearn"}` shape, sorted by parameter name; entries + produced by a cross-vocabulary mapping that failed verification carry an + extra `equivalence` field naming the relation that should have held. + """ + flow = dict(flow or {}) + sklearn = dict(sklearn or {}) + differences = [] + equivalences = [] + resolved = set() + + # Rule 1: cross-vocabulary mappings, verified numerically. + for mapping in CROSS_VOCABULARY_MAPPINGS: + flow_key = mapping.flow_parameter + sklearn_key = mapping.sklearn_parameter + # Applies only when each name sits on exactly its own side. If either + # project also records the other's name the two vocabularies are + # directly comparable and the plain key comparison below handles them. + if flow_key in sklearn or sklearn_key in flow: + continue + if flow_key not in flow or sklearn_key not in sklearn: + continue + + flow_value = flow[flow_key] + sklearn_value = sklearn[sklearn_key] + expected = mapping.convert(sklearn_value, n_train) + relative = None + if expected is not None: + try: + relative = _relative_difference(float(flow_value), expected) + except (TypeError, ValueError): + relative = None + holds = relative is not None and relative <= CONVERSION_RELATIVE_TOLERANCE + + resolved.add(flow_key) + resolved.add(sklearn_key) + + if holds: + equivalences.append({ + "rule": "cross-vocabulary mapping", + "parameter": flow_key, + "flow_parameter": flow_key, + "flow": flow_value, + "sklearn_parameter": sklearn_key, + "sklearn": sklearn_value, + "relation": mapping.relation, + "n_train": n_train, + "expected_flow_value": expected, + "relative_difference": relative, + "why": mapping.why, + }) + continue + + if expected is None: + note = ( + f"{mapping.relation} could not be evaluated " + f"(n_train={n_train}, {sklearn_key}={sklearn_value!r})" + ) + else: + note = ( + f"{mapping.relation} does not hold: n_train={n_train} and " + f"{sklearn_key}={sklearn_value!r} give {flow_key}={expected!r}, " + f"recorded {flow_value!r}" + ) + differences.append({ + "parameter": flow_key, + "flow": flow_value, + "sklearn": MISSING, + "equivalence": note, + }) + differences.append({ + "parameter": sklearn_key, + "flow": MISSING, + "sklearn": sklearn_value, + "equivalence": note, + }) + + for key in sorted((set(flow) | set(sklearn)) - resolved): + flow_value = flow.get(key, MISSING) + sklearn_value = sklearn.get(key, MISSING) + if flow_value == sklearn_value: + continue + + if (key in flow) != (key in sklearn): + present_side = "flow" if key in flow else "sklearn" + present_value = flow_value if key in flow else sklearn_value + counterpart = sklearn if key in flow else flow + + # Rule 2: absent against explicitly disabled. + if _is_disabled(present_value): + equivalences.append({ + "rule": "absent equals explicitly disabled", + "parameter": key, + "declared_by": present_side, + "value": present_value, + "why": ( + f"{present_side} records {key}={present_value!r}, which declares the " + "feature off; the other side has no such parameter. Same configuration." + ), + }) + continue + + # Rule 3: solver-private parameters. + private = SOLVER_PRIVATE_PARAMETERS.get(key) + if private is not None: + counterpart_solver = _declared_solver(counterpart) + if counterpart_solver is None or counterpart_solver not in private.owned_by: + equivalences.append({ + "rule": "solver-private parameter", + "parameter": key, + "declared_by": present_side, + "value": present_value, + "counterpart_solver": counterpart_solver, + "why": private.why, + }) + continue + + differences.append({"parameter": key, "flow": flow_value, "sklearn": sklearn_value}) + + differences.sort(key=lambda entry: entry["parameter"]) + equivalences.sort(key=lambda entry: entry.get("parameter", "")) + return differences, equivalences diff --git a/benchmarks/generate_disparity_report.py b/benchmarks/generate_disparity_report.py index 9a32d69..b3f6a34 100644 --- a/benchmarks/generate_disparity_report.py +++ b/benchmarks/generate_disparity_report.py @@ -5,6 +5,12 @@ raw numerical differences, contract tolerances, configuration differences, semantic exceptions, learned-state diagnostics and runtime ratios visible after a row becomes eligible. + +Configuration is compared through the declared equivalences in +`config_equivalence.py`, so a setting the two projects record under different +names stops reading as a difference. Every applied equivalence is written into +the row's `configuration_equivalences`, which keeps the evidence rather than +dropping it. """ from __future__ import annotations @@ -13,18 +19,33 @@ import math from pathlib import Path +from config_equivalence import compare_configs + ROOT = Path(__file__).resolve().parent BASE_DIAGNOSTIC_FIELDS = {"algorithm", "dataset", "metric", "parity_status", "score_abs_diff"} -def config_diff(flow: dict, sklearn: dict) -> list[dict]: - out = [] - for key in sorted(set(flow) | set(sklearn)): - fv = flow.get(key, "") - sv = sklearn.get(key, "") - if fv != sv: - out.append({"parameter": key, "flow": fv, "sklearn": sv}) - return out +def load_train_sizes(path: Path) -> dict[str, int]: + """Training-set size per dataset, from the canonical split fixture. + + Some cross-vocabulary conversions carry n_train (`l2 = 1 / (C * n_train)`), + so the comparator needs the real split rather than a constant per dataset + name. Missing or unreadable fixtures leave the size unknown, which makes + those conversions unverifiable and keeps the pair reported. + """ + if not path.exists(): + return {} + data = json.loads(path.read_text()) + sizes: dict[str, int] = {} + for dataset, row in data.items(): + if not isinstance(row, dict): + continue + n_train = row.get("n_train") + if n_train is None and isinstance(row.get("train_idx"), list): + n_train = len(row["train_idx"]) + if isinstance(n_train, int): + sizes[dataset] = n_train + return sizes def model_state_diagnostics(diag: dict) -> dict: @@ -160,6 +181,7 @@ def main() -> int: p.add_argument("--sklearn-raw", type=Path, default=ROOT / "sklearn_results_v2.txt") p.add_argument("--flow-raw", type=Path, default=ROOT / "flow_results_v2.txt") p.add_argument("--host-environment", type=Path, default=ROOT / "headline_environment.json") + p.add_argument("--splits", type=Path, default=ROOT / "split_indices.json") p.add_argument("--output", type=Path, default=ROOT / "disparity_report.json") args = p.parse_args() @@ -170,6 +192,7 @@ def main() -> int: sklearn_details = parse_details(args.sklearn_raw) flow_details = parse_details(args.flow_raw) host_env = json.loads(args.host_environment.read_text()) if args.host_environment.exists() else {} + train_sizes = load_train_sizes(args.splits) diag_by_key = {(r["algorithm"], r["dataset"], r["metric"]): r for r in diagnostics} contract_by_key = {(r["algorithm"], r["dataset"], r["metric"]): r for r in contract_doc["rows"]} @@ -211,7 +234,11 @@ def main() -> int: }, ]) - config = config_diff(contract.get("flow", {}), contract.get("sklearn", {})) + config, config_equivalences = compare_configs( + contract.get("flow", {}), + contract.get("sklearn", {}), + train_sizes.get(row["dataset"]), + ) state = enrich_state_from_raw_details( key, model_state_diagnostics(diag), @@ -263,6 +290,10 @@ def main() -> int: "sklearn_total_ms": _total_ms(row, "sklearn"), "runtime_log2_ratio": runtime_log2_ratio, "configuration_differences": config, + # Settings the two projects record under different names, or that + # only one side's solver has. Kept rather than dropped so removing + # them from `configuration_differences` hides nothing. + "configuration_equivalences": config_equivalences, "semantic_differences": semantic, "model_state_diagnostics": state, "diagnostics": {k: v for k, v in diag.items() if k not in BASE_DIAGNOSTIC_FIELDS}, @@ -271,7 +302,7 @@ def main() -> int: }) payload = { - "schema_version": 3, + "schema_version": 4, "environment_id": headline.get("environment_id"), # Hardware plus BLAS thread configuration. environment_id only covers the # software stack, so it cannot tell two runner machines apart; wall-clock @@ -284,6 +315,7 @@ def main() -> int: "rows_with_tracked_disparity": sum(r["has_tracked_disparity"] for r in rows), "rows_with_substantive_disparity": sum(1 for r in rows if any(d != "runtime" for d in r["disparity_dimensions"])), "rows_with_configuration_difference": sum(bool(r["configuration_differences"]) for r in rows), + "rows_with_configuration_equivalence": sum(bool(r["configuration_equivalences"]) for r in rows), "rows_with_semantic_difference": sum(bool(r["semantic_differences"]) for r in rows), "rows_with_model_state_diagnostics": sum(bool(r["model_state_diagnostics"]) for r in rows), "strict_final_status_disagreements": sum(r["strict_diagnostic_status"] != r["final_parity_status"] for r in rows), diff --git a/benchmarks/test_config_equivalence.py b/benchmarks/test_config_equivalence.py new file mode 100644 index 0000000..076ebcd --- /dev/null +++ b/benchmarks/test_config_equivalence.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Pin the behaviour of the configuration comparator. + +Two halves, and both matter: + +* A setting recorded under two vocabularies must stop being reported as a + difference. That is #469. +* A real divergence must still be reported. The unverified conversion, the + two-sided value mismatch and the solver that does own the knob hold that line. + #430 exists because a silent regularization mismatch survived unnoticed, so a + mapping that does not hold numerically has to come back as a difference. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from config_equivalence import compare_configs + +ROOT = Path(__file__).resolve().parent +FAILURES: list[str] = [] + +IRIS_N_TRAIN = 120 +DIGITS_N_TRAIN = 1437 + + +def check(name: str, actual, expected) -> None: + if actual == expected: + print(f" ok {name}") + return + FAILURES.append(name) + print(f" FAIL {name}\n expected {expected!r}\n actual {actual!r}") + + +def parameters(entries) -> list[str]: + return [e["parameter"] for e in entries] + + +def rules(entries) -> dict[str, str]: + return {e["parameter"]: e["rule"] for e in entries} + + +def test_cross_vocabulary_mapping() -> None: + print("cross-vocabulary mapping C <-> l2") + + for n_train, l2 in ((IRIS_N_TRAIN, 0.008333333), (DIGITS_N_TRAIN, 0.000695894)): + diff, equiv = compare_configs({"l2": l2}, {"C": 1.0}, n_train) + check(f"verified conversion at n_train={n_train} reports nothing", diff, []) + check(f"verified conversion at n_train={n_train} is recorded", rules(equiv), + {"l2": "cross-vocabulary mapping"}) + + diff, equiv = compare_configs({"l2": 0.01}, {"C": 1.0}, IRIS_N_TRAIN) + check("a conversion that does not hold is reported", parameters(diff), ["C", "l2"]) + check("the failed conversion is not recorded as an equivalence", equiv, []) + check("the failure names the relation", + all("l2 = 1 / (C * n_train)" in e["equivalence"] for e in diff), True) + + # The strength #430 found in the benchmark before the fix: 0.001 against + # C=1.0 on iris, 8.3x weaker than the sklearn model beside it. + diff, _ = compare_configs({"l2": 0.001}, {"C": 1.0}, IRIS_N_TRAIN) + check("the pre-#430 mismatch is reported", parameters(diff), ["C", "l2"]) + + diff, equiv = compare_configs({"l2": 0.008333333}, {"C": 1.0}, None) + check("an unknown n_train leaves the conversion unverifiable", parameters(diff), ["C", "l2"]) + check("an unverifiable conversion is not an equivalence", equiv, []) + + diff, equiv = compare_configs({"l2": 0.008333333}, {"C": 1.0}, DIGITS_N_TRAIN) + check("the wrong dataset's n_train does not verify", parameters(diff), ["C", "l2"]) + + # Both sides speak the same vocabulary: compare the values directly. + diff, equiv = compare_configs({"l2": 0.5, "C": 1.0}, {"C": 1.0, "l2": 0.25}, IRIS_N_TRAIN) + check("a two-sided l2 is compared directly", parameters(diff), ["l2"]) + check("a two-sided l2 applies no mapping", equiv, []) + + +def test_absent_equals_explicitly_disabled() -> None: + print("absent equals explicitly disabled") + + diff, equiv = compare_configs({"penalty": "none"}, {}, None) + check("penalty=none against nothing reports nothing", diff, []) + check("penalty=none is recorded", rules(equiv), {"penalty": "absent equals explicitly disabled"}) + + diff, _ = compare_configs({"penalty": "l2"}, {}, None) + check("a penalty that is on is still reported", parameters(diff), ["penalty"]) + + diff, _ = compare_configs({"fit_intercept": False}, {}, None) + check("a false boolean is not an absent knob", parameters(diff), ["fit_intercept"]) + + diff, _ = compare_configs({"penalty": "none"}, {"penalty": "l2"}, None) + check("a two-sided penalty is compared directly", parameters(diff), ["penalty"]) + + +def test_solver_private_parameters() -> None: + print("solver-private parameters") + + diff, equiv = compare_configs( + {"optimizer": "lbfgs_no_line_search", "learning_rate": 0.1}, + {"optimizer": "lbfgs"}, + None, + ) + check("learning_rate against an lbfgs counterpart reports only the optimizer", + parameters(diff), ["optimizer"]) + check("learning_rate is recorded as solver-private", + rules(equiv), {"learning_rate": "solver-private parameter"}) + + diff, _ = compare_configs( + {"optimizer": "gradient_descent"}, + {"optimizer": "gradient_descent", "learning_rate": 0.1}, + None, + ) + check("a solver that does own the knob keeps the omission visible", + parameters(diff), ["learning_rate"]) + + diff, equiv = compare_configs({"C": 1.0}, {"C": 1.0, "dual": "auto"}, None) + check("dual against a side with no such switch reports nothing", diff, []) + check("dual is recorded as solver-private", rules(equiv), {"dual": "solver-private parameter"}) + + diff, _ = compare_configs({"learning_rate": 0.1}, {"learning_rate": 0.01}, None) + check("a two-sided learning_rate is compared directly", parameters(diff), ["learning_rate"]) + + diff, _ = compare_configs({"max_iter": 1000}, {}, None) + check("a one-sided max_iter is still reported", parameters(diff), ["max_iter"]) + + diff, _ = compare_configs({"max_iter": 200}, {"max_iter": 1000}, None) + check("a two-sided max_iter mismatch is still reported", parameters(diff), ["max_iter"]) + check("the reported max_iter carries both values", + diff, [{"parameter": "max_iter", "flow": 200, "sklearn": 1000}]) + + +def test_canonical_contract() -> None: + """What survives on the real canonical rows.""" + print("canonical parity_contract.json rows") + + contract = json.loads((ROOT / "parity_contract.json").read_text()) + splits = json.loads((ROOT / "split_indices.json").read_text()) + sizes = {name: row["n_train"] for name, row in splits.items()} + + expected = { + ("LogisticRegression", "iris"): ["max_iter", "optimizer"], + ("LogisticRegression", "digits"): ["max_iter", "optimizer"], + ("KernelSVC_RBF", "iris"): ["max_iter"], + ("PCA", "iris"): ["solver"], + ("Ridge", "diabetes"): ["max_iter"], + ("LinearSVC", "iris"): [], + ("LinearSVC", "digits"): [], + ("Lasso", "diabetes"): [], + ("LinearRegression", "diabetes"): [], + } + + actual = {} + for row in contract["rows"]: + diff, _ = compare_configs(row.get("flow", {}), row.get("sklearn", {}), + sizes.get(row["dataset"])) + key = (row["algorithm"], row["dataset"]) + if key in expected or diff: + actual[key] = parameters(diff) + + check("surviving configuration differences on the canonical rows", actual, expected) + + unresolved = sum(1 for key, params in actual.items() if params) + check("rows still carrying a configuration difference", unresolved, 5) + + +def main() -> int: + test_cross_vocabulary_mapping() + test_absent_equals_explicitly_disabled() + test_solver_private_parameters() + test_canonical_contract() + + if FAILURES: + print(f"config equivalence fixtures: FAIL ({len(FAILURES)})") + for name in FAILURES: + print(" -", name) + return 1 + print("config equivalence fixtures: PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/test_disparity_regression_policy.py b/benchmarks/test_disparity_regression_policy.py index 73c9b0e..3ea6dbc 100644 --- a/benchmarks/test_disparity_regression_policy.py +++ b/benchmarks/test_disparity_regression_policy.py @@ -44,7 +44,7 @@ def row(algorithm="Ridge", dataset="diabetes", metric="r2", score_abs_diff=1.79e def report(rows, environment_id="env", runtime_environment_id="host-a"): return { - "schema_version": 3, + "schema_version": 4, "environment_id": environment_id, "runtime_environment_id": runtime_environment_id, "counts": {"rows": len(rows)},