From 178fefd6eb73a54dd175d8e12739496856e7e4cf Mon Sep 17 00:00:00 2001 From: highlander Date: Fri, 21 Aug 2026 14:55:18 -0500 Subject: [PATCH] fix(report): the console summary counted the wrong skips render() computes the catalog's own skip count, then the scope paragraph rebound the same name to the run-wide census: skipped = sum(... for t in s[5] if _lookup(...) == 'skip') # catalog ... skipped = JUNIT_CENSUS['skipped'] # whole run so the line CI operators actually read -- N sections, 374 tests (323 passed, 0 failed, 3 skipped, 48 pending) printed the whole run's skips inside a breakdown of the catalog. The four numbers did not add up to the total, and the error ran in the alarming direction: it inflates skips, which reads as "lots of this was not exercised" against a catalog where only three entries were actually gated. Use the census value inline where the paragraph needs it and leave `skipped` meaning one thing. Added the assertion that would have caught it, since the breakdown is only ever right when it reconciles: assert passed + failed + skipped + missing == total Now: 374 tests (323 passed, 0 failed, 3 skipped, 48 pending) -> 374. --- scripts/generate-test-report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/generate-test-report.py b/scripts/generate-test-report.py index faac2daf..789a9595 100644 --- a/scripts/generate-test-report.py +++ b/scripts/generate-test-report.py @@ -2845,13 +2845,12 @@ def _section_state(s): ran = JUNIT_CENSUS['ran'] if ran: pb.gap(3) - skipped = JUNIT_CENSUS['skipped'] for line in _w('Scope: this report is a curated catalog of %d tests. The CI run collected %d ' '(%d of them native firmware unit tests); %d SKIPPED and did not execute, ' 'usually because the emulator predates the firmware the test targets -- a skip ' 'is not evidence the feature works. Absence from this report is NOT ' 'evidence that a feature is untested -- check the JUnit artifacts.' - % (total, ran, JUNIT_CENSUS['native'], skipped), 100): + % (total, ran, JUNIT_CENSUS['native'], JUNIT_CENSUS['skipped']), 100): pb.text(8, line, color=GRAY) pb.gap(6) pb.text(12, 'Sections', bold=True) @@ -2994,6 +2993,9 @@ def _section_state(s): pb.finish() pdf.write(output_path) + assert passed + failed + skipped + missing == total, ( + 'catalog counts do not reconcile: %d+%d+%d+%d != %d' + % (passed, failed, skipped, missing, total)) print(f'{output_path}: fw={fw_version}, {len(active)} sections, {total} tests ' f'({passed} passed, {failed} failed, {skipped} skipped, {missing} pending)')