From 74459703aae4382422db8ca0cca51e35c8b76b5c Mon Sep 17 00:00:00 2001 From: Lukas Friedrich Date: Tue, 1 Sep 2026 07:09:30 +0200 Subject: [PATCH] Qualify a passing marking rule where the reader sees it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #19. Article 50(2) asks for output to be marked **and** detectable as artificially generated, and the Commission Guidelines say satisfying one limb does not discharge the other. markproof measures the first, against the operator's own watermark configuration. It cannot measure the second, and this is not a gap to be closed later: whether a third party who does not hold those keys can detect the mark is a property of the ecosystem, not of the endpoint under test, and no probe run against a system can establish it. So "MPF-T-001 PASS" is a smaller statement than it looks, and the risk is a reader concluding Article 50(2) is satisfied. docs/DISCLAIMER.md said so already, which is the wrong place: nobody reading a report opens the disclaimer of the tool that produced it. The Markdown summary and the PDF now carry the qualification immediately after the verdict — the PDF especially, because that is the artefact handed to an auditor and therefore where the over-reading happens. Both renderers state it in the same words, and a test asserts they cannot drift apart, because a reader comparing the two should find no difference to interpret. Only on a pass, and only for the two marking obligations. Nobody over-reads a failure, and a note printed on every run is one more line people learn to skip — the same defect as a warning that fires for every target. The gap is live rather than theoretical. Text-watermark detection tooling is largely announced rather than shipped, and where it exists it is key-gated, which the Guidelines treat as the weaker of the two footings. Co-Authored-By: Claude Opus 5 --- src/markproof/report/pdf_reportlab.py | 40 +++++++++++++ src/markproof/report/summary.py | 31 ++++++++++ src/markproof/rules/schema.py | 18 ++++++ tests/test_report.py | 83 ++++++++++++++++++++++++++- 4 files changed, 171 insertions(+), 1 deletion(-) diff --git a/src/markproof/report/pdf_reportlab.py b/src/markproof/report/pdf_reportlab.py index c1b7e24..56d3721 100644 --- a/src/markproof/report/pdf_reportlab.py +++ b/src/markproof/report/pdf_reportlab.py @@ -222,12 +222,32 @@ class FindingView: detail: tuple[tuple[str, str], ...] = () evidence_sha256: tuple[str, ...] = () + obligation: str = "" + """Which Article 50 duty this finding serves, when the report records one. + + Defaulted, like everything else this module reads: it renders whatever shape + it is handed, including reports written by a build that had no such field. + """ + @property def palette(self) -> tuple[str, str]: """Ink and wash for this result; unknown labels stay neutral grey.""" return RESULT_PALETTE.get(self.result, _UNKNOWN_PALETTE) +#: The qualification a passing marking rule needs, without its Markdown emphasis — +#: this page renders its own. Kept beside the summary's wording deliberately: the +#: two artefacts state the same limit, and a reader comparing them should find no +#: difference to interpret. +MARKING_LIMB_NOTE = ( + "The marking checks above measure whether the mark arrived, against your own " + "configuration. They do not measure whether a third party can detect it — that " + "is a property of the ecosystem, not of your endpoint, and no probe run against " + "your system can establish it. A passing marking check is not, on its own, " + "Article 50(2) compliance." +) + + def _join_names(names: list[str]) -> str: """Join obligation names so the sentence reads as English.""" if len(names) <= 1: @@ -257,6 +277,13 @@ class ReportView: rulepack_version: str generated_at: str markproof_version: str + marking_passed: bool = False + """Whether any Article 50(2) marking rule passed. + + Drives the two-limbs qualification. This page is what somebody hands an + auditor, so a reader concluding "marking: PASS, therefore Article 50(2) + satisfied" would do it here rather than anywhere else.""" + findings: tuple[FindingView, ...] = () declared_scope: tuple[tuple[str, bool], ...] = () """Obligations the target declared, and whether each was said to apply. @@ -323,6 +350,7 @@ def _finding_view(finding: Any) -> FindingView: rule_id=_string(finding, "rule_id", default="(unknown rule)"), title=_string(finding, "title"), article=_string(finding, "article"), + obligation=_string(finding, "obligation"), guideline_ref=_string(finding, "guideline_ref"), probe_id=_string(finding, "probe_id"), result=_string(finding, "result", default="SKIP").upper(), @@ -406,6 +434,9 @@ def report_view(report: Any) -> ReportView: default="(unknown)", ), findings=findings, + marking_passed=any( + f.result.upper() == "PASS" and "marking" in (f.obligation or "") for f in findings + ), declared_scope=_declared_scope(report), provenance=provenance, attribution=_string(report, "rulepack_attribution", "attribution", "rulepack.attribution"), @@ -803,6 +834,15 @@ def _story(rl: Any, view: ReportView, width: float) -> list[Any]: ) ) + if view.marking_passed: + story.append(rl.Spacer(1, 10)) + story.append( + rl.Paragraph( + "Article 50(2) has two limbs. " + _esc(MARKING_LIMB_NOTE), + styles["body"], + ) + ) + story.append(rl.Spacer(1, 18)) story.append(rl.Paragraph("Findings", styles["h2"])) diff --git a/src/markproof/report/summary.py b/src/markproof/report/summary.py index a216c7b..f93c437 100644 --- a/src/markproof/report/summary.py +++ b/src/markproof/report/summary.py @@ -52,6 +52,36 @@ def _verdict_line(report: Report) -> str: return f"**All {summary.passed} applicable checks passed.**" +#: Printed wherever a marking rule passes. Not a disclaimer in a footer: the risk +#: is a reader concluding from "MPF-T-001 PASS" that Article 50(2) is satisfied, +#: so the qualification has to sit where that conclusion is drawn. +MARKING_LIMB_NOTE = ( + "**Article 50(2) has two limbs.** The marking checks above measure whether the " + "mark arrived, against your own configuration. They do not measure whether a " + "third party can detect it — that is a property of the ecosystem, not of your " + "endpoint, and no probe run against your system can establish it. A passing " + "marking check is not, on its own, Article 50(2) compliance." +) + + +def _marking_note(report: Report) -> list[str]: + """The Article 50(2) qualification, when a marking rule actually passed. + + Only on a pass. A reader whose marking check failed or skipped is not at risk + of over-reading it, and a note that appears unconditionally is one more line + people learn to skip — the same failure as a warning that fires for every + target. + """ + passed = [ + f + for f in report.findings + if f.result is Result.PASS and f.obligation is not None and f.obligation.is_marking + ] + if not passed: + return [] + return ["", MARKING_LIMB_NOTE, ""] + + def _join(names: list[str]) -> str: """Backtick the names and join them so the sentence reads as English.""" quoted = [f"`{n}`" for n in names] @@ -151,6 +181,7 @@ def render_summary(report: Report) -> str: else: lines.append("_No rule in this pack applied to the configured probes._") + lines.extend(_marking_note(report)) lines.extend(["", "---", ""]) if report.signature is not None: diff --git a/src/markproof/rules/schema.py b/src/markproof/rules/schema.py index 7b65e1e..23b05e0 100644 --- a/src/markproof/rules/schema.py +++ b/src/markproof/rules/schema.py @@ -107,6 +107,24 @@ class Obligation(StrEnum): """Art. 50(4) second subparagraph — disclosure for published text informing on matters of public interest.""" + @property + def is_marking(self) -> bool: + """Whether this duty is one of Article 50(2)'s two limbs. + + Article 50(2) asks for output to be marked **and** detectable as + artificially generated, and the Commission Guidelines say satisfying one + does not discharge the other. markproof measures the first against the + operator's own configuration. It cannot measure the second: whether a + third party who does not hold those keys can detect the mark is a property + of the ecosystem, not of the endpoint under test, and no probe run against + a system can establish it. + + So a passing marking rule is a smaller statement than it looks, and the + renderers use this to say so next to the verdict — where the reader is, + rather than in a README they will never open. + """ + return self in (Obligation.SYNTHETIC_MEDIA_MARKING, Obligation.SYNTHETIC_TEXT_MARKING) + class Applicability(RootModel[dict[Obligation, bool]]): """The operator's declaration of which obligations bind this target. diff --git a/tests/test_report.py b/tests/test_report.py index f626947..035921c 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -28,7 +28,7 @@ ) from markproof.report.summary import render_summary from markproof.rules.engine import Finding, Result -from markproof.rules.schema import Rulepack +from markproof.rules.schema import Obligation, Rulepack, load_rulepack _TIMESTAMP = "2026-08-31T12:00:00+00:00" @@ -240,3 +240,84 @@ def test_pipes_in_messages_do_not_break_the_table(self, rulepack: Rulepack) -> N def test_disclaimer_is_always_present(self, report: Report) -> None: assert "not legal advice" in render_summary(report) + + +class TestTheMarkingLimbIsQualified: + """Issue #19: a passing marking rule is a smaller statement than it looks. + + Article 50(2) asks for output to be marked **and** detectable as artificially + generated, and the Guidelines are explicit that satisfying one limb does not + discharge the other. markproof measures the first against the operator's own + configuration. It cannot measure the second — whether a third party without + those keys can detect the mark is a property of the ecosystem, not of the + endpoint, and no probe run against a system can establish it. + + That gap is live rather than theoretical: text-watermark detection tooling is + largely announced rather than shipped, and where it exists it is key-gated. So + the qualification belongs next to the verdict, not in a README the reader of a + report will never open. + """ + + @staticmethod + def _report(result: Result, obligation: Obligation | None) -> Report: + packaged = Path(__file__).resolve().parent.parent / "src" / "markproof" / "rulepacks" + finding = Finding( + rule_id="MPF-T-001", + title="Generated text carries the operator's declared watermark", + article="Art. 50(2)", + obligation=obligation, + guideline_ref=None, + probe_id="chat", + result=result, + message="watermark detected", + ) + return build_report( + target="t", + rulepack=load_rulepack(packaged / "art50-eu-2026.07.yaml"), + findings=[finding], + timestamp="2026-09-01T12:00:00+00:00", + ) + + def test_a_passing_marking_rule_carries_the_qualification(self) -> None: + summary = render_summary(self._report(Result.PASS, Obligation.SYNTHETIC_TEXT_MARKING)) + assert "two limbs" in summary + assert "not, on its own, Article 50(2) compliance" in summary + + def test_media_marking_too(self) -> None: + summary = render_summary(self._report(Result.PASS, Obligation.SYNTHETIC_MEDIA_MARKING)) + assert "two limbs" in summary + + def test_a_failing_marking_rule_does_not(self) -> None: + """Nobody over-reads a failure, and a note on every run is one people skip.""" + summary = render_summary(self._report(Result.FAIL, Obligation.SYNTHETIC_TEXT_MARKING)) + assert "two limbs" not in summary + + def test_a_disclosure_rule_does_not(self) -> None: + """Article 50(1) has one limb; qualifying it would be noise.""" + summary = render_summary(self._report(Result.PASS, Obligation.AI_INTERACTION)) + assert "two limbs" not in summary + + def test_the_pdf_says_the_same_thing(self) -> None: + """The PDF is what gets handed to an auditor, so it is where over-reading happens.""" + from markproof.report.pdf_reportlab import report_view + + view = report_view( + self._report(Result.PASS, Obligation.SYNTHETIC_TEXT_MARKING).model_dump( + mode="json", exclude_none=True + ) + ) + assert view.marking_passed + + clean = report_view( + self._report(Result.PASS, Obligation.AI_INTERACTION).model_dump( + mode="json", exclude_none=True + ) + ) + assert not clean.marking_passed + + def test_both_renderers_state_the_same_limit(self) -> None: + """A reader comparing the two artefacts must find no difference to interpret.""" + from markproof.report import pdf_reportlab, summary + + stripped = summary.MARKING_LIMB_NOTE.replace("**Article 50(2) has two limbs.** ", "") + assert stripped == pdf_reportlab.MARKING_LIMB_NOTE