From 7a0ca47308babc4bcd2abee17271a7f6568380b2 Mon Sep 17 00:00:00 2001 From: Lukas Friedrich Date: Mon, 31 Aug 2026 21:43:37 +0200 Subject: [PATCH] Add `markproof init`, and the files a public repo is expected to have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI module docstring had promised `init` since M4. It did not exist, which tells a reader the documentation is not checked against the code — the wrong first impression for a tool that sells verification. The audit rated adoption weak partly on this. `markproof init` writes a starting config. Deliberately small: one chat probe, with media, UI and text marking commented out and the reason each is optional. A scaffold full of settings nobody chose is how people end up running checks they cannot interpret. It refuses to clobber an existing file, and it points a first-time user at examples/demo-bot, because someone evaluating this tool may not have an endpoint of their own yet. Six tests, the load-bearing one being that the scaffold actually loads through `load_config` and names a rulepack that ships. A starting config that fails validation would be worse than no command. Also the community files the repository lacked. Two of them are shaped by what this project is rather than copied from a template: The "a verdict looks wrong" issue form asks which direction the verdict went, and names a PASS-that-should-have-failed as the outcome nothing downstream will catch. It asks for report.json specifically, because since #25 that file carries the rulepack digest and the probe URLs, so a run can be reconstructed. A second form exists for "a rule reads the law wrong", pointing at docs/RULES_SOURCES.md first. Legal disagreement is the most valuable contribution this project can get, and the rulepacks are CC-BY and cite their paragraphs precisely so that argument can be had in the open. The code of conduct says the thing that actually matters here — argue with the work, and say what you actually verified rather than what you believe — and is honest that this is maintained by one person, so some good contributions will not be merged. Co-Authored-By: Claude Opus 5 --- .github/ISSUE_TEMPLATE/config.yml | 8 +++ .github/ISSUE_TEMPLATE/false-verdict.yml | 50 +++++++++++++ .github/ISSUE_TEMPLATE/rulepack.yml | 30 ++++++++ .github/PULL_REQUEST_TEMPLATE.md | 26 +++++++ CHANGELOG.md | 11 +++ CITATION.cff | 26 +++++++ CODE_OF_CONDUCT.md | 60 ++++++++++++++++ README.md | 1 + src/markproof/cli.py | 92 +++++++++++++++++++++++- tests/test_cli.py | 60 ++++++++++++++++ 10 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/false-verdict.yml create mode 100644 .github/ISSUE_TEMPLATE/rulepack.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CITATION.cff create mode 100644 CODE_OF_CONDUCT.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..db6e79b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Security vulnerability + url: https://github.com/Tippel-AI/markproof/security/advisories/new + about: Report privately. Signing keys and report integrity are in scope — see SECURITY.md. + - name: Is my system in scope of Article 50? + url: https://digital-strategy.ec.europa.eu/en/policies/regulatory-framework-ai + about: markproof is a conformance test, not legal advice. That question is for a lawyer. diff --git a/.github/ISSUE_TEMPLATE/false-verdict.yml b/.github/ISSUE_TEMPLATE/false-verdict.yml new file mode 100644 index 0000000..96b243e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/false-verdict.yml @@ -0,0 +1,50 @@ +name: A verdict looks wrong +description: markproof reported PASS, FAIL, WARN or SKIP where you expected something else. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + This is the most important kind of report this project can get. A tool that + decides compliance questions has one job, and a wrong verdict is the whole + failure — especially a **PASS that should have been a FAIL**, which is the + outcome nothing downstream will catch. + - type: input + id: rule + attributes: + label: Which rule + placeholder: MPF-D-001 + validations: {required: true} + - type: dropdown + id: direction + attributes: + label: What happened + options: + - PASS where I expected FAIL + - FAIL where I expected PASS + - WARN that fires on every target (carries no information) + - SKIP where I expected a verdict + - Something else + validations: {required: true} + - type: textarea + id: evidence + attributes: + label: The report, or the console output + description: | + `report.json` is ideal — it carries the rulepack digest and the probe URLs, + so the run can be reconstructed. Redact anything sensitive; the digests are + what matter. + render: json + validations: {required: true} + - type: textarea + id: expected + attributes: + label: What you expected, and why + description: If an Article 50 reading is involved, cite it. Disagreeing with the rulepack is a legitimate report. + validations: {required: true} + - type: input + id: version + attributes: + label: Version + description: "`markproof --version`, or the commit" + validations: {required: true} diff --git a/.github/ISSUE_TEMPLATE/rulepack.yml b/.github/ISSUE_TEMPLATE/rulepack.yml new file mode 100644 index 0000000..7d83093 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rulepack.yml @@ -0,0 +1,30 @@ +name: A rule reads the law wrong +description: The rulepack's interpretation of Article 50 is mistaken, outdated, or too broad. +labels: ["rulepack"] +body: + - type: markdown + attributes: + value: | + Rulepacks are published under CC-BY-4.0 and cite the paragraph they rest on, + specifically so this argument can be had in the open. `docs/RULES_SOURCES.md` + records the reasoning behind each rule, including the obligations deliberately + left unimplemented — worth reading first, it may already say why. + + Legal disagreement is welcome. This project does not claim to be right about + Article 50; it claims to be explicit about what it checks. + - type: input + id: rule + attributes: + label: Which rule (or which one is missing) + validations: {required: true} + - type: textarea + id: reading + attributes: + label: The reading you think is correct + description: Cite the Article, and the Guidelines paragraph if there is one. + validations: {required: true} + - type: textarea + id: consequence + attributes: + label: What changes if you are right + description: Does a verdict flip? Does a rule stop applying? Does a new one become checkable? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d3104f6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ + + +## What changes, and why + +## Does this change any verdict? + + + +- [ ] No verdict changes +- [ ] Verdicts change, and the golden diff is in this PR + +## Checks + +- [ ] `pytest` passes +- [ ] `ruff check .` and `mypy` pass +- [ ] Docs updated if the change is user-visible diff --git a/CHANGELOG.md b/CHANGELOG.md index bd9faa8..07dbc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,17 @@ explicitly: the **rulepack format** — what a rulepack file may contain — and ### Added +- **`markproof init`** writes a starting `markproof.yaml`. The CLI's own docstring + had promised this command since M4 and the build did not have it. The scaffold + configures one chat probe and leaves media, UI and text marking commented out + with the reason each is optional — a config full of settings nobody chose is how + people end up running checks they cannot interpret. + +- **PDF output is reachable** (#21). `report.formats` accepts `pdf` and + `pdf-html`; both renderers existed, were tested, and could not be produced by any + invocation of the shipped tool. `report.sign_key` and `report.output_dir` are + honoured too — they were validated and then ignored. + - **Targets can declare which Article 50 obligations bind them** (#14). A new `applicability` block in `markproof.yaml` maps obligations — `ai-interaction`, `synthetic-media-marking`, `synthetic-text-marking`, diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..76c312c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,26 @@ +cff-version: 1.2.0 +title: "markproof: deterministic CI checks for EU AI Act Article 50" +message: "If you cite this tool or its rulepacks, please cite it as below." +type: software +authors: + - family-names: Friedrich + given-names: Lukas + affiliation: Tippel +repository-code: "https://github.com/Tippel-AI/markproof" +license: Apache-2.0 +abstract: >- + markproof calls a running GenAI endpoint the way a user would and checks what + arrives: whether an interactive system discloses its artificial nature, whether + delivered media still carry a valid C2PA manifest declaring an AI source type, + and whether generated text still carries the operator's own watermark. The + evaluation path contains no language model, so identical evidence produces + identical verdicts, and every run emits a canonically serialised, Ed25519-signed + report that a third party can verify offline. Rulepacks are versioned, cite the + paragraph they implement, and ship under CC-BY-4.0. +keywords: + - EU AI Act + - Article 50 + - transparency + - C2PA + - SynthID + - conformance testing diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c953229 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,60 @@ + + +# Code of conduct + +## The short version + +Be straightforward with people and rigorous about claims. Those are different +things, and this project needs both. + +## What is expected + +**Argue with the work, not the person.** Disagreement about whether a rule reads +Article 50 correctly is the most valuable contribution this project can receive, +and it is welcome sharply. "This rule is wrong because paragraph 117 says X" is +exactly right. "Whoever wrote this has not read the Act" is not, and it is also +weaker — it gives the reader nothing to check. + +**Say what you actually verified.** This is a tool about not overclaiming, so the +same standard applies to the conversation around it. "I ran it and it failed" and +"I think it would fail" are different sentences, and mixing them up costs +everyone's time. + +**Assume the other person read the Act too, and reached a different conclusion.** +Article 50 is genuinely ambiguous in places. `docs/RULES_SOURCES.md` records where, +and what this project decided. If you think a decision is wrong, the reasoning is +written down so you can argue with it specifically. + +**Take a correction well, and give one plainly.** Being wrong in public about a +compliance question is not a character flaw; leaving a wrong claim standing +because it is uncomfortable to withdraw is worse. + +## What is not acceptable + +Harassment, personal attacks, demeaning comments about anyone's identity, or +sustained disruption. Also: knowingly asserting something you have not checked as +though you had, which in a project like this is a form of harm rather than a +faux pas. + +## Scope + +Anywhere the project happens — issues, pull requests, discussions, and in +communication about the project elsewhere. + +## Reporting + +Email **lukas.friedrich@tippel.ai**. Reports are read by one person and stay +confidential. You will get an acknowledgement within a few days, and I will tell +you what I decided. + +Maintainers may remove comments, close threads, or block accounts. That is a +judgement call by a person, not a process, and I will say why when I make one. + +## A note on the maintainer + +This project is maintained by one person alongside other work. Response times will +sometimes be slow, and some good contributions will not be merged because keeping +this small is itself a design decision. Neither is a slight. diff --git a/README.md b/README.md index 6523560..4b36bdc 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ markproof calls your *running* AI endpoint the way a user would, and checks what ```bash pipx install git+https://github.com/Tippel-AI/markproof +markproof init # writes a starting markproof.yaml markproof run --config markproof.yaml ``` diff --git a/src/markproof/cli.py b/src/markproof/cli.py index c9f2bf8..3bbeab1 100644 --- a/src/markproof/cli.py +++ b/src/markproof/cli.py @@ -8,7 +8,13 @@ ``rules list`` / ``rules schema`` List the rules of a rulepack; export the pydantic schemas as JSON Schema. -``verify-report``, ``keygen`` and ``init`` arrive with the signed report in M4. +``verify-report`` + Check a report's signature. The auditor's command: it needs the report, a + public key and this tool, on a machine that never saw the system under test. +``keygen`` + Generate an Ed25519 key pair for signing. +``init`` + Write a starting ``markproof.yaml``. The module-level object ``app`` is the console-script target declared in ``pyproject.toml`` (``markproof = "markproof.cli:app"``). @@ -541,6 +547,90 @@ def verify_report_command( raise typer.Exit(code=0 if valid else 1) +_SCAFFOLD = """\ +# SPDX-License-Identifier: Apache-2.0 +# +# Written by `markproof init`. Point it at the endpoint your users actually +# reach — a staging copy proves nothing about production. +version: 1 + +target: + name: {name} + probes: + - id: chat + type: http-chat + url: {url} + dialect: openai-chat # or generic-json + response_path + lang: de # de | en + # auth: {{header: Authorization, env: MARKPROOF_TOKEN}} + +# Which Article 50 obligations bind this target. Everything not listed is +# checked, so silence never removes a rule. A `false` here is a claim, not a mute +# switch: it travels into the signed report, so a green run states its own scope. +# +# ai-interaction · synthetic-media-marking · synthetic-text-marking +# emotion-recognition · deepfake-labelling · public-interest-text +# applicability: +# deepfake-labelling: false + +# Verifying text marking needs the parameters you generate with. Without them +# the text rule skips and says so — it never guesses. +# text_marking: +# method: synthid +# watermark_config: secrets/watermark_config.json + +rulepack: art50-eu-2026.07 + +report: + formats: [json, summary] # add `pdf` for the artefact an auditor reads + # sign_key: env:MARKPROOF_SIGNING_KEY +""" + + +@app.command() +def init( + path: Annotated[Path, typer.Option("--config", "-c", help="Where to write the config.")] = Path( + "markproof.yaml" + ), + url: Annotated[ + str, + typer.Option("--url", help="The chat endpoint to probe."), + ] = "https://api.example.com/v1/chat/completions", + name: Annotated[str, typer.Option("--name", help="A name for the target.")] = "my-service", + force: Annotated[bool, typer.Option("--force", help="Overwrite an existing file.")] = False, +) -> None: + """Write a starting markproof.yaml. + + Deliberately small. The scaffold configures one chat probe and leaves the + media, UI and text-marking blocks commented out with the reason each is + optional, because a config full of settings nobody chose is how people end up + running checks they cannot interpret. + """ + if path.exists() and not force: + err_console.print( + f"[bold red]error:[/] {path} already exists. Pass --force to overwrite it." + ) + raise typer.Exit(code=2) + + try: + path.write_text(_SCAFFOLD.format(name=name, url=url), encoding="utf-8") + except OSError as exc: + err_console.print(f"[bold red]error:[/] could not write {path}: {exc}") + raise typer.Exit(code=2) from exc + + console.print() + console.print(f" wrote [cyan]{path}[/cyan]") + console.print() + console.print(" Next:") + console.print(f" 1. set the url in {path} to the endpoint your users reach") + console.print(f" 2. [cyan]markproof run --config {path}[/cyan]") + console.print() + console.print( + " [dim]No endpoint yet? examples/demo-bot is a deliberately non-conformant one.[/dim]" + ) + console.print() + + @app.command() def keygen( out_dir: Annotated[ diff --git a/tests/test_cli.py b/tests/test_cli.py index 2faad08..26df03e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -440,3 +440,63 @@ def test_the_config_can_name_the_environment_variable( assert result.exit_code == 0, result.output report = json.loads((out / "report.json").read_text(encoding="utf-8")) assert "signature" in report, "the configured key was ignored" + + +class TestInit: + """The command the module docstring promised and the build did not have. + + A promised-but-absent command is worse than no command: it tells a reader the + documentation is not checked against the code, which for a tool selling + verification is the wrong first impression. + """ + + def test_it_writes_a_config_that_actually_loads(self, tmp_path: Path) -> None: + """A scaffold that fails validation would be worse than none at all.""" + from markproof.config import load_config + + target = tmp_path / "markproof.yaml" + result = RUNNER.invoke(app, ["init", "--config", str(target)]) + assert result.exit_code == 0, result.output + config = load_config(target) + assert config.target.probes + assert config.rulepack == "art50-eu-2026.07" + + def test_the_rulepack_it_names_is_one_that_ships(self, tmp_path: Path) -> None: + from markproof.config import load_config + + target = tmp_path / "markproof.yaml" + RUNNER.invoke(app, ["init", "--config", str(target)]) + packaged = Path(__file__).resolve().parent.parent / "src" / "markproof" / "rulepacks" + assert (packaged / f"{load_config(target).rulepack}.yaml").is_file() + + def test_it_refuses_to_clobber_an_existing_config(self, tmp_path: Path) -> None: + target = tmp_path / "markproof.yaml" + target.write_text("version: 1\n# hand-written\n", encoding="utf-8") + result = RUNNER.invoke(app, ["init", "--config", str(target)]) + assert result.exit_code == 2 + assert "hand-written" in target.read_text(encoding="utf-8") + + def test_force_overwrites(self, tmp_path: Path) -> None: + target = tmp_path / "markproof.yaml" + target.write_text("old\n", encoding="utf-8") + result = RUNNER.invoke(app, ["init", "--config", str(target), "--force"]) + assert result.exit_code == 0 + assert "old" not in target.read_text(encoding="utf-8") + + def test_the_url_and_name_reach_the_file(self, tmp_path: Path) -> None: + from markproof.config import load_config + + target = tmp_path / "markproof.yaml" + RUNNER.invoke( + app, + ["init", "--config", str(target), "--url", "https://x.test/v1/c", "--name", "prod-bot"], + ) + config = load_config(target) + assert config.target.name == "prod-bot" + assert config.target.probes[0].url == "https://x.test/v1/c" + + def test_it_points_somewhere_a_newcomer_can_actually_go(self, tmp_path: Path) -> None: + result = RUNNER.invoke(app, ["init", "--config", str(tmp_path / "markproof.yaml")]) + assert "demo-bot" in result.output, ( + "a first-time user with no endpoint of their own needs somewhere to point this" + )