diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..db6e79b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Security vulnerability + url: https://github.com/Tippel-AI/markproof/security/advisories/new + about: Report privately. Signing keys and report integrity are in scope — see SECURITY.md. + - name: Is my system in scope of Article 50? + url: https://digital-strategy.ec.europa.eu/en/policies/regulatory-framework-ai + about: markproof is a conformance test, not legal advice. That question is for a lawyer. diff --git a/.github/ISSUE_TEMPLATE/false-verdict.yml b/.github/ISSUE_TEMPLATE/false-verdict.yml new file mode 100644 index 0000000..96b243e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/false-verdict.yml @@ -0,0 +1,50 @@ +name: A verdict looks wrong +description: markproof reported PASS, FAIL, WARN or SKIP where you expected something else. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + This is the most important kind of report this project can get. A tool that + decides compliance questions has one job, and a wrong verdict is the whole + failure — especially a **PASS that should have been a FAIL**, which is the + outcome nothing downstream will catch. + - type: input + id: rule + attributes: + label: Which rule + placeholder: MPF-D-001 + validations: {required: true} + - type: dropdown + id: direction + attributes: + label: What happened + options: + - PASS where I expected FAIL + - FAIL where I expected PASS + - WARN that fires on every target (carries no information) + - SKIP where I expected a verdict + - Something else + validations: {required: true} + - type: textarea + id: evidence + attributes: + label: The report, or the console output + description: | + `report.json` is ideal — it carries the rulepack digest and the probe URLs, + so the run can be reconstructed. Redact anything sensitive; the digests are + what matter. + render: json + validations: {required: true} + - type: textarea + id: expected + attributes: + label: What you expected, and why + description: If an Article 50 reading is involved, cite it. Disagreeing with the rulepack is a legitimate report. + validations: {required: true} + - type: input + id: version + attributes: + label: Version + description: "`markproof --version`, or the commit" + validations: {required: true} diff --git a/.github/ISSUE_TEMPLATE/rulepack.yml b/.github/ISSUE_TEMPLATE/rulepack.yml new file mode 100644 index 0000000..7d83093 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rulepack.yml @@ -0,0 +1,30 @@ +name: A rule reads the law wrong +description: The rulepack's interpretation of Article 50 is mistaken, outdated, or too broad. +labels: ["rulepack"] +body: + - type: markdown + attributes: + value: | + Rulepacks are published under CC-BY-4.0 and cite the paragraph they rest on, + specifically so this argument can be had in the open. `docs/RULES_SOURCES.md` + records the reasoning behind each rule, including the obligations deliberately + left unimplemented — worth reading first, it may already say why. + + Legal disagreement is welcome. This project does not claim to be right about + Article 50; it claims to be explicit about what it checks. + - type: input + id: rule + attributes: + label: Which rule (or which one is missing) + validations: {required: true} + - type: textarea + id: reading + attributes: + label: The reading you think is correct + description: Cite the Article, and the Guidelines paragraph if there is one. + validations: {required: true} + - type: textarea + id: consequence + attributes: + label: What changes if you are right + description: Does a verdict flip? Does a rule stop applying? Does a new one become checkable? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d3104f6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ + + +## What changes, and why + +## Does this change any verdict? + + + +- [ ] No verdict changes +- [ ] Verdicts change, and the golden diff is in this PR + +## Checks + +- [ ] `pytest` passes +- [ ] `ruff check .` and `mypy` pass +- [ ] Docs updated if the change is user-visible diff --git a/CHANGELOG.md b/CHANGELOG.md index bd9faa8..07dbc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,17 @@ explicitly: the **rulepack format** — what a rulepack file may contain — and ### Added +- **`markproof init`** writes a starting `markproof.yaml`. The CLI's own docstring + had promised this command since M4 and the build did not have it. The scaffold + configures one chat probe and leaves media, UI and text marking commented out + with the reason each is optional — a config full of settings nobody chose is how + people end up running checks they cannot interpret. + +- **PDF output is reachable** (#21). `report.formats` accepts `pdf` and + `pdf-html`; both renderers existed, were tested, and could not be produced by any + invocation of the shipped tool. `report.sign_key` and `report.output_dir` are + honoured too — they were validated and then ignored. + - **Targets can declare which Article 50 obligations bind them** (#14). A new `applicability` block in `markproof.yaml` maps obligations — `ai-interaction`, `synthetic-media-marking`, `synthetic-text-marking`, diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..76c312c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,26 @@ +cff-version: 1.2.0 +title: "markproof: deterministic CI checks for EU AI Act Article 50" +message: "If you cite this tool or its rulepacks, please cite it as below." +type: software +authors: + - family-names: Friedrich + given-names: Lukas + affiliation: Tippel +repository-code: "https://github.com/Tippel-AI/markproof" +license: Apache-2.0 +abstract: >- + markproof calls a running GenAI endpoint the way a user would and checks what + arrives: whether an interactive system discloses its artificial nature, whether + delivered media still carry a valid C2PA manifest declaring an AI source type, + and whether generated text still carries the operator's own watermark. The + evaluation path contains no language model, so identical evidence produces + identical verdicts, and every run emits a canonically serialised, Ed25519-signed + report that a third party can verify offline. Rulepacks are versioned, cite the + paragraph they implement, and ship under CC-BY-4.0. +keywords: + - EU AI Act + - Article 50 + - transparency + - C2PA + - SynthID + - conformance testing diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c953229 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,60 @@ + + +# Code of conduct + +## The short version + +Be straightforward with people and rigorous about claims. Those are different +things, and this project needs both. + +## What is expected + +**Argue with the work, not the person.** Disagreement about whether a rule reads +Article 50 correctly is the most valuable contribution this project can receive, +and it is welcome sharply. "This rule is wrong because paragraph 117 says X" is +exactly right. "Whoever wrote this has not read the Act" is not, and it is also +weaker — it gives the reader nothing to check. + +**Say what you actually verified.** This is a tool about not overclaiming, so the +same standard applies to the conversation around it. "I ran it and it failed" and +"I think it would fail" are different sentences, and mixing them up costs +everyone's time. + +**Assume the other person read the Act too, and reached a different conclusion.** +Article 50 is genuinely ambiguous in places. `docs/RULES_SOURCES.md` records where, +and what this project decided. If you think a decision is wrong, the reasoning is +written down so you can argue with it specifically. + +**Take a correction well, and give one plainly.** Being wrong in public about a +compliance question is not a character flaw; leaving a wrong claim standing +because it is uncomfortable to withdraw is worse. + +## What is not acceptable + +Harassment, personal attacks, demeaning comments about anyone's identity, or +sustained disruption. Also: knowingly asserting something you have not checked as +though you had, which in a project like this is a form of harm rather than a +faux pas. + +## Scope + +Anywhere the project happens — issues, pull requests, discussions, and in +communication about the project elsewhere. + +## Reporting + +Email **lukas.friedrich@tippel.ai**. Reports are read by one person and stay +confidential. You will get an acknowledgement within a few days, and I will tell +you what I decided. + +Maintainers may remove comments, close threads, or block accounts. That is a +judgement call by a person, not a process, and I will say why when I make one. + +## A note on the maintainer + +This project is maintained by one person alongside other work. Response times will +sometimes be slow, and some good contributions will not be merged because keeping +this small is itself a design decision. Neither is a slight. diff --git a/README.md b/README.md index 6523560..4b36bdc 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ markproof calls your *running* AI endpoint the way a user would, and checks what ```bash pipx install git+https://github.com/Tippel-AI/markproof +markproof init # writes a starting markproof.yaml markproof run --config markproof.yaml ``` diff --git a/src/markproof/cli.py b/src/markproof/cli.py index c9f2bf8..3bbeab1 100644 --- a/src/markproof/cli.py +++ b/src/markproof/cli.py @@ -8,7 +8,13 @@ ``rules list`` / ``rules schema`` List the rules of a rulepack; export the pydantic schemas as JSON Schema. -``verify-report``, ``keygen`` and ``init`` arrive with the signed report in M4. +``verify-report`` + Check a report's signature. The auditor's command: it needs the report, a + public key and this tool, on a machine that never saw the system under test. +``keygen`` + Generate an Ed25519 key pair for signing. +``init`` + Write a starting ``markproof.yaml``. The module-level object ``app`` is the console-script target declared in ``pyproject.toml`` (``markproof = "markproof.cli:app"``). @@ -541,6 +547,90 @@ def verify_report_command( raise typer.Exit(code=0 if valid else 1) +_SCAFFOLD = """\ +# SPDX-License-Identifier: Apache-2.0 +# +# Written by `markproof init`. Point it at the endpoint your users actually +# reach — a staging copy proves nothing about production. +version: 1 + +target: + name: {name} + probes: + - id: chat + type: http-chat + url: {url} + dialect: openai-chat # or generic-json + response_path + lang: de # de | en + # auth: {{header: Authorization, env: MARKPROOF_TOKEN}} + +# Which Article 50 obligations bind this target. Everything not listed is +# checked, so silence never removes a rule. A `false` here is a claim, not a mute +# switch: it travels into the signed report, so a green run states its own scope. +# +# ai-interaction · synthetic-media-marking · synthetic-text-marking +# emotion-recognition · deepfake-labelling · public-interest-text +# applicability: +# deepfake-labelling: false + +# Verifying text marking needs the parameters you generate with. Without them +# the text rule skips and says so — it never guesses. +# text_marking: +# method: synthid +# watermark_config: secrets/watermark_config.json + +rulepack: art50-eu-2026.07 + +report: + formats: [json, summary] # add `pdf` for the artefact an auditor reads + # sign_key: env:MARKPROOF_SIGNING_KEY +""" + + +@app.command() +def init( + path: Annotated[Path, typer.Option("--config", "-c", help="Where to write the config.")] = Path( + "markproof.yaml" + ), + url: Annotated[ + str, + typer.Option("--url", help="The chat endpoint to probe."), + ] = "https://api.example.com/v1/chat/completions", + name: Annotated[str, typer.Option("--name", help="A name for the target.")] = "my-service", + force: Annotated[bool, typer.Option("--force", help="Overwrite an existing file.")] = False, +) -> None: + """Write a starting markproof.yaml. + + Deliberately small. The scaffold configures one chat probe and leaves the + media, UI and text-marking blocks commented out with the reason each is + optional, because a config full of settings nobody chose is how people end up + running checks they cannot interpret. + """ + if path.exists() and not force: + err_console.print( + f"[bold red]error:[/] {path} already exists. Pass --force to overwrite it." + ) + raise typer.Exit(code=2) + + try: + path.write_text(_SCAFFOLD.format(name=name, url=url), encoding="utf-8") + except OSError as exc: + err_console.print(f"[bold red]error:[/] could not write {path}: {exc}") + raise typer.Exit(code=2) from exc + + console.print() + console.print(f" wrote [cyan]{path}[/cyan]") + console.print() + console.print(" Next:") + console.print(f" 1. set the url in {path} to the endpoint your users reach") + console.print(f" 2. [cyan]markproof run --config {path}[/cyan]") + console.print() + console.print( + " [dim]No endpoint yet? examples/demo-bot is a deliberately non-conformant one.[/dim]" + ) + console.print() + + @app.command() def keygen( out_dir: Annotated[ diff --git a/tests/test_cli.py b/tests/test_cli.py index 2faad08..26df03e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -440,3 +440,63 @@ def test_the_config_can_name_the_environment_variable( assert result.exit_code == 0, result.output report = json.loads((out / "report.json").read_text(encoding="utf-8")) assert "signature" in report, "the configured key was ignored" + + +class TestInit: + """The command the module docstring promised and the build did not have. + + A promised-but-absent command is worse than no command: it tells a reader the + documentation is not checked against the code, which for a tool selling + verification is the wrong first impression. + """ + + def test_it_writes_a_config_that_actually_loads(self, tmp_path: Path) -> None: + """A scaffold that fails validation would be worse than none at all.""" + from markproof.config import load_config + + target = tmp_path / "markproof.yaml" + result = RUNNER.invoke(app, ["init", "--config", str(target)]) + assert result.exit_code == 0, result.output + config = load_config(target) + assert config.target.probes + assert config.rulepack == "art50-eu-2026.07" + + def test_the_rulepack_it_names_is_one_that_ships(self, tmp_path: Path) -> None: + from markproof.config import load_config + + target = tmp_path / "markproof.yaml" + RUNNER.invoke(app, ["init", "--config", str(target)]) + packaged = Path(__file__).resolve().parent.parent / "src" / "markproof" / "rulepacks" + assert (packaged / f"{load_config(target).rulepack}.yaml").is_file() + + def test_it_refuses_to_clobber_an_existing_config(self, tmp_path: Path) -> None: + target = tmp_path / "markproof.yaml" + target.write_text("version: 1\n# hand-written\n", encoding="utf-8") + result = RUNNER.invoke(app, ["init", "--config", str(target)]) + assert result.exit_code == 2 + assert "hand-written" in target.read_text(encoding="utf-8") + + def test_force_overwrites(self, tmp_path: Path) -> None: + target = tmp_path / "markproof.yaml" + target.write_text("old\n", encoding="utf-8") + result = RUNNER.invoke(app, ["init", "--config", str(target), "--force"]) + assert result.exit_code == 0 + assert "old" not in target.read_text(encoding="utf-8") + + def test_the_url_and_name_reach_the_file(self, tmp_path: Path) -> None: + from markproof.config import load_config + + target = tmp_path / "markproof.yaml" + RUNNER.invoke( + app, + ["init", "--config", str(target), "--url", "https://x.test/v1/c", "--name", "prod-bot"], + ) + config = load_config(target) + assert config.target.name == "prod-bot" + assert config.target.probes[0].url == "https://x.test/v1/c" + + def test_it_points_somewhere_a_newcomer_can_actually_go(self, tmp_path: Path) -> None: + result = RUNNER.invoke(app, ["init", "--config", str(tmp_path / "markproof.yaml")]) + assert "demo-bot" in result.output, ( + "a first-time user with no endpoint of their own needs somewhere to point this" + )